From 6dae62b15b4fc59e331a3023190d868d4376dfb1 Mon Sep 17 00:00:00 2001 From: Fabiana Cecin Date: Fri, 22 May 2026 19:29:11 -0300 Subject: [PATCH] Separate core (Waku) and MessagingClient using nim-brokers (WIP) --- layers/logos_delivery.nim | 117 +++ layers/mounts.nim | 38 + liblogosdelivery/declare_lib.nim | 30 - liblogosdelivery/liblogosdelivery.h | 21 +- liblogosdelivery/liblogosdelivery.nim | 13 +- liblogosdelivery/liblogosdelivery_common.h | 13 + liblogosdelivery/liblogosdelivery_kernel.h | 72 ++ .../logos_delivery_api/debug_api.nim | 56 -- .../logos_delivery_api/messaging_api.nim | 91 -- .../logos_delivery_api/node_api.nim | 197 ---- .../json_connection_status_change_event.nim | 2 +- messaging/api.nim | 8 + messaging/api/api.nim | 11 + messaging/api/api_send.nim | 52 + messaging/api/api_subscribe.nim | 26 + messaging/api/api_unsubscribe.nim | 26 + messaging/api/events.nim | 40 + .../api/ffi}/json_event.nim | 8 +- messaging/api/ffi/messaging_ffi.nim | 374 ++++++++ messaging/api/messaging.nim | 37 + messaging/api/requests.nim | 6 + {waku => messaging}/api/send_api.md | 0 {waku => messaging}/api/types.nim | 8 +- .../delivery_service/delivery_service.nim | 39 + .../not_delivered_storage/migrations.nim | 4 +- .../not_delivered_storage.nim | 32 + .../delivery_service/recv_service.nim | 0 .../recv_service/recv_service.nim | 69 +- .../delivery_service/send_service.nim | 0 .../send_service/delivery_task.nim | 2 +- .../send_service/lightpush_processor.nim | 56 +- .../send_service/relay_processor.nim | 60 +- .../send_service/send_processor.nim | 0 .../send_service/send_service.nim | 110 +-- .../delivery_service/subscription_manager.nim | 0 messaging/messaging_client.nim | 118 +++ messaging/messaging_client_type.nim | 22 + tests/api/test_api_health.nim | 53 +- tests/api/test_api_receive.nim | 24 +- tests/api/test_api_send.nim | 95 +- tests/api/test_api_subscription.nim | 144 +-- tests/api/test_node_conf.nim | 48 - tests/node/test_wakunode_health_monitor.nim | 80 +- tests/test_waku.nim | 1 + tests/waku_discv5/test_waku_discv5.nim | 37 +- waku/api.nim | 7 +- waku/api/api.nim | 56 -- waku/api/api_conf.nim | 533 ----------- waku/api/events.nim | 7 + .../events/health.nim} | 8 +- waku/api/events/message.nim | 8 + waku/api/ffi/kernel_ffi.nim | 175 ++++ waku/api/ffi/kernel_helpers.nim | 72 ++ waku/api/requests.nim | 12 + waku/api/requests/filter.nim | 47 + waku/api/requests/health.nim | 42 + waku/api/requests/lightpush.nim | 26 + .../requests/node.nim} | 0 waku/api/requests/peers.nim | 35 + waku/api/requests/protocols.nim | 18 + waku/api/requests/relay.nim | 30 + .../rln_requests.nim => api/requests/rln.nim} | 0 waku/api/requests/store.nim | 22 + waku/api/requests/subscription.nim | 110 +++ waku/events/delivery_events.nim | 12 - waku/events/events.nim | 5 +- waku/events/message_events.nim | 34 - waku/events/peer_events.nim | 6 +- waku/factory/builder.nim | 3 + waku/factory/waku.nim | 63 +- .../delivery_service/delivery_service.nim | 44 - .../not_delivered_storage.nim | 38 - .../node/health_monitor/connection_status.nim | 9 +- .../health_monitor/node_health_monitor.nim | 12 +- waku/node/kernel_api/filter.nim | 6 +- waku/node/kernel_api/lightpush.nim | 5 +- waku/node/kernel_api/relay.nim | 15 +- waku/node/kernel_api/store.nim | 6 +- waku/node/peer_manager/peer_manager.nim | 8 +- waku/node/providers/filter.nim | 124 +++ waku/node/providers/lightpush.nim | 54 ++ waku/node/providers/relay.nim | 76 ++ waku/node/providers/store.nim | 56 ++ waku/node/subscription_manager.nim | 903 ++++++++++++++++++ waku/node/waku_node.nim | 119 ++- waku/node/waku_telemetry.nim | 17 + waku/requests/health_requests.nim | 42 +- waku/requests/requests.nim | 5 +- waku/rest_api/endpoint/health/types.nim | 2 +- waku/waku_filter_v2/client.nim | 6 +- waku/waku_relay/protocol.nim | 14 +- waku/waku_rln_relay/rln_relay.nim | 2 +- 92 files changed, 3525 insertions(+), 1609 deletions(-) create mode 100644 layers/logos_delivery.nim create mode 100644 layers/mounts.nim create mode 100644 liblogosdelivery/liblogosdelivery_common.h create mode 100644 liblogosdelivery/liblogosdelivery_kernel.h delete mode 100644 liblogosdelivery/logos_delivery_api/debug_api.nim delete mode 100644 liblogosdelivery/logos_delivery_api/messaging_api.nim delete mode 100644 liblogosdelivery/logos_delivery_api/node_api.nim create mode 100644 messaging/api.nim create mode 100644 messaging/api/api.nim create mode 100644 messaging/api/api_send.nim create mode 100644 messaging/api/api_subscribe.nim create mode 100644 messaging/api/api_unsubscribe.nim create mode 100644 messaging/api/events.nim rename {liblogosdelivery => messaging/api/ffi}/json_event.nim (63%) create mode 100644 messaging/api/ffi/messaging_ffi.nim create mode 100644 messaging/api/messaging.nim create mode 100644 messaging/api/requests.nim rename {waku => messaging}/api/send_api.md (100%) rename {waku => messaging}/api/types.nim (91%) create mode 100644 messaging/delivery_service/delivery_service.nim rename {waku/node => messaging}/delivery_service/not_delivered_storage/migrations.nim (82%) create mode 100644 messaging/delivery_service/not_delivered_storage/not_delivered_storage.nim rename {waku/node => messaging}/delivery_service/recv_service.nim (100%) rename {waku/node => messaging}/delivery_service/recv_service/recv_service.nim (75%) rename {waku/node => messaging}/delivery_service/send_service.nim (100%) rename {waku/node => messaging}/delivery_service/send_service/delivery_task.nim (96%) rename {waku/node => messaging}/delivery_service/send_service/lightpush_processor.nim (54%) rename {waku/node => messaging}/delivery_service/send_service/relay_processor.nim (60%) rename {waku/node => messaging}/delivery_service/send_service/send_processor.nim (100%) rename {waku/node => messaging}/delivery_service/send_service/send_service.nim (75%) rename {waku/node => messaging}/delivery_service/subscription_manager.nim (100%) create mode 100644 messaging/messaging_client.nim create mode 100644 messaging/messaging_client_type.nim delete mode 100644 waku/api/api_conf.nim create mode 100644 waku/api/events.nim rename waku/{events/health_events.nim => api/events/health.nim} (67%) create mode 100644 waku/api/events/message.nim create mode 100644 waku/api/ffi/kernel_ffi.nim create mode 100644 waku/api/ffi/kernel_helpers.nim create mode 100644 waku/api/requests.nim create mode 100644 waku/api/requests/filter.nim create mode 100644 waku/api/requests/health.nim create mode 100644 waku/api/requests/lightpush.nim rename waku/{requests/node_requests.nim => api/requests/node.nim} (100%) create mode 100644 waku/api/requests/peers.nim create mode 100644 waku/api/requests/protocols.nim create mode 100644 waku/api/requests/relay.nim rename waku/{requests/rln_requests.nim => api/requests/rln.nim} (100%) create mode 100644 waku/api/requests/store.nim create mode 100644 waku/api/requests/subscription.nim delete mode 100644 waku/events/delivery_events.nim delete mode 100644 waku/events/message_events.nim delete mode 100644 waku/node/delivery_service/delivery_service.nim delete mode 100644 waku/node/delivery_service/not_delivered_storage/not_delivered_storage.nim create mode 100644 waku/node/providers/filter.nim create mode 100644 waku/node/providers/lightpush.nim create mode 100644 waku/node/providers/relay.nim create mode 100644 waku/node/providers/store.nim create mode 100644 waku/node/subscription_manager.nim create mode 100644 waku/node/waku_telemetry.nim diff --git a/layers/logos_delivery.nim b/layers/logos_delivery.nim new file mode 100644 index 000000000..39373c857 --- /dev/null +++ b/layers/logos_delivery.nim @@ -0,0 +1,117 @@ +{.push raises: [].} + +import chronos, chronicles, results +import brokers/broker_context + +import waku/factory/waku +import layers/mounts +import messaging/messaging_client +import tools/confutils/cli_args + +export messaging_client + +logScope: + topics = "logos-delivery" + +type LogosDelivery* = ref object + brokerCtx*: BrokerContext + waku*: Waku + ## Kernel layer. Always present. + messaging*: MessagingClient + ## Messaging layer. `nil` in the kernel-only composition. + +# The composition is selected by the primary layer typedesc: +# `new(Waku, ...)` is kernel-only, `new(MessagingClient, ...)` is the full stack. + +proc new*( + T: type LogosDelivery, primary: typedesc[Waku], node: Waku +): Result[LogosDelivery, string] = + ## Kernel-only. Waku is the primary. + if node.isNil(): + return err("LogosDelivery.new(Waku): node is nil") + mountLayer(Waku, node.brokerCtx).isOkOr: + return err("mount Waku layer failed: " & error) + ok(LogosDelivery(brokerCtx: node.brokerCtx, waku: node, messaging: nil)) + +proc new*( + T: type LogosDelivery, primary: typedesc[MessagingClient], node: Waku +): Result[LogosDelivery, string] = + ## Messaging primary. Mounts the kernel, then the messaging layer on top; + ## rolls back the kernel gate if messaging fails to mount. + if node.isNil(): + return err("LogosDelivery.new(MessagingClient): node is nil") + mountLayer(Waku, node.brokerCtx).isOkOr: + return err("mount Waku layer failed: " & error) + let messaging = MessagingClient.new(node.brokerCtx, node.conf.p2pReliability) + mountLayer(MessagingClient, messaging.brokerCtx).isOkOr: + discard unmountLayer(Waku, node.brokerCtx) + return err("mount MessagingClient layer failed: " & error) + ok(LogosDelivery(brokerCtx: node.brokerCtx, waku: node, messaging: messaging)) + +proc new*( + T: type LogosDelivery, + primary: typedesc[MessagingClient], + preset: string, + mode: WakuMode, +): Future[Result[LogosDelivery, string]] {.async: (raises: []).} = + ## Messaging primary, kernel built from `(preset, mode)` defaults. + var conf = defaultWakuNodeConf().valueOr: + return err("defaultWakuNodeConf failed: " & error) + conf.preset = preset + conf.mode = mode + + let wakuConf = conf.toWakuConf().valueOr: + return err("toWakuConf failed: " & error) + + let w = + try: + (await Waku.new(wakuConf)).valueOr: + return err("Waku.new failed: " & $error) + except CatchableError as e: + return err("Waku.new raised: " & e.msg) + + return LogosDelivery.new(MessagingClient, w) + +proc start*(self: LogosDelivery): Future[Result[void, string]] {.async: (raises: []).} = + ## Kernel first (its broker providers must be live before messaging queries + ## protocol-mount status), then the messaging layer if present. + if self.isNil() or self.waku.isNil(): + return err("LogosDelivery.start: delivery/waku is nil") + + (await startWaku(addr self.waku)).isOkOr: + return err("startWaku failed: " & error) + + if not self.messaging.isNil(): + (await self.messaging.start()).isOkOr: + return err("MessagingClient.start failed: " & error) + + ok() + +proc stop*(self: LogosDelivery): Future[Result[void, string]] {.async: (raises: []).} = + ## Tear down in reverse: messaging (if present) then the kernel, releasing + ## each layer's gate. Best-effort: reports the first error. + if self.isNil(): + return err("LogosDelivery.stop: delivery is nil") + + var firstErr = "" + + if not self.messaging.isNil(): + (await self.messaging.stop()).isOkOr: + firstErr = "MessagingClient.stop failed: " & error + let unmountMsgRes = unmountLayer(MessagingClient, self.messaging.brokerCtx) + if unmountMsgRes.isErr() and firstErr.len == 0: + firstErr = "unmount MessagingClient layer failed: " & unmountMsgRes.error + + if not self.waku.isNil(): + let stopRes = await self.waku.stop() + if stopRes.isErr() and firstErr.len == 0: + firstErr = "Waku.stop failed: " & stopRes.error + let unmountRes = unmountLayer(Waku, self.waku.brokerCtx) + if unmountRes.isErr() and firstErr.len == 0: + firstErr = "unmount Waku layer failed: " & unmountRes.error + + if firstErr.len > 0: + return err(firstErr) + ok() + +{.pop.} diff --git a/layers/mounts.nim b/layers/mounts.nim new file mode 100644 index 000000000..61854f00f --- /dev/null +++ b/layers/mounts.nim @@ -0,0 +1,38 @@ +{.push raises: [].} + +## Per-(layer, broker-context) mount gate: at most one mount per +## `(layer typedesc T, BrokerContext)`. Does not bind RequestBroker providers. +## +## Per-thread storage (threadvar). + +import std/sets +import results +import brokers/broker_context + +export results + +type LayerKey = tuple[layerName: string, ctxId: uint32] + +var layerMounts {.threadvar.}: HashSet[LayerKey] + +proc isLayerMounted*(T: typedesc, ctx: BrokerContext): bool = + let key: LayerKey = ($T, ctx.uint32) + key in layerMounts + +proc mountLayer*(T: typedesc, ctx: BrokerContext): Result[void, string] = + ## Claim the (T, ctx) instance slot. Errors if already mounted. + let key: LayerKey = ($T, ctx.uint32) + if key in layerMounts: + return err($T & " is already mounted in broker context " & $ctx.uint32) + layerMounts.incl(key) + ok() + +proc unmountLayer*(T: typedesc, ctx: BrokerContext): Result[void, string] = + ## Release the (T, ctx) instance slot. Errors if not mounted. + let key: LayerKey = ($T, ctx.uint32) + if key notin layerMounts: + return err($T & " is not mounted in broker context " & $ctx.uint32) + layerMounts.excl(key) + ok() + +{.pop.} diff --git a/liblogosdelivery/declare_lib.nim b/liblogosdelivery/declare_lib.nim index 5087a0dee..3f2ca49e0 100644 --- a/liblogosdelivery/declare_lib.nim +++ b/liblogosdelivery/declare_lib.nim @@ -1,33 +1,3 @@ import ffi -import std/locks -import waku/factory/waku declareLibrary("logosdelivery") - -var eventCallbackLock: Lock -initLock(eventCallbackLock) - -template requireInitializedNode*( - ctx: ptr FFIContext[Waku], opName: string, onError: untyped -) = - if isNil(ctx): - let errMsg {.inject.} = opName & " failed: invalid context" - onError - elif isNil(ctx.myLib) or isNil(ctx.myLib[]): - let errMsg {.inject.} = opName & " failed: node is not initialized" - onError - -proc logosdelivery_set_event_callback( - ctx: ptr FFIContext[Waku], callback: FFICallBack, userData: pointer -) {.dynlib, exportc, cdecl.} = - if isNil(ctx): - echo "error: invalid context in logosdelivery_set_event_callback" - return - - # prevent race conditions that might happen due incorrect usage. - eventCallbackLock.acquire() - defer: - eventCallbackLock.release() - - ctx[].eventCallback = cast[pointer](callback) - ctx[].eventUserData = userData diff --git a/liblogosdelivery/liblogosdelivery.h b/liblogosdelivery/liblogosdelivery.h index 5092db9f2..55f6f8dd9 100644 --- a/liblogosdelivery/liblogosdelivery.h +++ b/liblogosdelivery/liblogosdelivery.h @@ -5,21 +5,13 @@ #ifndef __liblogosdelivery__ #define __liblogosdelivery__ -#include -#include - -// The possible returned values for the functions that return int -#define RET_OK 0 -#define RET_ERR 1 -#define RET_MISSING_CALLBACK 2 +#include "liblogosdelivery_common.h" #ifdef __cplusplus extern "C" { #endif - typedef void (*FFICallBack)(int callerRet, const char *msg, size_t len, void *userData); - // Creates a new instance of the node from the given configuration JSON. // Returns a pointer to the Context needed by the rest of the API functions. // Configuration should be in JSON format using WakuNodeConf field names. @@ -30,6 +22,15 @@ extern "C" FFICallBack callback, void *userData); + // Creates a new node from a (preset, mode) shorthand (App-Dev entry point). + // preset: network preset string (e.g. "twn", "logos.dev", ""). + // mode: WakuMode string ("Core" or "Edge"). + void *logosdelivery_create_node_preset_mode( + const char *preset, + const char *mode, + FFICallBack callback, + void *userData); + // Starts the node. int logosdelivery_start_node(void *ctx, FFICallBack callback, @@ -40,7 +41,7 @@ extern "C" FFICallBack callback, void *userData); - // Destroys an instance of a node created with logosdelivery_create_node + // Destroys an instance of a node created with a logosdelivery_create_node... API int logosdelivery_destroy(void *ctx, FFICallBack callback, void *userData); diff --git a/liblogosdelivery/liblogosdelivery.nim b/liblogosdelivery/liblogosdelivery.nim index fc907498a..494302a74 100644 --- a/liblogosdelivery/liblogosdelivery.nim +++ b/liblogosdelivery/liblogosdelivery.nim @@ -1,11 +1,6 @@ -import std/[atomics, options] -import chronicles, chronos, chronos/threadsync, ffi -import waku/factory/waku, waku/node/waku_node, ./declare_lib - -################################################################################ -## Include different APIs, i.e. all procs with {.ffi.} pragma +import ffi +import ./declare_lib include - ./logos_delivery_api/node_api, - ./logos_delivery_api/messaging_api, - ./logos_delivery_api/debug_api + waku/api/ffi/kernel_ffi, + messaging/api/ffi/messaging_ffi diff --git a/liblogosdelivery/liblogosdelivery_common.h b/liblogosdelivery/liblogosdelivery_common.h new file mode 100644 index 000000000..fcb8abb8e --- /dev/null +++ b/liblogosdelivery/liblogosdelivery_common.h @@ -0,0 +1,13 @@ +#pragma once +#ifndef LOGOSDELIVERY_COMMON_DEFS +#define LOGOSDELIVERY_COMMON_DEFS + +#include +#include + +#define RET_OK 0 +#define RET_ERR 1 +#define RET_MISSING_CALLBACK 2 +typedef void (*FFICallBack)(int callerRet, const char *msg, size_t len, void *userData); + +#endif diff --git a/liblogosdelivery/liblogosdelivery_kernel.h b/liblogosdelivery/liblogosdelivery_kernel.h new file mode 100644 index 000000000..2f6912760 --- /dev/null +++ b/liblogosdelivery/liblogosdelivery_kernel.h @@ -0,0 +1,72 @@ +// Low-level library interfaces +// NOTE: This interface is unsupported and may be changed at any time +#pragma once +#ifndef __liblogosdelivery_kernel__ +#define __liblogosdelivery_kernel__ + +#include "liblogosdelivery_common.h" + +#ifdef __cplusplus +extern "C" +{ +#endif + + // Creates a new Waku node from a JSON WakuNodeConf blob. + // Returns an opaque handle (NULL on failure). Configuration field names match + // Nim identifiers from WakuNodeConf (case-insensitive; unknown fields rejected). + void *waku_new(const char *configJson, + FFICallBack callback, + void *userData); + + // Starts the Waku node. + int waku_start(void *ctx, + FFICallBack callback, + void *userData); + + // Stops the Waku node. + int waku_stop(void *ctx, + FFICallBack callback, + void *userData); + + // Subscribes the relay mesh to a shard (pubsub topic). A shard stays + // subscribed while a direct shard subscription OR any content-topic interest + // holds it. + int waku_relay_subscribe_shard(void *ctx, + FFICallBack callback, + void *userData, + const char *pubsubTopic); + + // Removes the direct shard subscription. The pubsub topic is only torn down + // if no content-topic interest still holds it. + int waku_relay_unsubscribe_shard(void *ctx, + FFICallBack callback, + void *userData, + const char *pubsubTopic); + + // Subscribes to a content topic. pubsubTopic is the optional shard: pass an + // empty string ("") to derive it via auto-sharding; under static/manual + // sharding a non-empty shard must be supplied. + int waku_relay_subscribe_content_topic(void *ctx, + FFICallBack callback, + void *userData, + const char *contentTopic, + const char *pubsubTopic); + + // Unsubscribes from a content topic. pubsubTopic is the optional shard, same + // convention as waku_relay_subscribe_content_topic. + int waku_relay_unsubscribe_content_topic(void *ctx, + FFICallBack callback, + void *userData, + const char *contentTopic, + const char *pubsubTopic); + + // Destroys a Waku node previously created with waku_new. + int waku_destroy(void *ctx, + FFICallBack callback, + void *userData); + +#ifdef __cplusplus +} +#endif + +#endif /* __liblogosdelivery_kernel__ */ diff --git a/liblogosdelivery/logos_delivery_api/debug_api.nim b/liblogosdelivery/logos_delivery_api/debug_api.nim deleted file mode 100644 index bb66a0e3f..000000000 --- a/liblogosdelivery/logos_delivery_api/debug_api.nim +++ /dev/null @@ -1,56 +0,0 @@ -import std/[json, strutils] -import waku/factory/waku_state_info -import tools/confutils/[cli_args, config_option_meta] - -proc logosdelivery_get_available_node_info_ids( - ctx: ptr FFIContext[Waku], callback: FFICallBack, userData: pointer -) {.ffi.} = - ## Returns the list of all available node info item ids that - ## can be queried with `get_node_info_item`. - requireInitializedNode(ctx, "GetNodeInfoIds"): - return err(errMsg) - - return ok($ctx.myLib[].stateInfo.getAllPossibleInfoItemIds()) - -proc logosdelivery_get_node_info( - ctx: ptr FFIContext[Waku], - callback: FFICallBack, - userData: pointer, - nodeInfoId: cstring, -) {.ffi.} = - ## Returns the content of the node info item with the given id if it exists. - requireInitializedNode(ctx, "GetNodeInfoItem"): - return err(errMsg) - - let infoItemIdEnum = - try: - parseEnum[NodeInfoId]($nodeInfoId) - except ValueError: - return err("Invalid node info id: " & $nodeInfoId) - - return ok(ctx.myLib[].stateInfo.getNodeInfoItem(infoItemIdEnum)) - -proc logosdelivery_get_available_configs( - ctx: ptr FFIContext[Waku], callback: FFICallBack, userData: pointer -) {.ffi.} = - ## Returns information about the accepted config items. - requireInitializedNode(ctx, "GetAvailableConfigs"): - return err(errMsg) - - let optionMetas: seq[ConfigOptionMeta] = extractConfigOptionMeta(WakuNodeConf) - var configOptionDetails = newJArray() - - # for confField, confValue in fieldPairs(conf): - # defaultConfig[confField] = $confValue - - for meta in optionMetas: - configOptionDetails.add( - %*{ - meta.fieldName: meta.typeName & "(" & meta.defaultValue & ")", "desc": meta.desc - } - ) - - var jsonNode = newJObject() - jsonNode["configOptions"] = configOptionDetails - let asString = pretty(jsonNode) - return ok(pretty(jsonNode)) diff --git a/liblogosdelivery/logos_delivery_api/messaging_api.nim b/liblogosdelivery/logos_delivery_api/messaging_api.nim deleted file mode 100644 index cb2771034..000000000 --- a/liblogosdelivery/logos_delivery_api/messaging_api.nim +++ /dev/null @@ -1,91 +0,0 @@ -import std/[json] -import chronos, results, ffi -import stew/byteutils -import - waku/common/base64, - waku/factory/waku, - waku/waku_core/topics/content_topic, - waku/api/[api, types], - ../declare_lib - -proc logosdelivery_subscribe( - ctx: ptr FFIContext[Waku], - callback: FFICallBack, - userData: pointer, - contentTopicStr: cstring, -) {.ffi.} = - requireInitializedNode(ctx, "Subscribe"): - return err(errMsg) - - # ContentTopic is just a string type alias - let contentTopic = ContentTopic($contentTopicStr) - - (await api.subscribe(ctx.myLib[], contentTopic)).isOkOr: - let errMsg = $error - return err("Subscribe failed: " & errMsg) - - return ok("") - -proc logosdelivery_unsubscribe( - ctx: ptr FFIContext[Waku], - callback: FFICallBack, - userData: pointer, - contentTopicStr: cstring, -) {.ffi.} = - requireInitializedNode(ctx, "Unsubscribe"): - return err(errMsg) - - # ContentTopic is just a string type alias - let contentTopic = ContentTopic($contentTopicStr) - - api.unsubscribe(ctx.myLib[], contentTopic).isOkOr: - let errMsg = $error - return err("Unsubscribe failed: " & errMsg) - - return ok("") - -proc logosdelivery_send( - ctx: ptr FFIContext[Waku], - callback: FFICallBack, - userData: pointer, - messageJson: cstring, -) {.ffi.} = - requireInitializedNode(ctx, "Send"): - return err(errMsg) - - ## Parse the message JSON and send the message - var jsonNode: JsonNode - try: - jsonNode = parseJson($messageJson) - except Exception as e: - return err("Failed to parse message JSON: " & e.msg) - - # Extract content topic - if not jsonNode.hasKey("contentTopic"): - return err("Missing contentTopic field") - - # ContentTopic is just a string type alias - let contentTopic = ContentTopic(jsonNode["contentTopic"].getStr()) - - # Extract payload (expect base64 encoded string) - if not jsonNode.hasKey("payload"): - return err("Missing payload field") - - let payloadStr = jsonNode["payload"].getStr() - let payload = base64.decode(Base64String(payloadStr)).valueOr: - return err("invalid payload format: " & error) - - # Extract ephemeral flag - let ephemeral = jsonNode.getOrDefault("ephemeral").getBool(false) - - # Create message envelope - let envelope = MessageEnvelope.init( - contentTopic = contentTopic, payload = payload, ephemeral = ephemeral - ) - - # Send the message - let requestId = (await api.send(ctx.myLib[], envelope)).valueOr: - let errMsg = $error - return err("Send failed: " & errMsg) - - return ok($requestId) diff --git a/liblogosdelivery/logos_delivery_api/node_api.nim b/liblogosdelivery/logos_delivery_api/node_api.nim deleted file mode 100644 index 2e30d1b43..000000000 --- a/liblogosdelivery/logos_delivery_api/node_api.nim +++ /dev/null @@ -1,197 +0,0 @@ -import std/[json, strutils, tables] -import chronos, chronicles, results, confutils, confutils/std/net, ffi -import - waku/factory/waku, - waku/node/waku_node, - waku/api/[api, types], - waku/events/[message_events, health_events], - tools/confutils/cli_args, - ../declare_lib, - ../json_event - -# Add JSON serialization for RequestId -proc `%`*(id: RequestId): JsonNode = - %($id) - -registerReqFFI(CreateNodeRequest, ctx: ptr FFIContext[Waku]): - proc(configJson: cstring): Future[Result[string, string]] {.async.} = - ## Parse the JSON configuration using fieldPairs approach (WakuNodeConf) - var conf = defaultWakuNodeConf().valueOr: - return err("Failed creating default conf: " & error) - - var jsonNode: JsonNode - try: - jsonNode = parseJson($configJson) - except Exception: - let exceptionMsg = getCurrentExceptionMsg() - error "Failed to parse config JSON", - error = exceptionMsg, configJson = $configJson - return err( - "Failed to parse config JSON: " & exceptionMsg & " configJson string: " & - $configJson - ) - - var jsonFields: Table[string, (string, JsonNode)] - for key, value in jsonNode: - let lowerKey = key.toLowerAscii() - - if jsonFields.hasKey(lowerKey): - error "Duplicate configuration option found when normalized to lowercase", - key = key - return err( - "Duplicate configuration option found when normalized to lowercase: '" & key & - "'" - ) - - jsonFields[lowerKey] = (key, value) - - for confField, confValue in fieldPairs(conf): - let lowerField = confField.toLowerAscii() - if jsonFields.hasKey(lowerField): - let (jsonKey, jsonValue) = jsonFields[lowerField] - let formattedString = ($jsonValue).strip(chars = {'\"'}) - try: - confValue = parseCmdArg(typeof(confValue), formattedString) - except Exception: - return err( - "Failed to parse field '" & confField & "' from JSON key '" & jsonKey & "': " & - getCurrentExceptionMsg() & ". Value: " & formattedString - ) - - jsonFields.del(lowerField) - - if jsonFields.len > 0: - var unknownKeys = newSeq[string]() - for _, (jsonKey, _) in pairs(jsonFields): - unknownKeys.add(jsonKey) - error "Unrecognized configuration option(s) found", option = unknownKeys - return err("Unrecognized configuration option(s) found: " & $unknownKeys) - - # Create the node - ctx.myLib[] = (await api.createNode(conf)).valueOr: - let errMsg = $error - chronicles.error "CreateNodeRequest failed", err = errMsg - return err(errMsg) - - return ok("") - -proc logosdelivery_destroy( - ctx: ptr FFIContext[Waku], callback: FFICallBack, userData: pointer -): cint {.dynlib, exportc, cdecl.} = - initializeLibrary() - checkParams(ctx, callback, userData) - - ffi.destroyFFIContext(ctx).isOkOr: - let msg = "liblogosdelivery error: " & $error - callback(RET_ERR, unsafeAddr msg[0], cast[csize_t](len(msg)), userData) - return RET_ERR - - ## always need to invoke the callback although we don't retrieve value to the caller - callback(RET_OK, nil, 0, userData) - - return RET_OK - -proc logosdelivery_create_node( - configJson: cstring, callback: FFICallback, userData: pointer -): pointer {.dynlib, exportc, cdecl.} = - initializeLibrary() - - if isNil(callback): - echo "error: missing callback in logosdelivery_create_node" - return nil - - var ctx = ffi.createFFIContext[Waku]().valueOr: - let msg = "Error in createFFIContext: " & $error - callback(RET_ERR, unsafeAddr msg[0], cast[csize_t](len(msg)), userData) - return nil - - ctx.userData = userData - - ffi.sendRequestToFFIThread( - ctx, CreateNodeRequest.ffiNewReq(callback, userData, configJson) - ).isOkOr: - let msg = "error in sendRequestToFFIThread: " & $error - callback(RET_ERR, unsafeAddr msg[0], cast[csize_t](len(msg)), userData) - # free allocated resources as they won't be available - ffi.destroyFFIContext(ctx).isOkOr: - chronicles.error "Error in destroyFFIContext after sendRequestToFFIThread during creation", - err = $error - return nil - - return ctx - -proc logosdelivery_start_node( - ctx: ptr FFIContext[Waku], callback: FFICallBack, userData: pointer -) {.ffi.} = - requireInitializedNode(ctx, "START_NODE"): - return err(errMsg) - - # setting up outgoing event listeners - let sentListener = MessageSentEvent.listen( - ctx.myLib[].brokerCtx, - proc(event: MessageSentEvent) {.async: (raises: []).} = - callEventCallback(ctx, "onMessageSent"): - $newJsonEvent("message_sent", event), - ).valueOr: - chronicles.error "MessageSentEvent.listen failed", err = $error - return err("MessageSentEvent.listen failed: " & $error) - - let errorListener = MessageErrorEvent.listen( - ctx.myLib[].brokerCtx, - proc(event: MessageErrorEvent) {.async: (raises: []).} = - callEventCallback(ctx, "onMessageError"): - $newJsonEvent("message_error", event), - ).valueOr: - chronicles.error "MessageErrorEvent.listen failed", err = $error - return err("MessageErrorEvent.listen failed: " & $error) - - let propagatedListener = MessagePropagatedEvent.listen( - ctx.myLib[].brokerCtx, - proc(event: MessagePropagatedEvent) {.async: (raises: []).} = - callEventCallback(ctx, "onMessagePropagated"): - $newJsonEvent("message_propagated", event), - ).valueOr: - chronicles.error "MessagePropagatedEvent.listen failed", err = $error - return err("MessagePropagatedEvent.listen failed: " & $error) - - let receivedListener = MessageReceivedEvent.listen( - ctx.myLib[].brokerCtx, - proc(event: MessageReceivedEvent) {.async: (raises: []).} = - callEventCallback(ctx, "onMessageReceived"): - $newJsonEvent("message_received", event), - ).valueOr: - chronicles.error "MessageReceivedEvent.listen failed", err = $error - return err("MessageReceivedEvent.listen failed: " & $error) - - let ConnectionStatusChangeListener = EventConnectionStatusChange.listen( - ctx.myLib[].brokerCtx, - proc(event: EventConnectionStatusChange) {.async: (raises: []).} = - callEventCallback(ctx, "onConnectionStatusChange"): - $newJsonEvent("connection_status_change", event), - ).valueOr: - chronicles.error "ConnectionStatusChange.listen failed", err = $error - return err("ConnectionStatusChange.listen failed: " & $error) - - (await startWaku(addr ctx.myLib[])).isOkOr: - let errMsg = $error - chronicles.error "START_NODE failed", err = errMsg - return err("failed to start: " & errMsg) - return ok("") - -proc logosdelivery_stop_node( - ctx: ptr FFIContext[Waku], callback: FFICallBack, userData: pointer -) {.ffi.} = - requireInitializedNode(ctx, "STOP_NODE"): - return err(errMsg) - - await MessageErrorEvent.dropAllListeners(ctx.myLib[].brokerCtx) - await MessageSentEvent.dropAllListeners(ctx.myLib[].brokerCtx) - await MessagePropagatedEvent.dropAllListeners(ctx.myLib[].brokerCtx) - await MessageReceivedEvent.dropAllListeners(ctx.myLib[].brokerCtx) - await EventConnectionStatusChange.dropAllListeners(ctx.myLib[].brokerCtx) - - (await ctx.myLib[].stop()).isOkOr: - let errMsg = $error - chronicles.error "STOP_NODE failed", err = errMsg - return err("failed to stop: " & errMsg) - return ok("") diff --git a/library/events/json_connection_status_change_event.nim b/library/events/json_connection_status_change_event.nim index 86bfda780..b7de08c5b 100644 --- a/library/events/json_connection_status_change_event.nim +++ b/library/events/json_connection_status_change_event.nim @@ -2,7 +2,7 @@ import system, std/json import ./json_base_event -import ../../waku/api/types +import ../../waku/node/health_monitor/connection_status type JsonConnectionStatusChangeEvent* = ref object of JsonEvent status*: ConnectionStatus diff --git a/messaging/api.nim b/messaging/api.nim new file mode 100644 index 000000000..fea881b02 --- /dev/null +++ b/messaging/api.nim @@ -0,0 +1,8 @@ +{.push raises: [].} + +import ./api/requests +import ./api/events + +export requests, events + +{.pop.} diff --git a/messaging/api/api.nim b/messaging/api/api.nim new file mode 100644 index 000000000..ae115b41e --- /dev/null +++ b/messaging/api/api.nim @@ -0,0 +1,11 @@ +{.push raises: [].} + +import ./api_subscribe +import ./api_unsubscribe +import ./api_send + +export api_subscribe +export api_unsubscribe +export api_send + +{.pop.} diff --git a/messaging/api/api_send.nim b/messaging/api/api_send.nim new file mode 100644 index 000000000..3f512b930 --- /dev/null +++ b/messaging/api/api_send.nim @@ -0,0 +1,52 @@ +{.push raises: [].} + +import chronos, chronicles, results +import std/options +import stew/byteutils +import waku/waku_core +import waku/api/requests/subscription as kernel_subscription_api +import messaging/messaging_client_type +import messaging/delivery_service/delivery_service +import messaging/delivery_service/send_service +import messaging/delivery_service/send_service/delivery_task +import ./api_subscribe +import ./types + +logScope: + topics = "messaging-api send" + +proc send*( + client: MessagingClient, envelope: MessageEnvelope +): Future[Result[RequestId, string]] {.async: (raises: []).} = + ## Send a message envelope. Auto-subscribes to the content topic if needed. + ## Returns a RequestId for tracking via the message-lifecycle events. + if client.isNil() or client.deliveryService.isNil(): + return err("MessagingClient.send: client/deliveryService is nil") + + let subR = kernel_subscription_api.RequestIsSubscribed.request( + client.brokerCtx, envelope.contentTopic, none[PubsubTopic]() + ) + let isSubbed = subR.isOk() and subR.get().subscribed + if not isSubbed: + info "Auto-subscribing to topic on send", contentTopic = envelope.contentTopic + (await subscribe(client, envelope.contentTopic)).isOkOr: + warn "Failed to auto-subscribe", error = error + return err("Failed to auto-subscribe before sending: " & error) + + let requestId = RequestId.new(client.rng) + let deliveryTask = DeliveryTask.new( + requestId, envelope, client.brokerCtx + ).valueOr: + return err("MessagingClient.send: failed to create delivery task: " & error) + + info "MessagingClient.send: scheduling delivery task", + requestId = $requestId, + pubsubTopic = deliveryTask.pubsubTopic, + contentTopic = deliveryTask.msg.contentTopic, + msgHash = deliveryTask.msgHash.to0xHex() + + asyncSpawn client.deliveryService.sendService.send(deliveryTask) + + return ok(requestId) + +{.pop.} diff --git a/messaging/api/api_subscribe.nim b/messaging/api/api_subscribe.nim new file mode 100644 index 000000000..44d21a681 --- /dev/null +++ b/messaging/api/api_subscribe.nim @@ -0,0 +1,26 @@ +{.push raises: [].} + +import chronos, results +import std/options +import waku/waku_core/[topics/content_topic, topics/pubsub_topic] +import waku/api/requests/subscription as kernel_subscription_api +import messaging/messaging_client_type + +proc subscribe*( + client: MessagingClient, contentTopic: ContentTopic +): Future[Result[void, string]] {.async: (raises: []).} = + if client.isNil(): + return err("MessagingClient.subscribe: client is nil") + if client.relayMounted: + kernel_subscription_api.RequestRelaySubscribeContentTopic.request( + client.brokerCtx, contentTopic, none[PubsubTopic]() + ).isOkOr: + return err(error) + else: + kernel_subscription_api.RequestEdgeSubscribe.request( + client.brokerCtx, contentTopic, none[PubsubTopic]() + ).isOkOr: + return err(error) + return ok() + +{.pop.} diff --git a/messaging/api/api_unsubscribe.nim b/messaging/api/api_unsubscribe.nim new file mode 100644 index 000000000..d0f7f49e1 --- /dev/null +++ b/messaging/api/api_unsubscribe.nim @@ -0,0 +1,26 @@ +{.push raises: [].} + +import results +import std/options +import waku/waku_core/[topics/content_topic, topics/pubsub_topic] +import waku/api/requests/subscription as kernel_subscription_api +import messaging/messaging_client_type + +proc unsubscribe*( + client: MessagingClient, contentTopic: ContentTopic +): Result[void, string] = + if client.isNil(): + return err("MessagingClient.unsubscribe: client is nil") + if client.relayMounted: + kernel_subscription_api.RequestRelayUnsubscribeContentTopic.request( + client.brokerCtx, contentTopic, none[PubsubTopic]() + ).isOkOr: + return err(error) + else: + kernel_subscription_api.RequestEdgeUnsubscribe.request( + client.brokerCtx, contentTopic, none[PubsubTopic]() + ).isOkOr: + return err(error) + return ok() + +{.pop.} diff --git a/messaging/api/events.nim b/messaging/api/events.nim new file mode 100644 index 000000000..506a0c780 --- /dev/null +++ b/messaging/api/events.nim @@ -0,0 +1,40 @@ +{.push raises: [].} + +## Messaging API event types. Re-exports the waku-tier event types too. + +import brokers/event_broker +import waku/waku_core +import waku/api/events/message +import waku/api/events/health +import ./types + +export message +export health +export types + +EventBroker: + # Emitted when a message is sent to the network. + type MessageSentEvent* = object + requestId*: RequestId + messageHash*: string + +EventBroker: + # Emitted when a message send operation fails. + type MessageErrorEvent* = object + requestId*: RequestId + messageHash*: string + error*: string + +EventBroker: + # Emitted when a message is delivered to neighbouring nodes. + type MessagePropagatedEvent* = object + requestId*: RequestId + messageHash*: string + +EventBroker: + # Emitted when a message is received via Waku. + type MessageReceivedEvent* = object + messageHash*: string + message*: WakuMessage + +{.pop.} diff --git a/liblogosdelivery/json_event.nim b/messaging/api/ffi/json_event.nim similarity index 63% rename from liblogosdelivery/json_event.nim rename to messaging/api/ffi/json_event.nim index 389e29120..ea39d28ff 100644 --- a/liblogosdelivery/json_event.nim +++ b/messaging/api/ffi/json_event.nim @@ -5,13 +5,12 @@ type JsonEvent*[T] = ref object payload*: T macro toFlatJson*(event: JsonEvent): JsonNode = - ## Serializes JsonEvent[T] to flat JSON with eventType first, - ## followed by all fields from T's payload + ## Serialize JsonEvent[T] to flat JSON: eventType first, then T's payload fields. result = quote: var jsonObj = newJObject() jsonObj["eventType"] = %`event`.eventType - # Serialize payload fields into the same object (flattening) + # Flatten payload fields into the same object. let payloadJson = %`event`.payload for key, val in payloadJson.pairs: jsonObj[key] = val @@ -22,6 +21,5 @@ proc `$`*[T](event: JsonEvent[T]): string = $toFlatJson(event) proc newJsonEvent*[T](eventType: string, payload: T): JsonEvent[T] = - ## Creates a new JsonEvent with the given eventType and payload. - ## The payload's fields will be flattened into the JSON output. + ## New JsonEvent with the given eventType and payload. JsonEvent[T](eventType: eventType, payload: payload) diff --git a/messaging/api/ffi/messaging_ffi.nim b/messaging/api/ffi/messaging_ffi.nim new file mode 100644 index 000000000..e0720dceb --- /dev/null +++ b/messaging/api/ffi/messaging_ffi.nim @@ -0,0 +1,374 @@ +## FFI surface for `liblogosdelivery.so`. Exported C functions use the +## `logosdelivery_*` prefix; C declarations live in `liblogosdelivery.h`. + +import std/[json, locks, strutils, tables] +import chronos, chronicles, results, ffi +import stew/byteutils +import waku/common/base64 +import waku/factory/waku +import waku/factory/waku_state_info +import waku/api/ffi/kernel_helpers +import waku/waku_core/topics/content_topic +import layers/logos_delivery +import messaging/api/types +import messaging/api/events +import messaging/api/messaging as messaging_brokers +import tools/confutils/cli_args +import tools/confutils/config_option_meta +import messaging/api/ffi/json_event + +# `RequestId` is rendered via `$`. +proc `%`*(id: RequestId): JsonNode = + %($id) + +var eventCallbackLock: Lock +initLock(eventCallbackLock) + +# Event listener handles registered at start, kept per broker context so stop +# drops exactly these. +type MessagingFFIListeners = object + sent: MessageSentEventListener + error: MessageErrorEventListener + propagated: MessagePropagatedEventListener + received: MessageReceivedEventListener + connStatus: EventConnectionStatusChangeListener + +var ffiListeners {.threadvar.}: Table[uint32, MessagingFFIListeners] + +template requireInitializedMessaging( + ctx: ptr FFIContext[LogosDelivery], opName: string, onError: untyped +) = + if isNil(ctx): + let errMsg {.inject.} = opName & " failed: invalid context" + onError + elif isNil(ctx.myLib) or isNil(ctx.myLib[]): + let errMsg {.inject.} = opName & " failed: client is not initialized" + onError + +# ---- Construction requests (run on the FFI worker thread) ---- + +registerReqFFI(CreateMessagingClientByPresetMode, ctx: ptr FFIContext[LogosDelivery]): + proc(preset: cstring, mode: cstring): Future[Result[string, string]] {.async.} = + let modeEnum = + try: + parseEnum[WakuMode]($mode) + except ValueError: + return err("Invalid mode value: " & $mode) + ctx.myLib[] = (await LogosDelivery.new(MessagingClient, $preset, modeEnum)).valueOr: + chronicles.error "CreateMessagingClientByPresetMode failed", err = error + return err(error) + return ok("") + +registerReqFFI(CreateMessagingClientByConf, ctx: ptr FFIContext[LogosDelivery]): + proc(configJson: cstring): Future[Result[string, string]] {.async.} = + let waku = (await createWakuFromJson(configJson)).valueOr: + chronicles.error "CreateMessagingClientByConf: createWakuFromJson failed", + err = error + return err(error) + ctx.myLib[] = LogosDelivery.new(MessagingClient, waku).valueOr: + chronicles.error "CreateMessagingClientByConf: LogosDelivery.new failed", + err = error + return err(error) + return ok("") + +# ---- C exports ---- + +proc logosdelivery_create_node_preset_mode( + preset: cstring, mode: cstring, callback: FFICallback, userData: pointer +): pointer {.dynlib, exportc, cdecl.} = + ## Create a node from a preset name and mode string. + initializeLibrary() + + if isNil(callback): + echo "error: missing callback in logosdelivery_create_node_preset_mode" + return nil + + var ctx = ffi.createFFIContext[LogosDelivery]().valueOr: + let msg = "Error in createFFIContext: " & $error + callback(RET_ERR, unsafeAddr msg[0], cast[csize_t](len(msg)), userData) + return nil + + ctx.userData = userData + + ffi.sendRequestToFFIThread( + ctx, + CreateMessagingClientByPresetMode.ffiNewReq(callback, userData, preset, mode), + ).isOkOr: + let msg = "error in sendRequestToFFIThread: " & $error + callback(RET_ERR, unsafeAddr msg[0], cast[csize_t](len(msg)), userData) + ffi.destroyFFIContext(ctx).isOkOr: + chronicles.error "destroyFFIContext failed after sendRequestToFFIThread error", + err = $error + return nil + + return ctx + +proc logosdelivery_create_node( + configJson: cstring, callback: FFICallback, userData: pointer +): pointer {.dynlib, exportc, cdecl.} = + initializeLibrary() + + if isNil(callback): + echo "error: missing callback in logosdelivery_create_node" + return nil + + var ctx = ffi.createFFIContext[LogosDelivery]().valueOr: + let msg = "Error in createFFIContext: " & $error + callback(RET_ERR, unsafeAddr msg[0], cast[csize_t](len(msg)), userData) + return nil + + ctx.userData = userData + + ffi.sendRequestToFFIThread( + ctx, CreateMessagingClientByConf.ffiNewReq(callback, userData, configJson) + ).isOkOr: + let msg = "error in sendRequestToFFIThread: " & $error + callback(RET_ERR, unsafeAddr msg[0], cast[csize_t](len(msg)), userData) + ffi.destroyFFIContext(ctx).isOkOr: + chronicles.error "destroyFFIContext failed after sendRequestToFFIThread error", + err = $error + return nil + + return ctx + +proc logosdelivery_destroy( + ctx: ptr FFIContext[LogosDelivery], callback: FFICallBack, userData: pointer +): cint {.dynlib, exportc, cdecl.} = + initializeLibrary() + checkParams(ctx, callback, userData) + + ffi.destroyFFIContext(ctx).isOkOr: + let msg = "logosdelivery_destroy error: " & $error + callback(RET_ERR, unsafeAddr msg[0], cast[csize_t](len(msg)), userData) + return RET_ERR + + callback(RET_OK, nil, 0, userData) + return RET_OK + +proc logosdelivery_set_event_callback( + ctx: ptr FFIContext[LogosDelivery], callback: FFICallBack, userData: pointer +) {.dynlib, exportc, cdecl.} = + if isNil(ctx): + echo "error: invalid context in logosdelivery_set_event_callback" + return + eventCallbackLock.acquire() + defer: + eventCallbackLock.release() + ctx[].eventCallback = cast[pointer](callback) + ctx[].eventUserData = userData + +# ---- Lifecycle: start (register event listeners + MessagingClient.start) ---- + +proc logosdelivery_start_node( + ctx: ptr FFIContext[LogosDelivery], callback: FFICallBack, userData: pointer +) {.ffi.} = + requireInitializedMessaging(ctx, "logosdelivery_start_node"): + return err(errMsg) + + let brokerCtx = ctx.myLib[].brokerCtx + + let sentListener = MessageSentEvent.listen( + brokerCtx, + proc(event: MessageSentEvent) {.async: (raises: []).} = + callEventCallback(ctx, "onMessageSent"): + $newJsonEvent("message_sent", event), + ).valueOr: + chronicles.error "MessageSentEvent.listen failed", err = $error + return err("MessageSentEvent.listen failed: " & $error) + + let errorListener = MessageErrorEvent.listen( + brokerCtx, + proc(event: MessageErrorEvent) {.async: (raises: []).} = + callEventCallback(ctx, "onMessageError"): + $newJsonEvent("message_error", event), + ).valueOr: + chronicles.error "MessageErrorEvent.listen failed", err = $error + return err("MessageErrorEvent.listen failed: " & $error) + + let propagatedListener = MessagePropagatedEvent.listen( + brokerCtx, + proc(event: MessagePropagatedEvent) {.async: (raises: []).} = + callEventCallback(ctx, "onMessagePropagated"): + $newJsonEvent("message_propagated", event), + ).valueOr: + chronicles.error "MessagePropagatedEvent.listen failed", err = $error + return err("MessagePropagatedEvent.listen failed: " & $error) + + let receivedListener = MessageReceivedEvent.listen( + brokerCtx, + proc(event: MessageReceivedEvent) {.async: (raises: []).} = + callEventCallback(ctx, "onMessageReceived"): + $newJsonEvent("message_received", event), + ).valueOr: + chronicles.error "MessageReceivedEvent.listen failed", err = $error + return err("MessageReceivedEvent.listen failed: " & $error) + + let connStatusListener = EventConnectionStatusChange.listen( + brokerCtx, + proc(event: EventConnectionStatusChange) {.async: (raises: []).} = + callEventCallback(ctx, "onConnectionStatusChange"): + $newJsonEvent("connection_status_change", event), + ).valueOr: + chronicles.error "EventConnectionStatusChange.listen failed", err = $error + return err("EventConnectionStatusChange.listen failed: " & $error) + + ffiListeners[brokerCtx.uint32] = MessagingFFIListeners( + sent: sentListener, + error: errorListener, + propagated: propagatedListener, + received: receivedListener, + connStatus: connStatusListener, + ) + + (await ctx.myLib[].start()).isOkOr: + chronicles.error "logosdelivery_start_node failed", err = error + return err("failed to start: " & error) + return ok("") + +# ---- Lifecycle: stop (drop listeners + MessagingClient.stop) ---- + +proc logosdelivery_stop_node( + ctx: ptr FFIContext[LogosDelivery], callback: FFICallBack, userData: pointer +) {.ffi.} = + requireInitializedMessaging(ctx, "logosdelivery_stop_node"): + return err(errMsg) + + let brokerCtx = ctx.myLib[].brokerCtx + var listeners: MessagingFFIListeners + if ffiListeners.pop(brokerCtx.uint32, listeners): + await MessageSentEvent.dropListener(brokerCtx, listeners.sent) + await MessageErrorEvent.dropListener(brokerCtx, listeners.error) + await MessagePropagatedEvent.dropListener(brokerCtx, listeners.propagated) + await MessageReceivedEvent.dropListener(brokerCtx, listeners.received) + await EventConnectionStatusChange.dropListener(brokerCtx, listeners.connStatus) + + (await ctx.myLib[].stop()).isOkOr: + chronicles.error "logosdelivery_stop_node failed", err = error + return err("failed to stop: " & error) + return ok("") + +# ---- Messaging operations ---- + +proc logosdelivery_subscribe( + ctx: ptr FFIContext[LogosDelivery], + callback: FFICallBack, + userData: pointer, + contentTopicStr: cstring, +) {.ffi.} = + requireInitializedMessaging(ctx, "logosdelivery_subscribe"): + return err(errMsg) + + let contentTopic = ContentTopic($contentTopicStr) + + ( + await messaging_brokers.RequestMessagingSubscribe.request( + ctx.myLib[].brokerCtx, contentTopic + ) + ).isOkOr: + return err("subscribe failed: " & error) + + return ok("") + +proc logosdelivery_unsubscribe( + ctx: ptr FFIContext[LogosDelivery], + callback: FFICallBack, + userData: pointer, + contentTopicStr: cstring, +) {.ffi.} = + requireInitializedMessaging(ctx, "logosdelivery_unsubscribe"): + return err(errMsg) + + let contentTopic = ContentTopic($contentTopicStr) + + messaging_brokers.RequestMessagingUnsubscribe.request( + ctx.myLib[].brokerCtx, contentTopic + ).isOkOr: + return err("unsubscribe failed: " & error) + + return ok("") + +proc logosdelivery_send( + ctx: ptr FFIContext[LogosDelivery], + callback: FFICallBack, + userData: pointer, + messageJson: cstring, +) {.ffi.} = + requireInitializedMessaging(ctx, "logosdelivery_send"): + return err(errMsg) + + var jsonNode: JsonNode + try: + jsonNode = parseJson($messageJson) + except Exception as e: + return err("Failed to parse message JSON: " & e.msg) + + if not jsonNode.hasKey("contentTopic"): + return err("Missing contentTopic field") + + let contentTopic = ContentTopic(jsonNode["contentTopic"].getStr()) + + if not jsonNode.hasKey("payload"): + return err("Missing payload field") + + let payloadStr = jsonNode["payload"].getStr() + let payload = base64.decode(Base64String(payloadStr)).valueOr: + return err("invalid payload format: " & error) + + let ephemeral = jsonNode.getOrDefault("ephemeral").getBool(false) + + let envelope = MessageEnvelope.init( + contentTopic = contentTopic, payload = payload, ephemeral = ephemeral + ) + + let sendResp = ( + await messaging_brokers.RequestMessagingSend.request( + ctx.myLib[].brokerCtx, envelope + ) + ).valueOr: + return err("send failed: " & error) + + return ok($sendResp.requestId) + +# ---- Debug / introspection ---- + +proc logosdelivery_get_available_node_info_ids( + ctx: ptr FFIContext[LogosDelivery], callback: FFICallBack, userData: pointer +) {.ffi.} = + ## List of all available node info item ids queryable via get_node_info. + requireInitializedMessaging(ctx, "logosdelivery_get_available_node_info_ids"): + return err(errMsg) + return ok($ctx.myLib[].waku.stateInfo.getAllPossibleInfoItemIds()) + +proc logosdelivery_get_node_info( + ctx: ptr FFIContext[LogosDelivery], + callback: FFICallBack, + userData: pointer, + nodeInfoId: cstring, +) {.ffi.} = + ## Content of the node info item with the given id, if it exists. + requireInitializedMessaging(ctx, "logosdelivery_get_node_info"): + return err(errMsg) + let infoItemIdEnum = + try: + parseEnum[NodeInfoId]($nodeInfoId) + except ValueError: + return err("Invalid node info id: " & $nodeInfoId) + return ok(ctx.myLib[].waku.stateInfo.getNodeInfoItem(infoItemIdEnum)) + +proc logosdelivery_get_available_configs( + ctx: ptr FFIContext[LogosDelivery], callback: FFICallBack, userData: pointer +) {.ffi.} = + ## Information about the accepted config items. + requireInitializedMessaging(ctx, "logosdelivery_get_available_configs"): + return err(errMsg) + let optionMetas: seq[ConfigOptionMeta] = extractConfigOptionMeta(WakuNodeConf) + var configOptionDetails = newJArray() + for meta in optionMetas: + configOptionDetails.add( + %*{ + meta.fieldName: meta.typeName & "(" & meta.defaultValue & ")", "desc": meta.desc + } + ) + var jsonNode = newJObject() + jsonNode["configOptions"] = configOptionDetails + return ok(pretty(jsonNode)) diff --git a/messaging/api/messaging.nim b/messaging/api/messaging.nim new file mode 100644 index 000000000..41ddedefc --- /dev/null +++ b/messaging/api/messaging.nim @@ -0,0 +1,37 @@ +{.push raises: [].} + +## Messaging API broker request types. + +import chronos +import brokers/[broker_context, request_broker] +import waku/waku_core/[topics/content_topic] +import ./types + +# Subscribe to a content topic. +RequestBroker: + type RequestMessagingSubscribe* = object + subscribed*: bool + + proc signature( + contentTopic: ContentTopic + ): Future[Result[RequestMessagingSubscribe, string]] + +# Unsubscribe from a content topic. Sync. +RequestBroker(sync): + type RequestMessagingUnsubscribe* = object + unsubscribed*: bool + + proc signature( + contentTopic: ContentTopic + ): Result[RequestMessagingUnsubscribe, string] + +# Send a message. Returns a RequestId for tracking via message-lifecycle events. +RequestBroker: + type RequestMessagingSend* = object + requestId*: RequestId + + proc signature( + envelope: MessageEnvelope + ): Future[Result[RequestMessagingSend, string]] + +{.pop.} diff --git a/messaging/api/requests.nim b/messaging/api/requests.nim new file mode 100644 index 000000000..adaf0fdef --- /dev/null +++ b/messaging/api/requests.nim @@ -0,0 +1,6 @@ +{.push raises: [].} + +import ./messaging +export messaging + +{.pop.} diff --git a/waku/api/send_api.md b/messaging/api/send_api.md similarity index 100% rename from waku/api/send_api.md rename to messaging/api/send_api.md diff --git a/waku/api/types.nim b/messaging/api/types.nim similarity index 91% rename from waku/api/types.nim rename to messaging/api/types.nim index 9eae503c8..330cdcfe9 100644 --- a/waku/api/types.nim +++ b/messaging/api/types.nim @@ -1,10 +1,9 @@ {.push raises: [].} -import bearssl/rand, std/times, chronos +import bearssl/rand, chronos import stew/byteutils import waku/utils/requests as request_utils import waku/waku_core/[topics/content_topic, message/message, time] -import waku/requests/requests type MessageEnvelope* = object @@ -14,11 +13,6 @@ type RequestId* = distinct string - ConnectionStatus* {.pure.} = enum - Disconnected - PartiallyConnected - Connected - proc new*(T: typedesc[RequestId], rng: ref HmacDrbgContext): T = ## Generate a new RequestId using the provided RNG. RequestId(request_utils.generateRequestId(rng)) diff --git a/messaging/delivery_service/delivery_service.nim b/messaging/delivery_service/delivery_service.nim new file mode 100644 index 000000000..eb247c0da --- /dev/null +++ b/messaging/delivery_service/delivery_service.nim @@ -0,0 +1,39 @@ +## This module helps to ensure the correct transmission and reception of messages + +import results +import chronos, chronicles +import brokers/broker_context +import ./recv_service, ./send_service + +type DeliveryService* = ref object + sendService*: SendService + recvService*: RecvService + +proc new*( + T: type DeliveryService, + useP2PReliability: bool, + brokerCtx: BrokerContext, + relayMounted: bool, + lightpushMounted: bool, + storeMounted: bool, +): Result[T, string] = + let sendService = ?SendService.new( + useP2PReliability, brokerCtx, relayMounted, lightpushMounted, storeMounted + ) + let recvService = RecvService.new(brokerCtx) + + return ok( + DeliveryService( + sendService: sendService, + recvService: recvService, + ) + ) + +proc startDeliveryService*(self: DeliveryService): Result[void, string] = + self.recvService.startRecvService() + self.sendService.startSendService() + return ok() + +proc stopDeliveryService*(self: DeliveryService) {.async.} = + await self.sendService.stopSendService() + await self.recvService.stopRecvService() diff --git a/waku/node/delivery_service/not_delivered_storage/migrations.nim b/messaging/delivery_service/not_delivered_storage/migrations.nim similarity index 82% rename from waku/node/delivery_service/not_delivered_storage/migrations.nim rename to messaging/delivery_service/not_delivered_storage/migrations.nim index 807074d64..9a3d309ba 100644 --- a/waku/node/delivery_service/not_delivered_storage/migrations.nim +++ b/messaging/delivery_service/not_delivered_storage/migrations.nim @@ -1,7 +1,7 @@ {.push raises: [].} import std/[tables, strutils, os], results, chronicles -import ../../../common/databases/db_sqlite, ../../../common/databases/common +import waku/common/databases/db_sqlite, waku/common/databases/common logScope: topics = "waku node delivery_service" @@ -10,7 +10,7 @@ const TargetSchemaVersion* = 1 # increase this when there is an update in the database schema template projectRoot(): string = - currentSourcePath.rsplit(DirSep, 1)[0] / ".." / ".." / ".." / ".." + currentSourcePath.rsplit(DirSep, 1)[0] / ".." / ".." / ".." const PeerStoreMigrationPath: string = projectRoot / "migrations" / "sent_msgs" diff --git a/messaging/delivery_service/not_delivered_storage/not_delivered_storage.nim b/messaging/delivery_service/not_delivered_storage/not_delivered_storage.nim new file mode 100644 index 000000000..7fadda8d1 --- /dev/null +++ b/messaging/delivery_service/not_delivered_storage/not_delivered_storage.nim @@ -0,0 +1,32 @@ +## Tracks sent messages considered not properly delivered, archiving them in a +## local sqlite database. A message is considered delivered once received by any +## store node. + +import results +import + waku/common/databases/db_sqlite, + waku/waku_core/message/message, + ./migrations + +const NotDeliveredMessagesDbUrl = "not-delivered-messages.db" + +type NotDeliveredStorage* = ref object + database: SqliteDatabase + +type TrackedWakuMessage = object + msg: WakuMessage + numTrials: uint + ## number of times the node has tried to publish it + +proc new*(T: type NotDeliveredStorage): Result[T, string] = + let db = ?SqliteDatabase.new(NotDeliveredMessagesDbUrl) + + ?migrate(db) + + return ok(NotDeliveredStorage(database: db)) + +proc archiveMessage*( + self: NotDeliveredStorage, msg: WakuMessage +): Result[void, string] = + ## Archives a waku message so it survives an app restart. + return ok() diff --git a/waku/node/delivery_service/recv_service.nim b/messaging/delivery_service/recv_service.nim similarity index 100% rename from waku/node/delivery_service/recv_service.nim rename to messaging/delivery_service/recv_service.nim diff --git a/waku/node/delivery_service/recv_service/recv_service.nim b/messaging/delivery_service/recv_service/recv_service.nim similarity index 75% rename from waku/node/delivery_service/recv_service/recv_service.nim rename to messaging/delivery_service/recv_service/recv_service.nim index 899f80f71..9abc6ab95 100644 --- a/waku/node/delivery_service/recv_service/recv_service.nim +++ b/messaging/delivery_service/recv_service/recv_service.nim @@ -4,18 +4,18 @@ import std/[tables, sequtils, options, sets] import chronos, chronicles, libp2p/utility -import ../[subscription_manager] import brokers/broker_context import waku/[ waku_core, waku_store/client, waku_store/common, - waku_filter_v2/client, waku_core/topics, - events/message_events, - waku_node, + api/events/message, ] +import waku/api/requests/subscription as kernel_subscription_api +import waku/api/requests/store as kernel_store_api +import messaging/api/events const StoreCheckPeriod = chronos.minutes(5) ## How often to perform store queries @@ -36,9 +36,7 @@ type RecvMessage = object type RecvService* = ref object of RootObj brokerCtx: BrokerContext - node: WakuNode seenMsgListener: MessageSeenEventListener - subscriptionManager: SubscriptionManager recentReceivedMsgs: seq[RecvMessage] @@ -51,12 +49,16 @@ type RecvService* = ref object of RootObj proc getMissingMsgsFromStore( self: RecvService, msgHashes: seq[WakuMessageHash] ): Future[Result[seq[TupleHashAndMsg], string]] {.async.} = - let storeResp: StoreQueryResponse = ( - await self.node.wakuStoreClient.queryToAny( - StoreQueryRequest(includeData: true, messageHashes: msgHashes) + let req = ( + await kernel_store_api.RequestStoreQueryToAny.request( + self.brokerCtx, + StoreQueryRequest(includeData: true, messageHashes: msgHashes), ) ).valueOr: - return err("getMissingMsgsFromStore: " & $error) + return err("getMissingMsgsFromStore: broker err: " & error) + if req.queryError.isSome(): + return err("getMissingMsgsFromStore: " & req.errorDesc) + let storeResp: StoreQueryResponse = req.response let otherwiseMsg = WakuMessage() let otherwiseTopic = PubsubTopic("") @@ -76,8 +78,14 @@ proc processIncomingMessage( ## Return false if the incoming message is from a non-subscribed topic, ## or if the message is a duplicate (recently-seen). Otherwise, save it as ## recently-seen, emit a MessageReceivedEvent, and return true. - - if not self.subscriptionManager.isSubscribed(pubsubTopic, message.contentTopic): + let subR = kernel_subscription_api.RequestIsSubscribed.request( + self.brokerCtx, message.contentTopic, some(PubsubTopic(pubsubTopic)) + ) + if subR.isErr(): + error "subscription check failed; skipping message", + shard = pubsubTopic, contentTopic = message.contentTopic, error = subR.error + return false + if not subR.get().subscribed: trace "skipping message as I am not subscribed", shard = pubsubTopic, contentTopic = message.contentTopic return false @@ -100,22 +108,36 @@ proc checkStore*(self: RecvService) {.async.} = ## delivers them via MessageReceivedEvent. self.endTimeToCheck = getNowInNanosecondTime() - ## query store and deliver new recovered messages per subscribed topic - for pubsubTopic, contentTopics in self.subscriptionManager.subscribedTopics: - let storeResp: StoreQueryResponse = ( - await self.node.wakuStoreClient.queryToAny( + ## Snapshot subscribed topics, then query the store per topic. + var subscribedTopics: seq[tuple[shard: PubsubTopic, contentTopics: seq[ContentTopic]]] + let subbed = kernel_subscription_api.RequestSubscribedTopics.request(self.brokerCtx) + if subbed.isErr(): + # Don't advance the check window: next cycle re-covers this span. + error "could not read subscribed topics; skipping store check this cycle", + error = subbed.error + return + subscribedTopics = subbed.get().topics + for (pubsubTopic, contentTopics) in subscribedTopics: + let req = ( + await kernel_store_api.RequestStoreQueryToAny.request( + self.brokerCtx, StoreQueryRequest( includeData: false, pubsubTopic: some(pubsubTopic), contentTopics: toSeq(contentTopics), startTime: some(self.startTimeToCheck - DelayExtra.nanos), endTime: some(self.endTimeToCheck + DelayExtra.nanos), - ) + ), ) ).valueOr: - error "msgChecker failed to get remote msgHashes", - pubsubTopic = pubsubTopic, cTopics = toSeq(contentTopics), error = $error + error "msgChecker broker err", + pubsubTopic = pubsubTopic, cTopics = toSeq(contentTopics), error = error continue + if req.queryError.isSome(): + error "msgChecker store err", + pubsubTopic = pubsubTopic, cTopics = toSeq(contentTopics), error = req.errorDesc + continue + let storeResp: StoreQueryResponse = req.response ## compare the msgHashes seen from the store vs the ones received directly let msgHashesInStore = storeResp.messages.mapIt(it.messageHash) @@ -146,15 +168,12 @@ proc msgChecker(self: RecvService) {.async.} = await sleepAsync(StoreCheckPeriod) await self.checkStore() -proc new*(T: typedesc[RecvService], node: WakuNode, s: SubscriptionManager): T = - ## The storeClient will help to acquire any possible missed messages - +proc new*(T: typedesc[RecvService], brokerCtx: BrokerContext): T = + ## Builds a RecvService bound to `brokerCtx`. let now = getNowInNanosecondTime() var recvService = RecvService( - node: node, startTimeToCheck: now, - brokerCtx: node.brokerCtx, - subscriptionManager: s, + brokerCtx: brokerCtx, recentReceivedMsgs: @[], ) diff --git a/waku/node/delivery_service/send_service.nim b/messaging/delivery_service/send_service.nim similarity index 100% rename from waku/node/delivery_service/send_service.nim rename to messaging/delivery_service/send_service.nim diff --git a/waku/node/delivery_service/send_service/delivery_task.nim b/messaging/delivery_service/send_service/delivery_task.nim similarity index 96% rename from waku/node/delivery_service/send_service/delivery_task.nim rename to messaging/delivery_service/send_service/delivery_task.nim index aa1dc17d7..e07d4043a 100644 --- a/waku/node/delivery_service/send_service/delivery_task.nim +++ b/messaging/delivery_service/send_service/delivery_task.nim @@ -1,6 +1,6 @@ import std/[options, times], chronos import brokers/broker_context -import waku/waku_core, waku/api/types, waku/requests/node_requests +import waku/waku_core, messaging/api/types, waku/api/requests/node type DeliveryState* {.pure.} = enum Entry diff --git a/waku/node/delivery_service/send_service/lightpush_processor.nim b/messaging/delivery_service/send_service/lightpush_processor.nim similarity index 54% rename from waku/node/delivery_service/send_service/lightpush_processor.nim rename to messaging/delivery_service/send_service/lightpush_processor.nim index 7a9f65c71..68c1a5a3a 100644 --- a/waku/node/delivery_service/send_service/lightpush_processor.nim +++ b/messaging/delivery_service/send_service/lightpush_processor.nim @@ -1,7 +1,12 @@ import chronicles, chronos, results import std/options import brokers/broker_context -import waku/node/peer_manager, waku/waku_core, waku/waku_lightpush/[common, client, rpc] +import waku/waku_core +import waku/waku_lightpush/rpc # LightPushStatusCode +import waku/waku_lightpush/common # LightPushErrorCode constants +import waku/waku_core/codecs # WakuLightPushCodec +import waku/api/requests/lightpush as kernel_lightpush_api +import waku/api/requests/peers as kernel_peers_api import ./[delivery_task, send_processor] @@ -9,22 +14,21 @@ logScope: topics = "send service lightpush processor" type LightpushSendProcessor* = ref object of BaseSendProcessor - peerManager: PeerManager - lightpushClient: WakuLightPushClient proc new*( - T: typedesc[LightpushSendProcessor], - peerManager: PeerManager, - lightpushClient: WakuLightPushClient, - brokerCtx: BrokerContext, + T: typedesc[LightpushSendProcessor], brokerCtx: BrokerContext ): T = - return - T(peerManager: peerManager, lightpushClient: lightpushClient, brokerCtx: brokerCtx) + return T(brokerCtx: brokerCtx) proc isLightpushPeerAvailable( self: LightpushSendProcessor, pubsubTopic: PubsubTopic ): bool = - return self.peerManager.selectPeer(WakuLightPushCodec, some(pubsubTopic)).isSome() + let req = kernel_peers_api.RequestSelectPeer.request( + self.brokerCtx, WakuLightPushCodec, some(pubsubTopic) + ).valueOr: + debug "isLightpushPeerAvailable: broker err", error = error + return false + return req.peer.isSome() method isValidProcessor*( self: LightpushSendProcessor, task: DeliveryTask @@ -40,34 +44,50 @@ method sendImpl*( msgHash = task.msgHash.to0xHex(), tryCount = task.tryCount - let peer = self.peerManager.selectPeer(WakuLightPushCodec, some(task.pubsubTopic)).valueOr: + let peerReq = kernel_peers_api.RequestSelectPeer.request( + self.brokerCtx, WakuLightPushCodec, some(task.pubsubTopic) + ).valueOr: + debug "LightpushSendProcessor.sendImpl: peer broker err", error = error + task.state = DeliveryState.NextRoundRetry + return + if peerReq.peer.isNone(): debug "No peer available for Lightpush, request pushed back for next round", requestId = task.requestId task.state = DeliveryState.NextRoundRetry return + let peer = peerReq.peer.get() - let numLightpushServers = ( - await self.lightpushClient.publish(some(task.pubsubTopic), task.msg, peer) + let pubReq = ( + await kernel_lightpush_api.RequestLightpushPublish.request( + self.brokerCtx, peer, task.pubsubTopic, task.msg + ) ).valueOr: - error "LightpushSendProcessor.sendImpl failed", error = error.desc.get($error.code) - case error.code + error "LightpushSendProcessor.sendImpl: broker err", error = error + task.state = DeliveryState.NextRoundRetry + return + + if pubReq.publishError.isSome(): + let code = pubReq.publishError.get() + error "LightpushSendProcessor.sendImpl failed", + code = $code, desc = pubReq.errorDesc + case code of LightPushErrorCode.NO_PEERS_TO_RELAY, LightPushErrorCode.TOO_MANY_REQUESTS, LightPushErrorCode.OUT_OF_RLN_PROOF, LightPushErrorCode.SERVICE_NOT_AVAILABLE, LightPushErrorCode.INTERNAL_SERVER_ERROR: task.state = DeliveryState.NextRoundRetry else: - # the message is malformed, send error + # malformed message task.state = DeliveryState.FailedToDeliver - task.errorDesc = error.desc.get($error.code) + task.errorDesc = pubReq.errorDesc task.deliveryTime = Moment.now() return + let numLightpushServers = pubReq.relayedPeerCount if numLightpushServers > 0: info "Message propagated via Lightpush", requestId = task.requestId, msgHash = task.msgHash.to0xHex() task.state = DeliveryState.SuccessfullyPropagated task.deliveryTime = Moment.now() - # TODO: with a simple retry processor it might be more accurate to say `Sent` else: # Controversial state, publish says ok but no peer. It should not happen. debug "Lightpush publish returned zero peers, request pushed back for next round", diff --git a/waku/node/delivery_service/send_service/relay_processor.nim b/messaging/delivery_service/send_service/relay_processor.nim similarity index 60% rename from waku/node/delivery_service/send_service/relay_processor.nim rename to messaging/delivery_service/send_service/relay_processor.nim index e06b664fb..44761926c 100644 --- a/waku/node/delivery_service/send_service/relay_processor.nim +++ b/messaging/delivery_service/send_service/relay_processor.nim @@ -1,22 +1,22 @@ import std/options import chronos, chronicles import brokers/broker_context -import waku/[waku_core], waku/waku_lightpush/[common, rpc] -import waku/requests/health_requests -import waku/api/types +import waku/waku_core +import waku/waku_relay/protocol # PublishOutcome +import waku/api/requests/health +import waku/api/requests/relay as kernel_relay_api +import messaging/api/types import ./[delivery_task, send_processor] logScope: topics = "send service relay processor" type RelaySendProcessor* = ref object of BaseSendProcessor - publishProc: PushMessageHandler fallbackStateToSet: DeliveryState proc new*( T: typedesc[RelaySendProcessor], lightpushAvailable: bool, - publishProc: PushMessageHandler, brokerCtx: BrokerContext, ): RelaySendProcessor = let fallbackStateToSet = @@ -26,7 +26,6 @@ proc new*( DeliveryState.FailedToDeliver return RelaySendProcessor( - publishProc: publishProc, fallbackStateToSet: fallbackStateToSet, brokerCtx: brokerCtx, ) @@ -57,17 +56,48 @@ method sendImpl*(self: RelaySendProcessor, task: DeliveryTask) {.async.} = msgHash = task.msgHash.to0xHex(), tryCount = task.tryCount - let noOfPublishedPeers = (await self.publishProc(task.pubsubTopic, task.msg)).valueOr: - let errorMessage = error.desc.get($error.code) - error "Failed to publish message with relay", - request = task.requestId, msgHash = task.msgHash.to0xHex(), error = errorMessage - if error.code != LightPushErrorCode.NO_PEERS_TO_RELAY: - task.state = DeliveryState.FailedToDeliver - task.errorDesc = errorMessage - else: - task.state = self.fallbackStateToSet + let pubReq = ( + await kernel_relay_api.RequestRelayPublish.request( + self.brokerCtx, task.pubsubTopic, task.msg + ) + ).valueOr: + # Broker-level failure: publish provider unreachable. Fail permanently. + error "RelaySendProcessor.sendImpl: broker err", error = error + task.state = DeliveryState.FailedToDeliver + task.errorDesc = error return + # RLN proof failure: permanent failure. + if pubReq.rlnProofFailed: + error "RelaySendProcessor: RLN proof generation failed", + request = task.requestId, msgHash = task.msgHash.to0xHex(), error = pubReq.errorDesc + task.state = DeliveryState.FailedToDeliver + task.errorDesc = pubReq.errorDesc + return + + # Message validation failure: permanent failure (malformed). + if pubReq.validationFailed: + error "RelaySendProcessor: message validation failed", + request = task.requestId, msgHash = task.msgHash.to0xHex(), error = pubReq.errorDesc + task.state = DeliveryState.FailedToDeliver + task.errorDesc = pubReq.errorDesc + return + + # Underlying wakuRelay.publish failure mode. + if pubReq.publishError.isSome(): + error "Failed to publish message with relay", + request = task.requestId, + msgHash = task.msgHash.to0xHex(), + error = pubReq.errorDesc + case pubReq.publishError.get() + of NoPeersToPublish: + task.state = self.fallbackStateToSet + else: + task.state = DeliveryState.FailedToDeliver + task.errorDesc = pubReq.errorDesc + return + + let noOfPublishedPeers = pubReq.relayedPeerCount if noOfPublishedPeers > 0: info "Message propagated via Relay", requestId = task.requestId, diff --git a/waku/node/delivery_service/send_service/send_processor.nim b/messaging/delivery_service/send_service/send_processor.nim similarity index 100% rename from waku/node/delivery_service/send_service/send_processor.nim rename to messaging/delivery_service/send_service/send_processor.nim diff --git a/waku/node/delivery_service/send_service/send_service.nim b/messaging/delivery_service/send_service/send_service.nim similarity index 75% rename from waku/node/delivery_service/send_service/send_service.nim rename to messaging/delivery_service/send_service/send_service.nim index 902f3aa1c..f817f0f57 100644 --- a/waku/node/delivery_service/send_service/send_service.nim +++ b/messaging/delivery_service/send_service/send_service.nim @@ -6,19 +6,13 @@ import chronos, chronicles, libp2p/utility import brokers/broker_context import ./[send_processor, relay_processor, lightpush_processor, delivery_task], - ../[subscription_manager], - waku/[ - waku_core, - node/waku_node, - node/peer_manager, - waku_store/client, - waku_store/common, - waku_relay/protocol, - waku_rln_relay/rln_relay, - waku_lightpush/client, - waku_lightpush/callbacks, - events/message_events, - ] + waku/[waku_core, waku_store/common, api/events/message] +import waku/api/requests/store as kernel_store_api +import waku/api/requests/peers as kernel_peers_api +import waku/api/requests/relay as kernel_relay_api +import waku/api/requests/lightpush as kernel_lightpush_api +import waku/api/requests/subscription as kernel_subscription_api +import messaging/api/events logScope: topics = "send service" @@ -56,36 +50,22 @@ type SendService* = ref object of RootObj serviceLoopHandle: Future[void] ## handle that allows to stop the async task sendProcessor: BaseSendProcessor - node: WakuNode + relayMounted: bool + ## Drives auto-subscribe routing at send time. checkStoreForMessages: bool - subscriptionManager: SubscriptionManager proc setupSendProcessorChain( - peerManager: PeerManager, - lightpushClient: WakuLightPushClient, - relay: WakuRelay, - rlnRelay: WakuRLNRelay, - brokerCtx: BrokerContext, + relayMounted, lightpushMounted: bool, brokerCtx: BrokerContext ): Result[BaseSendProcessor, string] = - let isRelayAvail = not relay.isNil() - let isLightPushAvail = not lightpushClient.isNil() - - if not isRelayAvail and not isLightPushAvail: + if not relayMounted and not lightpushMounted: return err("No valid send processor found for the delivery task") var processors = newSeq[BaseSendProcessor]() - if isRelayAvail: - let rln: Option[WakuRLNRelay] = - if rlnRelay.isNil(): - none[WakuRLNRelay]() - else: - some(rlnRelay) - let publishProc = getRelayPushHandler(relay, rln) - - processors.add(RelaySendProcessor.new(isLightPushAvail, publishProc, brokerCtx)) - if isLightPushAvail: - processors.add(LightpushSendProcessor.new(peerManager, lightpushClient, brokerCtx)) + if relayMounted: + processors.add(RelaySendProcessor.new(lightpushMounted, brokerCtx)) + if lightpushMounted: + processors.add(LightpushSendProcessor.new(brokerCtx)) var currentProcessor: BaseSendProcessor = processors[0] for i in 1 ..< processors.len: @@ -98,29 +78,29 @@ proc setupSendProcessorChain( proc new*( T: typedesc[SendService], preferP2PReliability: bool, - w: WakuNode, - s: SubscriptionManager, + brokerCtx: BrokerContext, + relayMounted: bool, + lightpushMounted: bool, + storeMounted: bool, ): Result[T, string] = - if w.wakuRelay.isNil() and w.wakuLightpushClient.isNil(): + if not relayMounted and not lightpushMounted: return err( "Could not create SendService. wakuRelay or wakuLightpushClient should be set" ) - let checkStoreForMessages = preferP2PReliability and not w.wakuStoreClient.isNil() + let checkStoreForMessages = preferP2PReliability and storeMounted - let sendProcessorChain = setupSendProcessorChain( - w.peerManager, w.wakuLightPushClient, w.wakuRelay, w.wakuRlnRelay, w.brokerCtx - ).valueOr: - return err("failed to setup SendProcessorChain: " & $error) + let sendProcessorChain = + setupSendProcessorChain(relayMounted, lightpushMounted, brokerCtx).valueOr: + return err("failed to setup SendProcessorChain: " & $error) let sendService = SendService( - brokerCtx: w.brokerCtx, + brokerCtx: brokerCtx, taskCache: newSeq[DeliveryTask](), serviceLoopHandle: nil, sendProcessor: sendProcessorChain, - node: w, + relayMounted: relayMounted, checkStoreForMessages: checkStoreForMessages, - subscriptionManager: s, ) return ok(sendService) @@ -129,7 +109,11 @@ proc addTask(self: SendService, task: DeliveryTask) = self.taskCache.addUnique(task) proc isStorePeerAvailable*(sendService: SendService): bool = - return sendService.node.peerManager.selectPeer(WakuStoreCodec).isSome() + let req = kernel_peers_api.RequestSelectPeer.request( + sendService.brokerCtx, WakuStoreCodec, none(PubsubTopic) + ).valueOr: + return false + req.peer.isSome() proc checkMsgsInStore(self: SendService, tasksToValidate: seq[DeliveryTask]) {.async.} = if tasksToValidate.len() == 0: @@ -143,14 +127,20 @@ proc checkMsgsInStore(self: SendService, tasksToValidate: seq[DeliveryTask]) {.a var hashesToValidate = tasksToValidate.mapIt(it.msgHash) # TODO: confirm hash format for store query!!! - let storeResp: StoreQueryResponse = ( - await self.node.wakuStoreClient.queryToAny( - StoreQueryRequest(includeData: false, messageHashes: hashesToValidate) + let req = ( + await kernel_store_api.RequestStoreQueryToAny.request( + self.brokerCtx, + StoreQueryRequest(includeData: false, messageHashes: hashesToValidate), ) ).valueOr: - error "Failed to get store validation for messages", - hashes = hashesToValidate.mapIt(shortLog(it)), error = $error + error "Failed store validation (broker err)", + hashes = hashesToValidate.mapIt(shortLog(it)), error = error return + if req.queryError.isSome(): + error "Failed store validation (store err)", + hashes = hashesToValidate.mapIt(shortLog(it)), error = req.errorDesc + return + let storeResp: StoreQueryResponse = req.response let storedItems = storeResp.messages.mapIt(it.messageHash) @@ -263,9 +253,19 @@ proc send*(self: SendService, task: DeliveryTask) {.async.} = info "SendService.send: processing delivery task", requestId = task.requestId, msgHash = task.msgHash.to0xHex() - self.subscriptionManager.subscribe(task.msg.contentTopic).isOkOr: - error "SendService.send: failed to subscribe to content topic", - contentTopic = task.msg.contentTopic, error = error + # Auto-subscribe routes relay/edge: relay if mounted, else edge (filter). + if self.relayMounted: + kernel_subscription_api.RequestRelaySubscribeContentTopic.request( + self.brokerCtx, task.msg.contentTopic, none[PubsubTopic]() + ).isOkOr: + error "SendService.send: failed to subscribe to content topic", + contentTopic = task.msg.contentTopic, error = error + else: + kernel_subscription_api.RequestEdgeSubscribe.request( + self.brokerCtx, task.msg.contentTopic, none[PubsubTopic]() + ).isOkOr: + error "SendService.send: failed to subscribe to content topic", + contentTopic = task.msg.contentTopic, error = error await self.sendProcessor.process(task) reportTaskResult(self, task) diff --git a/waku/node/delivery_service/subscription_manager.nim b/messaging/delivery_service/subscription_manager.nim similarity index 100% rename from waku/node/delivery_service/subscription_manager.nim rename to messaging/delivery_service/subscription_manager.nim diff --git a/messaging/messaging_client.nim b/messaging/messaging_client.nim new file mode 100644 index 000000000..0155d9ee7 --- /dev/null +++ b/messaging/messaging_client.nim @@ -0,0 +1,118 @@ +{.push raises: [].} + +import chronos, chronicles, results +import libp2p/crypto/crypto +import brokers/broker_context + +import waku/waku_core +import waku/api/requests/protocols as protocols_api +import messaging/delivery_service/delivery_service +import messaging/api/types +import messaging/messaging_client_type + +import messaging/api/api_subscribe +import messaging/api/api_unsubscribe +import messaging/api/api_send + +import messaging/api/messaging as messaging_brokers + +export messaging_client_type +export api_subscribe, api_unsubscribe, api_send + +logScope: + topics = "messaging-client" + +proc registerMessagingApiProviders( + client: MessagingClient +): Result[void, string] = + ## Bind the messaging broker providers to the client API procs. + messaging_brokers.RequestMessagingSubscribe.setProvider( + client.brokerCtx, + proc( + contentTopic: ContentTopic + ): Future[Result[messaging_brokers.RequestMessagingSubscribe, string]] {.async.} = + (await subscribe(client, contentTopic)).isOkOr: + return err(error) + return ok(messaging_brokers.RequestMessagingSubscribe(subscribed: true)), + ).isOkOr: + return err("registerMessagingApiProviders: RequestMessagingSubscribe: " & error) + + messaging_brokers.RequestMessagingUnsubscribe.setProvider( + client.brokerCtx, + proc( + contentTopic: ContentTopic + ): Result[messaging_brokers.RequestMessagingUnsubscribe, string] = + unsubscribe(client, contentTopic).isOkOr: + return err(error) + return ok(messaging_brokers.RequestMessagingUnsubscribe(unsubscribed: true)), + ).isOkOr: + return err("registerMessagingApiProviders: RequestMessagingUnsubscribe: " & error) + + messaging_brokers.RequestMessagingSend.setProvider( + client.brokerCtx, + proc( + envelope: MessageEnvelope + ): Future[Result[messaging_brokers.RequestMessagingSend, string]] {.async.} = + let reqId = (await send(client, envelope)).valueOr: + return err(error) + return ok(messaging_brokers.RequestMessagingSend(requestId: reqId)), + ).isOkOr: + return err("registerMessagingApiProviders: RequestMessagingSend: " & error) + + ok() + +proc new*( + T: type MessagingClient, brokerCtx: BrokerContext, preferP2PReliability: bool +): MessagingClient = + ## Construct a messaging-layer client bound to `brokerCtx`. + MessagingClient( + brokerCtx: brokerCtx, + rng: crypto.newRng(), + preferP2PReliability: preferP2PReliability, + ) + +proc start*(self: MessagingClient): Future[Result[void, string]] {.async: (raises: []).} = + ## Bring the messaging layer up. + if self.isNil(): + return err("MessagingClient.start: client is nil") + + # Mounted protocols come from the kernel broker. + let status = protocols_api.RequestProtocolMountStatus.request(self.brokerCtx).valueOr: + return err("MessagingClient.start: protocol mount status query failed: " & error) + self.relayMounted = status.relayMounted + self.filterMounted = status.filterMounted + + self.deliveryService = DeliveryService.new( + self.preferP2PReliability, + self.brokerCtx, + status.relayMounted, + status.lightpushMounted, + status.storeMounted, + ).valueOr: + return err("DeliveryService.new failed: " & error) + + self.deliveryService.startDeliveryService().isOkOr: + return err("startDeliveryService failed: " & error) + + registerMessagingApiProviders(self).isOkOr: + return err("registerMessagingApiProviders failed: " & error) + ok() + +proc stop*(self: MessagingClient): Future[Result[void, string]] {.async: (raises: []).} = + ## Stop inner components and clear the messaging API providers. + if self.isNil(): + return err("MessagingClient.stop: client is nil") + + if not self.deliveryService.isNil(): + try: + await self.deliveryService.stopDeliveryService() + except CatchableError as e: + return err("stopDeliveryService raised: " & e.msg) + + messaging_brokers.RequestMessagingSubscribe.clearProvider(self.brokerCtx) + messaging_brokers.RequestMessagingUnsubscribe.clearProvider(self.brokerCtx) + messaging_brokers.RequestMessagingSend.clearProvider(self.brokerCtx) + + return ok() + +{.pop.} diff --git a/messaging/messaging_client_type.nim b/messaging/messaging_client_type.nim new file mode 100644 index 000000000..a308ad3ea --- /dev/null +++ b/messaging/messaging_client_type.nim @@ -0,0 +1,22 @@ +{.push raises: [].} + +## MessagingClient type definition. +## Addressed by its broker context; all kernel interaction goes through the +## broker surface (waku/api). + +import bearssl/rand +import brokers/broker_context +import messaging/delivery_service/delivery_service + +type MessagingClient* = ref object + brokerCtx*: BrokerContext + rng*: ref HmacDrbgContext + ## RNG for request-id generation + preferP2PReliability*: bool + deliveryService*: DeliveryService + relayMounted*: bool + ## Cached at `start`: is the relay protocol mounted? + filterMounted*: bool + ## Cached at `start`: is the filter client mounted? + +{.pop.} diff --git a/tests/api/test_api_health.nim b/tests/api/test_api_health.nim index d949db24f..3d7cee8d4 100644 --- a/tests/api/test_api_health.nim +++ b/tests/api/test_api_health.nim @@ -5,13 +5,15 @@ import chronos, testutils/unittests, stew/byteutils, libp2p/[switch, peerinfo] import brokers/broker_context import ../testlib/[common, wakucore, wakunode, testasync] +import waku/api/api +import layers/logos_delivery import waku, waku/[waku_node, waku_core, waku_relay/protocol], waku/node/health_monitor/[topic_health, health_status, protocol_health, health_report], - waku/requests/health_requests, - waku/requests/node_requests, - waku/events/health_events, + waku/api/requests/health, + waku/api/requests/node, + waku/api/events/health, waku/common/waku_protocol, waku/factory/waku_conf import tools/confutils/cli_args @@ -73,7 +75,7 @@ proc waitForShardHealthy( suite "LM API health checking": var serviceNode {.threadvar.}: WakuNode - client {.threadvar.}: Waku + client {.threadvar.}: LogosDelivery servicePeerInfo {.threadvar.}: RemotePeerInfo asyncSetup: @@ -101,9 +103,11 @@ suite "LM API health checking": conf.numShardsInNetwork = 1 conf.rest = false - client = (await createNode(conf)).valueOr: + let waku = (await createNode(conf)).valueOr: raiseAssert error - (await startWaku(addr client)).isOkOr: + client = LogosDelivery.new(MessagingClient, waku).valueOr: + raiseAssert "Failed to wrap in LogosDelivery: " & error + (await client.start()).isOkOr: raiseAssert error asyncTeardown: @@ -111,8 +115,8 @@ suite "LM API health checking": await serviceNode.stop() asyncTest "RequestShardTopicsHealth, check PubsubTopic health": - client.node.wakuRelay.subscribe(DefaultShard, dummyHandler) - await client.node.connectToNodes(@[servicePeerInfo]) + client.waku.node.wakuRelay.subscribe(DefaultShard, dummyHandler) + await client.waku.node.connectToNodes(@[servicePeerInfo]) var isHealthy = false let start = Moment.now() @@ -131,7 +135,7 @@ suite "LM API health checking": asyncTest "RequestShardTopicsHealth, check disconnected PubsubTopic": const GhostShard = PubsubTopic("/waku/2/rs/1/666") - client.node.wakuRelay.subscribe(GhostShard, dummyHandler) + client.waku.node.wakuRelay.subscribe(GhostShard, dummyHandler) let req = RequestShardTopicsHealth.request(client.brokerCtx, @[GhostShard]).valueOr: raiseAssert "Request failed" @@ -140,7 +144,7 @@ suite "LM API health checking": check req.topicHealth[0].health == TopicHealth.UNHEALTHY asyncTest "RequestProtocolHealth, check relay status": - await client.node.connectToNodes(@[servicePeerInfo]) + await client.waku.node.connectToNodes(@[servicePeerInfo]) var isReady = false let start = Moment.now() @@ -174,7 +178,7 @@ suite "LM API health checking": raiseAssert "RequestConnectionStatus failed" check initialReq.connectionStatus == ConnectionStatus.Disconnected - await client.node.connectToNodes(@[servicePeerInfo]) + await client.waku.node.connectToNodes(@[servicePeerInfo]) var isConnected = false let start = Moment.now() @@ -194,20 +198,20 @@ suite "LM API health checking": let connectFuture = waitForConnectionStatus(client.brokerCtx, ConnectionStatus.PartiallyConnected) - await client.node.connectToNodes(@[servicePeerInfo]) + await client.waku.node.connectToNodes(@[servicePeerInfo]) await connectFuture let disconnectFuture = waitForConnectionStatus(client.brokerCtx, ConnectionStatus.Disconnected) - await client.node.disconnectNode(servicePeerInfo) + await client.waku.node.disconnectNode(servicePeerInfo) await disconnectFuture asyncTest "EventShardTopicHealthChange, detect health improvement": - client.node.wakuRelay.subscribe(DefaultShard, dummyHandler) + client.waku.node.wakuRelay.subscribe(DefaultShard, dummyHandler) let healthEventFuture = waitForShardHealthy(client.brokerCtx) - await client.node.connectToNodes(@[servicePeerInfo]) + await client.waku.node.connectToNodes(@[servicePeerInfo]) let event = await healthEventFuture check event.topic == DefaultShard @@ -243,10 +247,10 @@ suite "LM API health checking": check shardReq.isOk() let targetShard = $shardReq.get().relayShard - client.node.wakuRelay.subscribe(targetShard, dummyHandler) + client.waku.node.wakuRelay.subscribe(targetShard, dummyHandler) serviceNode.wakuRelay.subscribe(targetShard, dummyHandler) - await client.node.connectToNodes(@[servicePeerInfo]) + await client.waku.node.connectToNodes(@[servicePeerInfo]) var isHealthy = false let start = Moment.now() @@ -265,7 +269,7 @@ suite "LM API health checking": check isHealthy == true asyncTest "RequestProtocolHealth, edge mode smoke test": - var edgeWaku: Waku + var edgeClient: LogosDelivery lockNewGlobalBrokerContext: var edgeConf = defaultWakuNodeConf().valueOr: @@ -278,18 +282,21 @@ suite "LM API health checking": edgeConf.maxMessageSize = "150 KiB" edgeConf.rest = false - edgeWaku = (await createNode(edgeConf)).valueOr: + let edgeWaku = (await createNode(edgeConf)).valueOr: raiseAssert "Failed to create edge node: " & error - (await startWaku(addr edgeWaku)).isOkOr: + edgeClient = LogosDelivery.new(MessagingClient, edgeWaku).valueOr: + raiseAssert "Failed to wrap edge in LogosDelivery: " & error + + (await edgeClient.start()).isOkOr: raiseAssert "Failed to start edge waku: " & error let relayReq = await RequestProtocolHealth.request( - edgeWaku.brokerCtx, WakuProtocol.RelayProtocol + edgeClient.brokerCtx, WakuProtocol.RelayProtocol ) check relayReq.isOk() check relayReq.get().healthStatus.health == HealthStatus.NOT_MOUNTED - check not edgeWaku.node.wakuFilterClient.isNil() + check not edgeClient.waku.node.wakuFilterClient.isNil() - discard await edgeWaku.stop() + discard await edgeClient.stop() diff --git a/tests/api/test_api_receive.nim b/tests/api/test_api_receive.nim index d6aa954a4..52000d8b4 100644 --- a/tests/api/test_api_receive.nim +++ b/tests/api/test_api_receive.nim @@ -7,18 +7,21 @@ import brokers/broker_context import ../testlib/[common, wakucore, wakunode, testasync] import ../waku_archive/archive_utils +import waku/api/api +import layers/logos_delivery +import messaging/api/events import waku, waku/[ waku_node, waku_core, - events/message_events, + api/events/message, waku_relay/protocol, waku_archive, waku_archive/common as archive_common, - node/delivery_service/delivery_service, - node/delivery_service/recv_service, ] +import messaging/delivery_service/delivery_service +import messaging/delivery_service/recv_service import waku/factory/waku_conf import tools/confutils/cli_args @@ -142,12 +145,15 @@ suite "Messaging API, Receive Service (store recovery)": # RecvService captures startTimeToCheck at construction time; the # message's timestamp must land after that point to fall inside # checkStore's time window. - var subscriber: Waku + var subscriber: LogosDelivery lockNewGlobalBrokerContext: - subscriber = (await createNode(createApiNodeConf(numShards))).expect( + let waku = (await createNode(createApiNodeConf(numShards))).expect( "Failed to create subscriber" ) - (await startWaku(addr subscriber)).expect("Failed to start subscriber") + subscriber = LogosDelivery.new(MessagingClient, waku).expect( + "Failed to wrap subscriber in LogosDelivery" + ) + (await subscriber.start()).expect("Failed to start subscriber") # publish after the subscriber exists but before it connects to the # store; the message reaches the archive but the subscriber doesn't @@ -174,10 +180,10 @@ suite "Messaging API, Receive Service (store recovery)": # connect subscriber to store after the message is already archived so # gossipsub doesn't replay it via the live path - await subscriber.node.connectToNodes(@[storeNodePeerInfo]) + await subscriber.waku.node.connectToNodes(@[storeNodePeerInfo]) # subscribe to content topic - (await subscriber.subscribe(testTopic)).expect("Failed to subscribe") + (await subscriber.messaging.subscribe(testTopic)).expect("Failed to subscribe") # listen before triggering store check let eventManager = newReceiveEventListenerManager(subscriber.brokerCtx, 1) @@ -185,7 +191,7 @@ suite "Messaging API, Receive Service (store recovery)": await eventManager.teardown() # trigger store check, should recover and deliver via MessageReceivedEvent - await subscriber.deliveryService.recvService.checkStore() + await subscriber.messaging.deliveryService.recvService.checkStore() let received = await eventManager.waitForEvents(TestTimeout) check received diff --git a/tests/api/test_api_send.nim b/tests/api/test_api_send.nim index 084119041..82f014eb6 100644 --- a/tests/api/test_api_send.nim +++ b/tests/api/test_api_send.nim @@ -6,6 +6,9 @@ import brokers/broker_context import ../testlib/[common, wakucore, wakunode, testasync] import ../waku_archive/archive_utils import waku, waku/[waku_node, waku_core, waku_relay/protocol] +import waku/api/api +import layers/logos_delivery +import messaging/api/events import waku/factory/waku_conf import tools/confutils/cli_args @@ -237,11 +240,13 @@ suite "Waku API - Send": ) asyncTest "Check API availability (unhealthy node)": - var node: Waku + var client: LogosDelivery lockNewGlobalBrokerContext: - node = (await createNode(createApiNodeConf())).valueOr: + let waku = (await createNode(createApiNodeConf())).valueOr: raiseAssert error - (await startWaku(addr node)).isOkOr: + client = LogosDelivery.new(MessagingClient, waku).valueOr: + raiseAssert "Failed to wrap in LogosDelivery: " & error + (await client.start()).isOkOr: raiseAssert "Failed to start Waku node: " & error # node is not connected ! @@ -249,28 +254,30 @@ suite "Waku API - Send": ContentTopic("/waku/2/default-content/proto"), "test payload" ) - let sendResult = await node.send(envelope) + let sendResult = await client.messaging.send(envelope) # TODO: The API is not enforcing a health check before the send, # so currently this test cannot successfully fail to send. check sendResult.isOk() - (await node.stop()).isOkOr: + (await client.stop()).isOkOr: raiseAssert "Failed to stop node: " & error asyncTest "Send fully validated": - var node: Waku + var client: LogosDelivery lockNewGlobalBrokerContext: - node = (await createNode(createApiNodeConf())).valueOr: + let waku = (await createNode(createApiNodeConf())).valueOr: raiseAssert error - (await startWaku(addr node)).isOkOr: + client = LogosDelivery.new(MessagingClient, waku).valueOr: + raiseAssert "Failed to wrap in LogosDelivery: " & error + (await client.start()).isOkOr: raiseAssert "Failed to start Waku node: " & error - await node.node.connectToNodes( + await client.waku.node.connectToNodes( @[relayNode1PeerInfo, lightpushNodePeerInfo, storeNodePeerInfo] ) - let eventManager = newSendEventListenerManager(node.brokerCtx) + let eventManager = newSendEventListenerManager(client.brokerCtx) defer: await eventManager.teardown() @@ -278,7 +285,7 @@ suite "Waku API - Send": ContentTopic("/waku/2/default-content/proto"), "test payload" ) - let requestId = (await node.send(envelope)).valueOr: + let requestId = (await client.messaging.send(envelope)).valueOr: raiseAssert error # Wait for events with timeout @@ -289,20 +296,22 @@ suite "Waku API - Send": {SendEventOutcome.Sent, SendEventOutcome.Propagated}, requestId ) - (await node.stop()).isOkOr: + (await client.stop()).isOkOr: raiseAssert "Failed to stop node: " & error asyncTest "Send only propagates": - var node: Waku + var client: LogosDelivery lockNewGlobalBrokerContext: - node = (await createNode(createApiNodeConf())).valueOr: + let waku = (await createNode(createApiNodeConf())).valueOr: raiseAssert error - (await startWaku(addr node)).isOkOr: + client = LogosDelivery.new(MessagingClient, waku).valueOr: + raiseAssert "Failed to wrap in LogosDelivery: " & error + (await client.start()).isOkOr: raiseAssert "Failed to start Waku node: " & error - await node.node.connectToNodes(@[relayNode1PeerInfo]) + await client.waku.node.connectToNodes(@[relayNode1PeerInfo]) - let eventManager = newSendEventListenerManager(node.brokerCtx) + let eventManager = newSendEventListenerManager(client.brokerCtx) defer: await eventManager.teardown() @@ -310,7 +319,7 @@ suite "Waku API - Send": ContentTopic("/waku/2/default-content/proto"), "test payload" ) - let requestId = (await node.send(envelope)).valueOr: + let requestId = (await client.messaging.send(envelope)).valueOr: raiseAssert error # Wait for events with timeout @@ -319,20 +328,22 @@ suite "Waku API - Send": eventManager.validate({SendEventOutcome.Propagated}, requestId) - (await node.stop()).isOkOr: + (await client.stop()).isOkOr: raiseAssert "Failed to stop node: " & error asyncTest "Send only propagates fallback to lightpush": - var node: Waku + var client: LogosDelivery lockNewGlobalBrokerContext: - node = (await createNode(createApiNodeConf())).valueOr: + let waku = (await createNode(createApiNodeConf())).valueOr: raiseAssert error - (await startWaku(addr node)).isOkOr: + client = LogosDelivery.new(MessagingClient, waku).valueOr: + raiseAssert "Failed to wrap in LogosDelivery: " & error + (await client.start()).isOkOr: raiseAssert "Failed to start Waku node: " & error - await node.node.connectToNodes(@[lightpushNodePeerInfo]) + await client.waku.node.connectToNodes(@[lightpushNodePeerInfo]) - let eventManager = newSendEventListenerManager(node.brokerCtx) + let eventManager = newSendEventListenerManager(client.brokerCtx) defer: await eventManager.teardown() @@ -340,7 +351,7 @@ suite "Waku API - Send": ContentTopic("/waku/2/default-content/proto"), "test payload" ) - let requestId = (await node.send(envelope)).valueOr: + let requestId = (await client.messaging.send(envelope)).valueOr: raiseAssert error # Wait for events with timeout @@ -349,20 +360,22 @@ suite "Waku API - Send": eventManager.validate({SendEventOutcome.Propagated}, requestId) - (await node.stop()).isOkOr: + (await client.stop()).isOkOr: raiseAssert "Failed to stop node: " & error asyncTest "Send fully validates fallback to lightpush": - var node: Waku + var client: LogosDelivery lockNewGlobalBrokerContext: - node = (await createNode(createApiNodeConf())).valueOr: + let waku = (await createNode(createApiNodeConf())).valueOr: raiseAssert error - (await startWaku(addr node)).isOkOr: + client = LogosDelivery.new(MessagingClient, waku).valueOr: + raiseAssert "Failed to wrap in LogosDelivery: " & error + (await client.start()).isOkOr: raiseAssert "Failed to start Waku node: " & error - await node.node.connectToNodes(@[lightpushNodePeerInfo, storeNodePeerInfo]) + await client.waku.node.connectToNodes(@[lightpushNodePeerInfo, storeNodePeerInfo]) - let eventManager = newSendEventListenerManager(node.brokerCtx) + let eventManager = newSendEventListenerManager(client.brokerCtx) defer: await eventManager.teardown() @@ -370,7 +383,7 @@ suite "Waku API - Send": ContentTopic("/waku/2/default-content/proto"), "test payload" ) - let requestId = (await node.send(envelope)).valueOr: + let requestId = (await client.messaging.send(envelope)).valueOr: raiseAssert error # Wait for events with timeout @@ -380,7 +393,7 @@ suite "Waku API - Send": eventManager.validate( {SendEventOutcome.Propagated, SendEventOutcome.Sent}, requestId ) - (await node.stop()).isOkOr: + (await client.stop()).isOkOr: raiseAssert "Failed to stop node: " & error asyncTest "Send fails with event": @@ -407,16 +420,18 @@ suite "Waku API - Send": ).isOkOr: raiseAssert "Failed to subscribe fakeLightpushNode: " & error - var node: Waku + var client: LogosDelivery lockNewGlobalBrokerContext: - node = (await createNode(createApiNodeConf(cli_args.WakuMode.Edge))).valueOr: + let waku = (await createNode(createApiNodeConf(cli_args.WakuMode.Edge))).valueOr: raiseAssert error - (await startWaku(addr node)).isOkOr: + client = LogosDelivery.new(MessagingClient, waku).valueOr: + raiseAssert "Failed to wrap in LogosDelivery: " & error + (await client.start()).isOkOr: raiseAssert "Failed to start Waku node: " & error - await node.node.connectToNodes(@[fakeLightpushNodePeerInfo]) + await client.waku.node.connectToNodes(@[fakeLightpushNodePeerInfo]) - let eventManager = newSendEventListenerManager(node.brokerCtx) + let eventManager = newSendEventListenerManager(client.brokerCtx) defer: await eventManager.teardown() @@ -424,7 +439,7 @@ suite "Waku API - Send": ContentTopic("/waku/2/default-content/proto"), "test payload" ) - let requestId = (await node.send(envelope)).valueOr: + let requestId = (await client.messaging.send(envelope)).valueOr: raiseAssert error echo "Sent message with requestId=", requestId @@ -433,5 +448,5 @@ suite "Waku API - Send": discard await eventManager.waitForEvents(eventTimeout) eventManager.validate({SendEventOutcome.Error}, requestId) - (await node.stop()).isOkOr: + (await client.stop()).isOkOr: raiseAssert "Failed to stop node: " & error diff --git a/tests/api/test_api_subscription.nim b/tests/api/test_api_subscription.nim index 32d4e742f..2208ca374 100644 --- a/tests/api/test_api_subscription.nim +++ b/tests/api/test_api_subscription.nim @@ -11,11 +11,14 @@ import waku/[ waku_node, waku_core, - events/message_events, + api/events/message, waku_relay/protocol, node/kernel_api/filter, - node/delivery_service/subscription_manager, ] +import waku/api/api +import layers/logos_delivery +import waku/node/subscription_manager +import messaging/api/events import waku/factory/waku_conf import tools/confutils/cli_args @@ -62,7 +65,7 @@ proc waitForEvents( type TestNetwork = ref object publisher: WakuNode # Relay node that publishes messages in tests. meshBuddy: WakuNode # Extra relay peer for publisher's mesh (Edge tests only). - subscriber: Waku + subscriber: LogosDelivery # The receiver node in tests. Edge node in edge tests, Core node in relay tests. publisherPeerInfo: RemotePeerInfo @@ -81,12 +84,13 @@ proc createApiNodeConf( conf.rest = false result = conf -proc setupSubscriberNode(conf: WakuNodeConf): Future[Waku] {.async.} = - var node: Waku +proc setupSubscriberNode(conf: WakuNodeConf): Future[LogosDelivery] {.async.} = + var client: LogosDelivery lockNewGlobalBrokerContext: - node = (await createNode(conf)).expect("Failed to create subscriber node") - (await startWaku(addr node)).expect("Failed to start subscriber node") - return node + let node = (await createNode(conf)).expect("Failed to create subscriber node") + client = LogosDelivery.new(MessagingClient, node).expect("Failed to wrap subscriber in LogosDelivery") + (await client.start()).expect("Failed to start subscriber node") + return client proc setupNetwork( numShards: uint16 = 1, mode: cli_args.WakuMode = cli_args.WakuMode.Core @@ -138,7 +142,7 @@ proc setupNetwork( net.subscriber = await setupSubscriberNode(createApiNodeConf(mode, numShards)) - await net.subscriber.node.connectToNodes(@[net.publisherPeerInfo]) + await net.subscriber.waku.node.connectToNodes(@[net.publisherPeerInfo]) return net @@ -167,8 +171,8 @@ proc waitForMesh(node: WakuNode, shard: PubsubTopic) {.async.} = await sleepAsync(100.milliseconds) raise newException(ValueError, "GossipSub Mesh failed to stabilize on " & shard) -proc waitForEdgeSubs(w: Waku, shard: PubsubTopic) {.async.} = - let sm = w.deliveryService.subscriptionManager +proc waitForEdgeSubs(w: LogosDelivery, shard: PubsubTopic) {.async.} = + let sm = w.waku.node.subscriptionManager for _ in 0 ..< 50: if sm.edgeFilterPeerCount(shard) > 0: return @@ -179,7 +183,7 @@ proc publishToMesh( net: TestNetwork, contentTopic: ContentTopic, payload: seq[byte] ): Future[Result[int, string]] {.async.} = # Publishes a message from "publisher" via relay into the gossipsub mesh. - let shard = net.subscriber.node.getRelayShard(contentTopic) + let shard = net.subscriber.waku.node.getRelayShard(contentTopic) await waitForMesh(net.publisher, shard) let msg = WakuMessage( payload: payload, contentTopic: contentTopic, version: 0, timestamp: now() @@ -191,18 +195,18 @@ proc publishToMeshAfterEdgeReady( ): Future[Result[int, string]] {.async.} = # First, ensure "subscriber" node (an edge node) is subscribed and ready to receive. # Afterwards, "publisher" (relay node) sends the message in the gossipsub network. - let shard = net.subscriber.node.getRelayShard(contentTopic) + let shard = net.subscriber.waku.node.getRelayShard(contentTopic) await waitForEdgeSubs(net.subscriber, shard) return await net.publishToMesh(contentTopic, payload) -suite "Messaging API, SubscriptionManager": +suite "Messaging API, WakuSubscriptionManager": asyncTest "Subscription API, relay node auto subscribe and receive message": let net = await setupNetwork(1) defer: await net.teardown() let testTopic = ContentTopic("/waku/2/test-content/proto") - (await net.subscriber.subscribe(testTopic)).expect( + (await net.subscriber.messaging.subscribe(testTopic)).expect( "subscriberNode failed to subscribe" ) @@ -225,7 +229,7 @@ suite "Messaging API, SubscriptionManager": let subbedTopic = ContentTopic("/waku/2/subbed-topic/proto") let ignoredTopic = ContentTopic("/waku/2/ignored-topic/proto") - (await net.subscriber.subscribe(subbedTopic)).expect("failed to subscribe") + (await net.subscriber.messaging.subscribe(subbedTopic)).expect("failed to subscribe") let eventManager = newReceiveEventListenerManager(net.subscriber.brokerCtx, 1) defer: @@ -245,8 +249,8 @@ suite "Messaging API, SubscriptionManager": let testTopic = ContentTopic("/waku/2/unsub-test/proto") - (await net.subscriber.subscribe(testTopic)).expect("failed to subscribe") - net.subscriber.unsubscribe(testTopic).expect("failed to unsubscribe") + (await net.subscriber.messaging.subscribe(testTopic)).expect("failed to subscribe") + net.subscriber.messaging.unsubscribe(testTopic).expect("failed to unsubscribe") let eventManager = newReceiveEventListenerManager(net.subscriber.brokerCtx, 1) defer: @@ -266,14 +270,14 @@ suite "Messaging API, SubscriptionManager": let topicA = ContentTopic("/waku/2/topic-a/proto") let topicB = ContentTopic("/waku/2/topic-b/proto") - (await net.subscriber.subscribe(topicA)).expect("failed to sub A") - (await net.subscriber.subscribe(topicB)).expect("failed to sub B") + (await net.subscriber.messaging.subscribe(topicA)).expect("failed to sub A") + (await net.subscriber.messaging.subscribe(topicB)).expect("failed to sub B") let eventManager = newReceiveEventListenerManager(net.subscriber.brokerCtx, 1) defer: await eventManager.teardown() - net.subscriber.unsubscribe(topicA).expect("failed to unsub A") + net.subscriber.messaging.unsubscribe(topicA).expect("failed to unsub A") discard (await net.publishToMesh(topicA, "Dropped Message".toBytes())).expect( "Publish A failed" @@ -292,9 +296,9 @@ suite "Messaging API, SubscriptionManager": let glitchTopic = ContentTopic("/waku/2/glitch/proto") - (await net.subscriber.subscribe(glitchTopic)).expect("failed to sub") - (await net.subscriber.subscribe(glitchTopic)).expect("failed to double sub") - net.subscriber.unsubscribe(glitchTopic).expect("failed to unsub") + (await net.subscriber.messaging.subscribe(glitchTopic)).expect("failed to sub") + (await net.subscriber.messaging.subscribe(glitchTopic)).expect("failed to double sub") + net.subscriber.messaging.unsubscribe(glitchTopic).expect("failed to unsub") let eventManager = newReceiveEventListenerManager(net.subscriber.brokerCtx, 1) defer: @@ -315,7 +319,7 @@ suite "Messaging API, SubscriptionManager": let testTopic = ContentTopic("/waku/2/resub-test/proto") # Subscribe - (await net.subscriber.subscribe(testTopic)).expect("Initial sub failed") + (await net.subscriber.messaging.subscribe(testTopic)).expect("Initial sub failed") var eventManager = newReceiveEventListenerManager(net.subscriber.brokerCtx, 1) discard @@ -325,7 +329,7 @@ suite "Messaging API, SubscriptionManager": await eventManager.teardown() # Unsubscribe and verify teardown - net.subscriber.unsubscribe(testTopic).expect("Unsub failed") + net.subscriber.messaging.unsubscribe(testTopic).expect("Unsub failed") eventManager = newReceiveEventListenerManager(net.subscriber.brokerCtx, 1) discard @@ -335,7 +339,7 @@ suite "Messaging API, SubscriptionManager": await eventManager.teardown() # Resubscribe - (await net.subscriber.subscribe(testTopic)).expect("Resub failed") + (await net.subscriber.messaging.subscribe(testTopic)).expect("Resub failed") eventManager = newReceiveEventListenerManager(net.subscriber.brokerCtx, 1) discard @@ -354,13 +358,13 @@ suite "Messaging API, SubscriptionManager": # generate two content topics that land in two different shards var i = 0 - while net.subscriber.node.getRelayShard(topicA) == - net.subscriber.node.getRelayShard(topicB): + while net.subscriber.waku.node.getRelayShard(topicA) == + net.subscriber.waku.node.getRelayShard(topicB): topicB = ContentTopic("/appB" & $i & "/2/shard-test-b/proto") inc i - (await net.subscriber.subscribe(topicA)).expect("failed to sub A") - (await net.subscriber.subscribe(topicB)).expect("failed to sub B") + (await net.subscriber.messaging.subscribe(topicA)).expect("failed to sub A") + (await net.subscriber.messaging.subscribe(topicB)).expect("failed to sub B") let eventManager = newReceiveEventListenerManager(net.subscriber.brokerCtx, 2) defer: @@ -417,7 +421,7 @@ suite "Messaging API, SubscriptionManager": # subscribe to all content topics we generated for t in allTopics: - (await net.subscriber.subscribe(t)).expect("sub failed") + (await net.subscriber.messaging.subscribe(t)).expect("sub failed") activeSubs.add(t) await verifyNetworkState(activeSubs) @@ -425,7 +429,7 @@ suite "Messaging API, SubscriptionManager": # unsubscribe from some content topics for i in 0 ..< 50: let t = allTopics[i] - net.subscriber.unsubscribe(t).expect("unsub failed") + net.subscriber.messaging.unsubscribe(t).expect("unsub failed") let idx = activeSubs.find(t) if idx >= 0: @@ -436,7 +440,7 @@ suite "Messaging API, SubscriptionManager": # re-subscribe to some content topics for i in 0 ..< 25: let t = allTopics[i] - (await net.subscriber.subscribe(t)).expect("resub failed") + (await net.subscriber.messaging.subscribe(t)).expect("resub failed") activeSubs.add(t) await verifyNetworkState(activeSubs) @@ -447,7 +451,7 @@ suite "Messaging API, SubscriptionManager": await net.teardown() let testTopic = ContentTopic("/waku/2/test-content/proto") - (await net.subscriber.subscribe(testTopic)).expect("failed to subscribe") + (await net.subscriber.messaging.subscribe(testTopic)).expect("failed to subscribe") let eventManager = newReceiveEventListenerManager(net.subscriber.brokerCtx, 1) defer: @@ -468,7 +472,7 @@ suite "Messaging API, SubscriptionManager": let subbedTopic = ContentTopic("/waku/2/subbed-topic/proto") let ignoredTopic = ContentTopic("/waku/2/ignored-topic/proto") - (await net.subscriber.subscribe(subbedTopic)).expect("failed to subscribe") + (await net.subscriber.messaging.subscribe(subbedTopic)).expect("failed to subscribe") let eventManager = newReceiveEventListenerManager(net.subscriber.brokerCtx, 1) defer: @@ -488,8 +492,8 @@ suite "Messaging API, SubscriptionManager": let testTopic = ContentTopic("/waku/2/unsub-test/proto") - (await net.subscriber.subscribe(testTopic)).expect("failed to subscribe") - net.subscriber.unsubscribe(testTopic).expect("failed to unsubscribe") + (await net.subscriber.messaging.subscribe(testTopic)).expect("failed to subscribe") + net.subscriber.messaging.unsubscribe(testTopic).expect("failed to unsubscribe") let eventManager = newReceiveEventListenerManager(net.subscriber.brokerCtx, 1) defer: @@ -509,17 +513,17 @@ suite "Messaging API, SubscriptionManager": let topicA = ContentTopic("/waku/2/topic-a/proto") let topicB = ContentTopic("/waku/2/topic-b/proto") - (await net.subscriber.subscribe(topicA)).expect("failed to sub A") - (await net.subscriber.subscribe(topicB)).expect("failed to sub B") + (await net.subscriber.messaging.subscribe(topicA)).expect("failed to sub A") + (await net.subscriber.messaging.subscribe(topicB)).expect("failed to sub B") - let shard = net.subscriber.node.getRelayShard(topicA) + let shard = net.subscriber.waku.node.getRelayShard(topicA) await waitForEdgeSubs(net.subscriber, shard) let eventManager = newReceiveEventListenerManager(net.subscriber.brokerCtx, 1) defer: await eventManager.teardown() - net.subscriber.unsubscribe(topicA).expect("failed to unsub A") + net.subscriber.messaging.unsubscribe(topicA).expect("failed to unsub A") discard (await net.publishToMesh(topicA, "Dropped Message".toBytes())).expect( "Publish A failed" @@ -538,7 +542,7 @@ suite "Messaging API, SubscriptionManager": let testTopic = ContentTopic("/waku/2/resub-test/proto") - (await net.subscriber.subscribe(testTopic)).expect("Initial sub failed") + (await net.subscriber.messaging.subscribe(testTopic)).expect("Initial sub failed") var eventManager = newReceiveEventListenerManager(net.subscriber.brokerCtx, 1) discard (await net.publishToMeshAfterEdgeReady(testTopic, "Msg 1".toBytes())).expect( @@ -548,7 +552,7 @@ suite "Messaging API, SubscriptionManager": require await eventManager.waitForEvents(TestTimeout) await eventManager.teardown() - net.subscriber.unsubscribe(testTopic).expect("Unsub failed") + net.subscriber.messaging.unsubscribe(testTopic).expect("Unsub failed") eventManager = newReceiveEventListenerManager(net.subscriber.brokerCtx, 1) discard @@ -557,7 +561,7 @@ suite "Messaging API, SubscriptionManager": check not await eventManager.waitForEvents(NegativeTestTimeout) await eventManager.teardown() - (await net.subscriber.subscribe(testTopic)).expect("Resub failed") + (await net.subscriber.messaging.subscribe(testTopic)).expect("Resub failed") eventManager = newReceiveEventListenerManager(net.subscriber.brokerCtx, 1) discard (await net.publishToMeshAfterEdgeReady(testTopic, "Msg 2".toBytes())).expect( @@ -618,26 +622,29 @@ suite "Messaging API, SubscriptionManager": await meshBuddy.connectToNodes(@[publisherPeerInfo]) let conf = createApiNodeConf(cli_args.WakuMode.Edge, numShards) - var subscriber: Waku + var subscriber: LogosDelivery lockNewGlobalBrokerContext: - subscriber = (await createNode(conf)).expect("Failed to create edge subscriber") - (await startWaku(addr subscriber)).expect("Failed to start edge subscriber") + let node = (await createNode(conf)).expect("Failed to create edge subscriber") + subscriber = LogosDelivery.new(MessagingClient, node).expect( + "Failed to wrap edge subscriber in LogosDelivery" + ) + (await subscriber.start()).expect("Failed to start edge subscriber") # Connect edge subscriber to both filter servers so selectPeers finds both - await subscriber.node.connectToNodes(@[publisherPeerInfo, meshBuddyPeerInfo]) + await subscriber.waku.node.connectToNodes(@[publisherPeerInfo, meshBuddyPeerInfo]) let testTopic = ContentTopic("/waku/2/failover-test/proto") - let shard = subscriber.node.getRelayShard(testTopic) + let shard = subscriber.waku.node.getRelayShard(testTopic) - (await subscriber.subscribe(testTopic)).expect("Failed to subscribe") + (await subscriber.messaging.subscribe(testTopic)).expect("Failed to subscribe") # Wait for dialing both filter servers (HealthyThreshold = 2) for _ in 0 ..< 100: - if subscriber.deliveryService.subscriptionManager.edgeFilterPeerCount(shard) >= 2: + if subscriber.waku.node.subscriptionManager.edgeFilterPeerCount(shard) >= 2: break await sleepAsync(100.milliseconds) - check subscriber.deliveryService.subscriptionManager.edgeFilterPeerCount(shard) >= 2 + check subscriber.waku.node.subscriptionManager.edgeFilterPeerCount(shard) >= 2 # Verify message delivery with both servers alive await waitForMesh(publisher, shard) @@ -656,15 +663,15 @@ suite "Messaging API, SubscriptionManager": await eventManager.teardown() # Disconnect meshBuddy from edge (keeps relay mesh alive for publishing) - await subscriber.node.disconnectNode(meshBuddyPeerInfo) + await subscriber.waku.node.disconnectNode(meshBuddyPeerInfo) # Wait for the dead peer to be pruned for _ in 0 ..< 50: - if subscriber.deliveryService.subscriptionManager.edgeFilterPeerCount(shard) < 2: + if subscriber.waku.node.subscriptionManager.edgeFilterPeerCount(shard) < 2: break await sleepAsync(100.milliseconds) - check subscriber.deliveryService.subscriptionManager.edgeFilterPeerCount(shard) >= 1 + check subscriber.waku.node.subscriptionManager.edgeFilterPeerCount(shard) >= 1 # Verify messages still arrive through the surviving filter server (publisher) eventManager = newReceiveEventListenerManager(subscriber.brokerCtx, 1) @@ -755,38 +762,41 @@ suite "Messaging API, SubscriptionManager": await sparePeer.connectToNodes(@[publisherPeerInfo]) let conf = createApiNodeConf(cli_args.WakuMode.Edge, numShards) - var subscriber: Waku + var subscriber: LogosDelivery lockNewGlobalBrokerContext: - subscriber = (await createNode(conf)).expect("Failed to create edge subscriber") - (await startWaku(addr subscriber)).expect("Failed to start edge subscriber") + let node = (await createNode(conf)).expect("Failed to create edge subscriber") + subscriber = LogosDelivery.new(MessagingClient, node).expect( + "Failed to wrap edge subscriber in LogosDelivery" + ) + (await subscriber.start()).expect("Failed to start edge subscriber") - await subscriber.node.connectToNodes( + await subscriber.waku.node.connectToNodes( @[publisherPeerInfo, meshBuddyPeerInfo, sparePeerInfo] ) let testTopic = ContentTopic("/waku/2/replacement-test/proto") - let shard = subscriber.node.getRelayShard(testTopic) + let shard = subscriber.waku.node.getRelayShard(testTopic) - (await subscriber.subscribe(testTopic)).expect("Failed to subscribe") + (await subscriber.messaging.subscribe(testTopic)).expect("Failed to subscribe") # Wait for 2 confirmed peers (HealthyThreshold). The 3rd is available but not dialed. for _ in 0 ..< 100: - if subscriber.deliveryService.subscriptionManager.edgeFilterPeerCount(shard) >= 2: + if subscriber.waku.node.subscriptionManager.edgeFilterPeerCount(shard) >= 2: break await sleepAsync(100.milliseconds) - require subscriber.deliveryService.subscriptionManager.edgeFilterPeerCount(shard) == + require subscriber.waku.node.subscriptionManager.edgeFilterPeerCount(shard) == 2 - await subscriber.node.disconnectNode(meshBuddyPeerInfo) + await subscriber.waku.node.disconnectNode(meshBuddyPeerInfo) # Wait for the sub loop to detect the loss and dial a replacement for _ in 0 ..< 100: - if subscriber.deliveryService.subscriptionManager.edgeFilterPeerCount(shard) >= 2: + if subscriber.waku.node.subscriptionManager.edgeFilterPeerCount(shard) >= 2: break await sleepAsync(100.milliseconds) - check subscriber.deliveryService.subscriptionManager.edgeFilterPeerCount(shard) >= 2 + check subscriber.waku.node.subscriptionManager.edgeFilterPeerCount(shard) >= 2 await waitForMesh(publisher, shard) diff --git a/tests/api/test_node_conf.nim b/tests/api/test_node_conf.nim index 8798c5cc5..147bb2457 100644 --- a/tests/api/test_node_conf.nim +++ b/tests/api/test_node_conf.nim @@ -4,7 +4,6 @@ import std/[options, json, strutils], results, stint, testutils/unittests import json_serialization, confutils, confutils/std/net import tools/confutils/cli_args, - waku/api/api_conf, waku/factory/waku_conf, waku/factory/networks_config, waku/factory/conf_builder/conf_builder, @@ -341,53 +340,6 @@ suite "WakuNodeConf JSON -> WakuConf integration": check: wakuConf.maxMessageSizeBytes == 100'u64 * 1024'u64 -# ---- Deprecated NodeConfig tests (kept for backward compatibility) ---- - -{.push warning[Deprecated]: off.} - -import waku/api/api_conf - -suite "NodeConfig (deprecated) - toWakuConf": - test "Minimal configuration": - let nodeConfig = NodeConfig.init(ethRpcEndpoints = @["http://someaddress"]) - let wakuConfRes = api_conf.toWakuConf(nodeConfig) - let wakuConf = wakuConfRes.valueOr: - raiseAssert error - wakuConf.validate().isOkOr: - raiseAssert error - check: - wakuConf.clusterId == 1 - wakuConf.shardingConf.numShardsInCluster == 8 - wakuConf.staticNodes.len == 0 - - test "Edge mode configuration": - let protocolsConfig = ProtocolsConfig.init(entryNodes = @[], clusterId = 1) - let nodeConfig = - NodeConfig.init(mode = api_conf.WakuMode.Edge, protocolsConfig = protocolsConfig) - let wakuConfRes = api_conf.toWakuConf(nodeConfig) - require wakuConfRes.isOk() - let wakuConf = wakuConfRes.get() - require wakuConf.validate().isOk() - check: - wakuConf.relay == false - wakuConf.lightPush == false - wakuConf.peerExchangeService == true - - test "Core mode configuration": - let protocolsConfig = ProtocolsConfig.init(entryNodes = @[], clusterId = 1) - let nodeConfig = - NodeConfig.init(mode = api_conf.WakuMode.Core, protocolsConfig = protocolsConfig) - let wakuConfRes = api_conf.toWakuConf(nodeConfig) - require wakuConfRes.isOk() - let wakuConf = wakuConfRes.get() - require wakuConf.validate().isOk() - check: - wakuConf.relay == true - wakuConf.lightPush == true - wakuConf.peerExchangeService == true - -{.pop.} - suite "WakuConfBuilder - store retention policies": test "Multiple retention policies": ## Given diff --git a/tests/node/test_wakunode_health_monitor.nim b/tests/node/test_wakunode_health_monitor.nim index 08f641a75..a08a9a714 100644 --- a/tests/node/test_wakunode_health_monitor.nim +++ b/tests/node/test_wakunode_health_monitor.nim @@ -4,6 +4,8 @@ import std/[json, options, sequtils, strutils, tables], testutils/unittests, chronos, results import brokers/broker_context +import layers/logos_delivery + import waku/[ waku_core, @@ -15,16 +17,19 @@ import node/health_monitor/protocol_health, node/health_monitor/topic_health, node/health_monitor/node_health_monitor, - node/delivery_service/delivery_service, - node/delivery_service/subscription_manager, node/kernel_api/relay, node/kernel_api/store, node/kernel_api/lightpush, node/kernel_api/filter, - events/health_events, + api/events/health, events/peer_events, waku_archive, ] +import waku/node/subscription_manager +import waku/api/api +import waku/factory/waku +import waku/factory/waku_conf +import tools/confutils/cli_args import ../testlib/[wakunode, wakucore], ../waku_archive/archive_utils @@ -228,9 +233,9 @@ suite "Health Monitor - events": nodeA.mountMetadata(1, @[0'u16]).expect("Node A failed to mount metadata") await nodeA.start() - let ds = - DeliveryService.new(false, nodeA).expect("Failed to create DeliveryService") - ds.startDeliveryService().expect("Failed to start DeliveryService") + nodeA.subscriptionManager.startWakuSubscriptionManager().expect( + "Failed to start subscription manager" + ) let monitorA = NodeHealthMonitor.new(nodeA) @@ -263,12 +268,12 @@ suite "Health Monitor - events": await nodeB.start() var metadataFut = newFuture[void]("waitForMetadata") - let metadataLis = WakuPeerEvent + let metadataLis = EventWakuPeer .listen( nodeA.brokerCtx, - proc(evt: WakuPeerEvent): Future[void] {.async: (raises: []), gcsafe.} = + proc(evt: EventWakuPeer): Future[void] {.async: (raises: []), gcsafe.} = if not metadataFut.finished and - evt.kind == WakuPeerEventKind.EventMetadataUpdated: + evt.kind == EventWakuPeerKind.EventMetadataUpdated: metadataFut.complete() , ) @@ -277,7 +282,7 @@ suite "Health Monitor - events": await nodeA.connectToNodes(@[nodeB.switch.peerInfo.toRemotePeerInfo()]) let metadataOk = await metadataFut.withTimeout(TestConnectivityTimeLimit) - await WakuPeerEvent.dropListener(nodeA.brokerCtx, metadataLis) + await EventWakuPeer.dropListener(nodeA.brokerCtx, metadataLis) require metadataOk let connectTimeLimit = Moment.now() + TestConnectivityTimeLimit @@ -317,25 +322,27 @@ suite "Health Monitor - events": lastStatus == ConnectionStatus.Disconnected await monitorA.stopHealthMonitor() - await ds.stopDeliveryService() + await nodeA.subscriptionManager.stopWakuSubscriptionManager() await nodeA.stop() asyncTest "Edge health driven by confirmed filter subscriptions": - var nodeA: WakuNode + var clientA: LogosDelivery lockNewGlobalBrokerContext: - let nodeAKey = generateSecp256k1Key() - nodeA = newTestWakuNode(nodeAKey, parseIpAddress("127.0.0.1"), Port(0)) - await nodeA.mountFilterClient() - nodeA.mountLightpushClient() - nodeA.mountStoreClient() - require nodeA.mountAutoSharding(1, 8).isOk - nodeA.mountMetadata(1, @[0'u16]).expect("Node A failed to mount metadata") - await nodeA.start() - - let ds = - DeliveryService.new(false, nodeA).expect("Failed to create DeliveryService") - ds.startDeliveryService().expect("Failed to start DeliveryService") - let subMgr = ds.subscriptionManager + var confA = defaultWakuNodeConf().valueOr: + raiseAssert error + confA.mode = cli_args.WakuMode.Edge + confA.listenAddress = parseIpAddress("127.0.0.1") + confA.tcpPort = Port(0) + confA.discv5UdpPort = Port(0) + confA.clusterId = 1'u16 + confA.numShardsInNetwork = 8 + confA.rest = false + let wakuA = (await createNode(confA)).expect("Failed to create nodeA") + clientA = LogosDelivery.new(MessagingClient, wakuA).expect( + "Failed to wrap nodeA in LogosDelivery" + ) + (await clientA.start()).expect("Failed to start nodeA") + let subMgr = clientA.waku.node.subscriptionManager var nodeB: WakuNode lockNewGlobalBrokerContext: @@ -353,7 +360,7 @@ suite "Health Monitor - events": ) await nodeB.start() - let monitorA = NodeHealthMonitor.new(nodeA) + let monitorA = NodeHealthMonitor.new(clientA.waku.node) var lastStatus = ConnectionStatus.Disconnected @@ -366,21 +373,21 @@ suite "Health Monitor - events": monitorA.startHealthMonitor().expect("Health monitor failed to start") var metadataFut = newFuture[void]("waitForMetadata") - let metadataLis = WakuPeerEvent + let metadataLis = EventWakuPeer .listen( - nodeA.brokerCtx, - proc(evt: WakuPeerEvent): Future[void] {.async: (raises: []), gcsafe.} = + clientA.brokerCtx, + proc(evt: EventWakuPeer): Future[void] {.async: (raises: []), gcsafe.} = if not metadataFut.finished and - evt.kind == WakuPeerEventKind.EventMetadataUpdated: + evt.kind == EventWakuPeerKind.EventMetadataUpdated: metadataFut.complete() , ) .expect("Failed to listen for metadata") - await nodeA.connectToNodes(@[nodeB.switch.peerInfo.toRemotePeerInfo()]) + await clientA.waku.node.connectToNodes(@[nodeB.switch.peerInfo.toRemotePeerInfo()]) let metadataOk = await metadataFut.withTimeout(TestConnectivityTimeLimit) - await WakuPeerEvent.dropListener(nodeA.brokerCtx, metadataLis) + await EventWakuPeer.dropListener(clientA.brokerCtx, metadataLis) require metadataOk var deadline = Moment.now() + TestConnectivityTimeLimit @@ -396,7 +403,7 @@ suite "Health Monitor - events": let shardHealthLis = EventShardTopicHealthChange .listen( - nodeA.brokerCtx, + clientA.brokerCtx, proc( evt: EventShardTopicHealthChange ): Future[void] {.async: (raises: []), gcsafe.} = @@ -410,10 +417,10 @@ suite "Health Monitor - events": .expect("Failed to listen for shard health") let contentTopic = ContentTopic("/waku/2/default-content/proto") - subMgr.subscribe(contentTopic).expect("Failed to subscribe") + subMgr.edgeSubscribe(contentTopic).expect("Failed to subscribe") let shardHealthOk = await shardHealthFut.withTimeout(TestConnectivityTimeLimit) - await EventShardTopicHealthChange.dropListener(nodeA.brokerCtx, shardHealthLis) + await EventShardTopicHealthChange.dropListener(clientA.brokerCtx, shardHealthLis) check shardHealthOk == true check subMgr.edgeFilterSubStates.len > 0 @@ -428,7 +435,6 @@ suite "Health Monitor - events": check lastStatus == ConnectionStatus.PartiallyConnected - await ds.stopDeliveryService() await monitorA.stopHealthMonitor() await nodeB.stop() - await nodeA.stop() + (await clientA.stop()).expect("Failed to stop clientA") diff --git a/tests/test_waku.nim b/tests/test_waku.nim index cf5675716..045152706 100644 --- a/tests/test_waku.nim +++ b/tests/test_waku.nim @@ -3,6 +3,7 @@ import chronos, testutils/unittests, std/options import waku +import waku/api/api import tools/confutils/cli_args suite "Waku API - Create node": diff --git a/tests/waku_discv5/test_waku_discv5.nim b/tests/waku_discv5/test_waku_discv5.nim index 936c01826..0346e7d58 100644 --- a/tests/waku_discv5/test_waku_discv5.nim +++ b/tests/waku_discv5/test_waku_discv5.nim @@ -12,6 +12,8 @@ import libp2p/crypto/secp, libp2p/protocols/rendezvous +import brokers/broker_context + import waku/[ waku_core/topics, @@ -429,10 +431,14 @@ suite "Waku Discovery v5": let conf = confBuilder.build().valueOr: raiseAssert error - let waku0 = (await Waku.new(conf)).valueOr: - raiseAssert error - (waitFor startWaku(addr waku0)).isOkOr: - raiseAssert error + var waku0, waku1, waku2: Waku + + # Each Waku instance must have its own broker context; they must not share one. + lockNewGlobalBrokerContext: + waku0 = (await Waku.new(conf)).valueOr: + raiseAssert error + (waitFor startWaku(addr waku0)).isOkOr: + raiseAssert error confBuilder.withNodeKey(crypto.PrivateKey.random(Secp256k1, myRng[])[]) confBuilder.discv5Conf.withBootstrapNodes(@[waku0.node.enr.toURI()]) @@ -443,13 +449,13 @@ suite "Waku Discovery v5": let conf1 = confBuilder.build().valueOr: raiseAssert error - let waku1 = (await Waku.new(conf1)).valueOr: - raiseAssert error - (waitFor startWaku(addr waku1)).isOkOr: - raiseAssert error - - await waku1.node.mountPeerExchange() - await waku1.node.mountRendezvous(conf.clusterId) + lockNewGlobalBrokerContext: + waku1 = (await Waku.new(conf1)).valueOr: + raiseAssert error + (waitFor startWaku(addr waku1)).isOkOr: + raiseAssert error + await waku1.node.mountPeerExchange() + await waku1.node.mountRendezvous(conf.clusterId) confBuilder.discv5Conf.withBootstrapNodes(@[waku1.node.enr.toURI()]) confBuilder.withP2pTcpPort(60003.Port) @@ -459,10 +465,11 @@ suite "Waku Discovery v5": let conf2 = confBuilder.build().valueOr: raiseAssert error - let waku2 = (await Waku.new(conf2)).valueOr: - raiseAssert error - (waitFor startWaku(addr waku2)).isOkOr: - raiseAssert error + lockNewGlobalBrokerContext: + waku2 = (await Waku.new(conf2)).valueOr: + raiseAssert error + (waitFor startWaku(addr waku2)).isOkOr: + raiseAssert error # leave some time for discv5 to act await sleepAsync(chronos.seconds(10)) diff --git a/waku/api.nim b/waku/api.nim index a977a062a..82722afda 100644 --- a/waku/api.nim +++ b/waku/api.nim @@ -1,5 +1,4 @@ -import ./api/[api, api_conf] -import ./events/message_events -import tools/confutils/entry_nodes +import ./api/requests +import ./api/events -export api, api_conf, entry_nodes, message_events +export requests, events diff --git a/waku/api/api.nim b/waku/api/api.nim index 1eee982fd..0df8a5bee 100644 --- a/waku/api/api.nim +++ b/waku/api/api.nim @@ -1,12 +1,7 @@ import chronicles, chronos, results import waku/factory/waku -import waku/[requests/health_requests, waku_core, waku_node] -import waku/node/delivery_service/send_service -import waku/node/delivery_service/subscription_manager -import libp2p/peerid import ../../tools/confutils/cli_args -import ./[api_conf, types] export cli_args @@ -23,54 +18,3 @@ proc createNode*(conf: WakuNodeConf): Future[Result[Waku, string]] {.async.} = return err("Failed setting up Waku: " & $error) return ok(wakuRes) - -proc checkApiAvailability(w: Waku): Result[void, string] = - if w.isNil(): - return err("Waku node is not initialized") - - # TODO: Conciliate request-bouncing health checks here with unit testing. - # (For now, better to just allow all sends and rely on retries.) - - return ok() - -proc subscribe*( - w: Waku, contentTopic: ContentTopic -): Future[Result[void, string]] {.async.} = - ?checkApiAvailability(w) - - return w.deliveryService.subscriptionManager.subscribe(contentTopic) - -proc unsubscribe*(w: Waku, contentTopic: ContentTopic): Result[void, string] = - ?checkApiAvailability(w) - - return w.deliveryService.subscriptionManager.unsubscribe(contentTopic) - -proc send*( - w: Waku, envelope: MessageEnvelope -): Future[Result[RequestId, string]] {.async.} = - ?checkApiAvailability(w) - - let isSubbed = w.deliveryService.subscriptionManager - .isSubscribed(envelope.contentTopic) - .valueOr(false) - if not isSubbed: - info "Auto-subscribing to topic on send", contentTopic = envelope.contentTopic - w.deliveryService.subscriptionManager.subscribe(envelope.contentTopic).isOkOr: - warn "Failed to auto-subscribe", error = error - return err("Failed to auto-subscribe before sending: " & error) - - let requestId = RequestId.new(w.rng) - - let deliveryTask = DeliveryTask.new(requestId, envelope, w.brokerCtx).valueOr: - return err("API send: Failed to create delivery task: " & error) - - info "API send: scheduling delivery task", - requestId = $requestId, - pubsubTopic = deliveryTask.pubsubTopic, - contentTopic = deliveryTask.msg.contentTopic, - msgHash = deliveryTask.msgHash.to0xHex(), - myPeerId = w.node.peerId() - - asyncSpawn w.deliveryService.sendService.send(deliveryTask) - - return ok(requestId) diff --git a/waku/api/api_conf.nim b/waku/api/api_conf.nim deleted file mode 100644 index 3606be596..000000000 --- a/waku/api/api_conf.nim +++ /dev/null @@ -1,533 +0,0 @@ -import std/[net, options] - -import results -import json_serialization, json_serialization/std/options as json_options - -import - waku/common/utils/parse_size_units, - waku/common/logging, - waku/factory/waku_conf, - waku/factory/conf_builder/conf_builder, - waku/factory/networks_config, - tools/confutils/entry_nodes - -export json_serialization, json_options - -type AutoShardingConfig* = object - numShardsInCluster*: uint16 - -type RlnConfig* = object - contractAddress*: string - chainId*: uint - epochSizeSec*: uint64 - -type NetworkingConfig* = object - listenIpv4*: string - p2pTcpPort*: uint16 - discv5UdpPort*: uint16 - -type MessageValidation* = object - maxMessageSize*: string # Accepts formats like "150 KiB", "1500 B" - rlnConfig*: Option[RlnConfig] - -type ProtocolsConfig* = object - entryNodes: seq[string] - staticStoreNodes: seq[string] - clusterId: uint16 - autoShardingConfig: AutoShardingConfig - messageValidation: MessageValidation - -const DefaultNetworkingConfig* = - NetworkingConfig(listenIpv4: "0.0.0.0", p2pTcpPort: 60000, discv5UdpPort: 9000) - -const DefaultAutoShardingConfig* = AutoShardingConfig(numShardsInCluster: 1) - -const DefaultMessageValidation* = - MessageValidation(maxMessageSize: "150 KiB", rlnConfig: none(RlnConfig)) - -proc init*( - T: typedesc[ProtocolsConfig], - entryNodes: seq[string], - staticStoreNodes: seq[string] = @[], - clusterId: uint16, - autoShardingConfig: AutoShardingConfig = DefaultAutoShardingConfig, - messageValidation: MessageValidation = DefaultMessageValidation, -): T = - return T( - entryNodes: entryNodes, - staticStoreNodes: staticStoreNodes, - clusterId: clusterId, - autoShardingConfig: autoShardingConfig, - messageValidation: messageValidation, - ) - -const TheWakuNetworkPreset* = ProtocolsConfig( - entryNodes: @[ - "enrtree://AIRVQ5DDA4FFWLRBCHJWUWOO6X6S4ZTZ5B667LQ6AJU6PEYDLRD5O@sandbox.waku.nodes.status.im" - ], - staticStoreNodes: @[], - clusterId: 1, - autoShardingConfig: AutoShardingConfig(numShardsInCluster: 8), - messageValidation: MessageValidation( - maxMessageSize: "150 KiB", - rlnConfig: some( - RlnConfig( - contractAddress: "0xB9cd878C90E49F797B4431fBF4fb333108CB90e6", - chainId: 59141, - epochSizeSec: 600, # 10 minutes - ) - ), - ), -) - -type WakuMode* {.pure.} = enum - Edge - Core - -type NodeConfig* {. - requiresInit, deprecated: "Use WakuNodeConf from tools/confutils/cli_args instead" -.} = object - mode: WakuMode - protocolsConfig: ProtocolsConfig - networkingConfig: NetworkingConfig - ethRpcEndpoints: seq[string] - p2pReliability: bool - logLevel: LogLevel - logFormat: LogFormat - -proc init*( - T: typedesc[NodeConfig], - mode: WakuMode = WakuMode.Core, - protocolsConfig: ProtocolsConfig = TheWakuNetworkPreset, - networkingConfig: NetworkingConfig = DefaultNetworkingConfig, - ethRpcEndpoints: seq[string] = @[], - p2pReliability: bool = false, - logLevel: LogLevel = LogLevel.INFO, - logFormat: LogFormat = LogFormat.TEXT, -): T = - return T( - mode: mode, - protocolsConfig: protocolsConfig, - networkingConfig: networkingConfig, - ethRpcEndpoints: ethRpcEndpoints, - p2pReliability: p2pReliability, - logLevel: logLevel, - logFormat: logFormat, - ) - -# -- Getters for ProtocolsConfig (private fields) - used for testing -- - -proc entryNodes*(c: ProtocolsConfig): seq[string] = - c.entryNodes - -proc staticStoreNodes*(c: ProtocolsConfig): seq[string] = - c.staticStoreNodes - -proc clusterId*(c: ProtocolsConfig): uint16 = - c.clusterId - -proc autoShardingConfig*(c: ProtocolsConfig): AutoShardingConfig = - c.autoShardingConfig - -proc messageValidation*(c: ProtocolsConfig): MessageValidation = - c.messageValidation - -# -- Getters for NodeConfig (private fields) - used for testing -- - -proc mode*(c: NodeConfig): WakuMode = - c.mode - -proc protocolsConfig*(c: NodeConfig): ProtocolsConfig = - c.protocolsConfig - -proc networkingConfig*(c: NodeConfig): NetworkingConfig = - c.networkingConfig - -proc ethRpcEndpoints*(c: NodeConfig): seq[string] = - c.ethRpcEndpoints - -proc p2pReliability*(c: NodeConfig): bool = - c.p2pReliability - -proc logLevel*(c: NodeConfig): LogLevel = - c.logLevel - -proc logFormat*(c: NodeConfig): LogFormat = - c.logFormat - -proc toWakuConf*( - nodeConfig: NodeConfig -): Result[WakuConf, string] {.deprecated: "Use WakuNodeConf.toWakuConf instead".} = - var b = WakuConfBuilder.init() - - # Apply log configuration - b.withLogLevel(nodeConfig.logLevel) - b.withLogFormat(nodeConfig.logFormat) - - # Apply networking configuration - let networkingConfig = nodeConfig.networkingConfig - let ip = parseIpAddress(networkingConfig.listenIpv4) - - b.withP2pListenAddress(ip) - b.withP2pTcpPort(networkingConfig.p2pTcpPort) - b.discv5Conf.withUdpPort(networkingConfig.discv5UdpPort) - - case nodeConfig.mode - of Core: - b.withRelay(true) - - # Metadata is always mounted - - b.filterServiceConf.withEnabled(true) - b.filterServiceConf.withMaxPeersToServe(20) - - b.withLightPush(true) - - b.discv5Conf.withEnabled(true) - b.withPeerExchange(true) - b.withRendezvous(true) - - # TODO: fix store as client usage - - b.rateLimitConf.withRateLimits(@["filter:100/1s", "lightpush:5/1s", "px:5/1s"]) - of Edge: - # All client side protocols are mounted by default - # Peer exchange client is always enabled and start_node will start the px loop - # Metadata is always mounted - b.withPeerExchange(true) - # switch off all service side protocols and relay - b.withRelay(false) - b.filterServiceConf.withEnabled(false) - b.withLightPush(false) - b.storeServiceConf.withEnabled(false) - # Leave discv5 and rendezvous for user choice - - ## Network Conf - let protocolsConfig = nodeConfig.protocolsConfig - - # Set cluster ID - b.withClusterId(protocolsConfig.clusterId) - - # Set sharding configuration - b.withShardingConf(ShardingConfKind.AutoSharding) - let autoShardingConfig = protocolsConfig.autoShardingConfig - b.withNumShardsInCluster(autoShardingConfig.numShardsInCluster) - - # Process entry nodes - supports enrtree:, enr:, and multiaddress formats - if protocolsConfig.entryNodes.len > 0: - let (enrTreeUrls, bootstrapEnrs, staticNodesFromEntry) = processEntryNodes( - protocolsConfig.entryNodes - ).valueOr: - return err("Failed to process entry nodes: " & error) - - # Set ENRTree URLs for DNS discovery - if enrTreeUrls.len > 0: - for url in enrTreeUrls: - b.dnsDiscoveryConf.withEnrTreeUrl(url) - b.dnsDiscoveryconf.withNameServers( - @[parseIpAddress("1.1.1.1"), parseIpAddress("1.0.0.1")] - ) - - # Set ENR records as bootstrap nodes for discv5 - if bootstrapEnrs.len > 0: - b.discv5Conf.withBootstrapNodes(bootstrapEnrs) - - # Add static nodes (multiaddrs and those extracted from ENR entries) - if staticNodesFromEntry.len > 0: - b.withStaticNodes(staticNodesFromEntry) - - # TODO: verify behaviour - # Set static store nodes - if protocolsConfig.staticStoreNodes.len > 0: - b.withStaticNodes(protocolsConfig.staticStoreNodes) - - # Set message validation - let msgValidation = protocolsConfig.messageValidation - let maxSizeBytes = parseMsgSize(msgValidation.maxMessageSize).valueOr: - return err("Failed to parse max message size: " & error) - b.withMaxMessageSize(maxSizeBytes) - - # Set RLN config if provided - if msgValidation.rlnConfig.isSome(): - let rlnConfig = msgValidation.rlnConfig.get() - b.rlnRelayConf.withEnabled(true) - b.rlnRelayConf.withEthContractAddress(rlnConfig.contractAddress) - b.rlnRelayConf.withChainId(rlnConfig.chainId) - b.rlnRelayConf.withEpochSizeSec(rlnConfig.epochSizeSec) - b.rlnRelayConf.withDynamic(true) - b.rlnRelayConf.withEthClientUrls(nodeConfig.ethRpcEndpoints) - - # TODO: we should get rid of those two - b.rlnRelayconf.withUserMessageLimit(100) - - ## Various configurations - b.withNatStrategy("any") - b.withP2PReliability(nodeConfig.p2pReliability) - - let wakuConf = b.build().valueOr: - return err("Failed to build configuration: " & error) - - wakuConf.validate().isOkOr: - return err("Failed to validate configuration: " & error) - - return ok(wakuConf) - -# ---- JSON serialization (writeValue / readValue) ---- -# ---------- AutoShardingConfig ---------- - -proc writeValue*(w: var JsonWriter, val: AutoShardingConfig) {.raises: [IOError].} = - w.beginRecord() - w.writeField("numShardsInCluster", val.numShardsInCluster) - w.endRecord() - -proc readValue*( - r: var JsonReader, val: var AutoShardingConfig -) {.raises: [SerializationError, IOError].} = - var numShardsInCluster: Option[uint16] - - for fieldName in readObjectFields(r): - case fieldName - of "numShardsInCluster": - numShardsInCluster = some(r.readValue(uint16)) - else: - r.raiseUnexpectedField(fieldName, "AutoShardingConfig") - - if numShardsInCluster.isNone(): - r.raiseUnexpectedValue("Missing required field 'numShardsInCluster'") - - val = AutoShardingConfig(numShardsInCluster: numShardsInCluster.get()) - -# ---------- RlnConfig ---------- - -proc writeValue*(w: var JsonWriter, val: RlnConfig) {.raises: [IOError].} = - w.beginRecord() - w.writeField("contractAddress", val.contractAddress) - w.writeField("chainId", val.chainId) - w.writeField("epochSizeSec", val.epochSizeSec) - w.endRecord() - -proc readValue*( - r: var JsonReader, val: var RlnConfig -) {.raises: [SerializationError, IOError].} = - var - contractAddress: Option[string] - chainId: Option[uint] - epochSizeSec: Option[uint64] - - for fieldName in readObjectFields(r): - case fieldName - of "contractAddress": - contractAddress = some(r.readValue(string)) - of "chainId": - chainId = some(r.readValue(uint)) - of "epochSizeSec": - epochSizeSec = some(r.readValue(uint64)) - else: - r.raiseUnexpectedField(fieldName, "RlnConfig") - - if contractAddress.isNone(): - r.raiseUnexpectedValue("Missing required field 'contractAddress'") - if chainId.isNone(): - r.raiseUnexpectedValue("Missing required field 'chainId'") - if epochSizeSec.isNone(): - r.raiseUnexpectedValue("Missing required field 'epochSizeSec'") - - val = RlnConfig( - contractAddress: contractAddress.get(), - chainId: chainId.get(), - epochSizeSec: epochSizeSec.get(), - ) - -# ---------- NetworkingConfig ---------- - -proc writeValue*(w: var JsonWriter, val: NetworkingConfig) {.raises: [IOError].} = - w.beginRecord() - w.writeField("listenIpv4", val.listenIpv4) - w.writeField("p2pTcpPort", val.p2pTcpPort) - w.writeField("discv5UdpPort", val.discv5UdpPort) - w.endRecord() - -proc readValue*( - r: var JsonReader, val: var NetworkingConfig -) {.raises: [SerializationError, IOError].} = - var - listenIpv4: Option[string] - p2pTcpPort: Option[uint16] - discv5UdpPort: Option[uint16] - - for fieldName in readObjectFields(r): - case fieldName - of "listenIpv4": - listenIpv4 = some(r.readValue(string)) - of "p2pTcpPort": - p2pTcpPort = some(r.readValue(uint16)) - of "discv5UdpPort": - discv5UdpPort = some(r.readValue(uint16)) - else: - r.raiseUnexpectedField(fieldName, "NetworkingConfig") - - if listenIpv4.isNone(): - r.raiseUnexpectedValue("Missing required field 'listenIpv4'") - if p2pTcpPort.isNone(): - r.raiseUnexpectedValue("Missing required field 'p2pTcpPort'") - if discv5UdpPort.isNone(): - r.raiseUnexpectedValue("Missing required field 'discv5UdpPort'") - - val = NetworkingConfig( - listenIpv4: listenIpv4.get(), - p2pTcpPort: p2pTcpPort.get(), - discv5UdpPort: discv5UdpPort.get(), - ) - -# ---------- MessageValidation ---------- - -proc writeValue*(w: var JsonWriter, val: MessageValidation) {.raises: [IOError].} = - w.beginRecord() - w.writeField("maxMessageSize", val.maxMessageSize) - w.writeField("rlnConfig", val.rlnConfig) - w.endRecord() - -proc readValue*( - r: var JsonReader, val: var MessageValidation -) {.raises: [SerializationError, IOError].} = - var - maxMessageSize: Option[string] - rlnConfig: Option[Option[RlnConfig]] - - for fieldName in readObjectFields(r): - case fieldName - of "maxMessageSize": - maxMessageSize = some(r.readValue(string)) - of "rlnConfig": - rlnConfig = some(r.readValue(Option[RlnConfig])) - else: - r.raiseUnexpectedField(fieldName, "MessageValidation") - - if maxMessageSize.isNone(): - r.raiseUnexpectedValue("Missing required field 'maxMessageSize'") - - val = MessageValidation( - maxMessageSize: maxMessageSize.get(), rlnConfig: rlnConfig.get(none(RlnConfig)) - ) - -# ---------- ProtocolsConfig ---------- - -proc writeValue*(w: var JsonWriter, val: ProtocolsConfig) {.raises: [IOError].} = - w.beginRecord() - w.writeField("entryNodes", val.entryNodes) - w.writeField("staticStoreNodes", val.staticStoreNodes) - w.writeField("clusterId", val.clusterId) - w.writeField("autoShardingConfig", val.autoShardingConfig) - w.writeField("messageValidation", val.messageValidation) - w.endRecord() - -proc readValue*( - r: var JsonReader, val: var ProtocolsConfig -) {.raises: [SerializationError, IOError].} = - var - entryNodes: Option[seq[string]] - staticStoreNodes: Option[seq[string]] - clusterId: Option[uint16] - autoShardingConfig: Option[AutoShardingConfig] - messageValidation: Option[MessageValidation] - - for fieldName in readObjectFields(r): - case fieldName - of "entryNodes": - entryNodes = some(r.readValue(seq[string])) - of "staticStoreNodes": - staticStoreNodes = some(r.readValue(seq[string])) - of "clusterId": - clusterId = some(r.readValue(uint16)) - of "autoShardingConfig": - autoShardingConfig = some(r.readValue(AutoShardingConfig)) - of "messageValidation": - messageValidation = some(r.readValue(MessageValidation)) - else: - r.raiseUnexpectedField(fieldName, "ProtocolsConfig") - - if entryNodes.isNone(): - r.raiseUnexpectedValue("Missing required field 'entryNodes'") - if clusterId.isNone(): - r.raiseUnexpectedValue("Missing required field 'clusterId'") - - val = ProtocolsConfig.init( - entryNodes = entryNodes.get(), - staticStoreNodes = staticStoreNodes.get(@[]), - clusterId = clusterId.get(), - autoShardingConfig = autoShardingConfig.get(DefaultAutoShardingConfig), - messageValidation = messageValidation.get(DefaultMessageValidation), - ) - -# ---------- NodeConfig ---------- - -proc writeValue*(w: var JsonWriter, val: NodeConfig) {.raises: [IOError].} = - w.beginRecord() - w.writeField("mode", val.mode) - w.writeField("protocolsConfig", val.protocolsConfig) - w.writeField("networkingConfig", val.networkingConfig) - w.writeField("ethRpcEndpoints", val.ethRpcEndpoints) - w.writeField("p2pReliability", val.p2pReliability) - w.writeField("logLevel", val.logLevel) - w.writeField("logFormat", val.logFormat) - w.endRecord() - -proc readValue*( - r: var JsonReader, val: var NodeConfig -) {.raises: [SerializationError, IOError].} = - var - mode: Option[WakuMode] - protocolsConfig: Option[ProtocolsConfig] - networkingConfig: Option[NetworkingConfig] - ethRpcEndpoints: Option[seq[string]] - p2pReliability: Option[bool] - logLevel: Option[LogLevel] - logFormat: Option[LogFormat] - - for fieldName in readObjectFields(r): - case fieldName - of "mode": - mode = some(r.readValue(WakuMode)) - of "protocolsConfig": - protocolsConfig = some(r.readValue(ProtocolsConfig)) - of "networkingConfig": - networkingConfig = some(r.readValue(NetworkingConfig)) - of "ethRpcEndpoints": - ethRpcEndpoints = some(r.readValue(seq[string])) - of "p2pReliability": - p2pReliability = some(r.readValue(bool)) - of "logLevel": - logLevel = some(r.readValue(LogLevel)) - of "logFormat": - logFormat = some(r.readValue(LogFormat)) - else: - r.raiseUnexpectedField(fieldName, "NodeConfig") - - val = NodeConfig.init( - mode = mode.get(WakuMode.Core), - protocolsConfig = protocolsConfig.get(TheWakuNetworkPreset), - networkingConfig = networkingConfig.get(DefaultNetworkingConfig), - ethRpcEndpoints = ethRpcEndpoints.get(@[]), - p2pReliability = p2pReliability.get(false), - logLevel = logLevel.get(LogLevel.INFO), - logFormat = logFormat.get(LogFormat.TEXT), - ) - -# ---------- Decode helper ---------- -# Json.decode returns T via `result`, which conflicts with {.requiresInit.} -# on Nim 2.x. This helper avoids the issue by using readValue into a var. - -proc decodeNodeConfigFromJson*( - jsonStr: string -): NodeConfig {. - raises: [SerializationError], - deprecated: "Use WakuNodeConf with fieldPairs-based JSON parsing instead" -.} = - var val = NodeConfig.init() # default-initialized - try: - var stream = unsafeMemoryInput(jsonStr) - var reader = (JsonReader[DefaultFlavor].init(stream)) - reader.readValue(val) - except IOError as err: - raise (ref SerializationError)(msg: err.msg) - return val diff --git a/waku/api/events.nim b/waku/api/events.nim new file mode 100644 index 000000000..ba356614b --- /dev/null +++ b/waku/api/events.nim @@ -0,0 +1,7 @@ +{.push raises: [].} + +import ./events/[health, message] + +export health, message + +{.pop.} diff --git a/waku/events/health_events.nim b/waku/api/events/health.nim similarity index 67% rename from waku/events/health_events.nim rename to waku/api/events/health.nim index 95912941e..38df32daf 100644 --- a/waku/events/health_events.nim +++ b/waku/api/events/health.nim @@ -1,6 +1,6 @@ import brokers/event_broker -import waku/api/types +import waku/node/health_monitor/connection_status # ConnectionStatus import waku/node/health_monitor/[protocol_health, topic_health] import waku/waku_core/topics @@ -11,10 +11,8 @@ EventBroker: type EventConnectionStatusChange* = object connectionStatus*: ConnectionStatus -# Notify health changes to a subscribed topic -# TODO: emit content topic health change events when subscribe/unsubscribe -# from/to content topic is provided in the new API (so we know which -# content topics are of interest to the application) +# Notify health changes to a subscribed content topic. A content topic's health +# is its shard's health. EventBroker: type EventContentTopicHealthChange* = object contentTopic*: ContentTopic diff --git a/waku/api/events/message.nim b/waku/api/events/message.nim new file mode 100644 index 000000000..f1eff2ccd --- /dev/null +++ b/waku/api/events/message.nim @@ -0,0 +1,8 @@ +import brokers/event_broker +import waku/[waku_core/message, waku_core/topics] + +EventBroker: + # Emitted when a message arrives from the network via any protocol + type MessageSeenEvent* = object + topic*: PubsubTopic + message*: WakuMessage diff --git a/waku/api/ffi/kernel_ffi.nim b/waku/api/ffi/kernel_ffi.nim new file mode 100644 index 000000000..31d601c8f --- /dev/null +++ b/waku/api/ffi/kernel_ffi.nim @@ -0,0 +1,175 @@ +## Kernel-tier FFI surface for `liblogosdelivery.so`. Exposes raw `Waku` +## lifecycle for fleet/operator callers: `waku_new`, `waku_start`, +## `waku_stop`, `waku_destroy`. C declarations live in +## `liblogosdelivery_kernel.h`. + +import std/[atomics, options] +import chronos, chronicles, results, ffi +import brokers/broker_context +# Imported ahead of the kernel/sequtils-heavy block to keep the messaging +# broker-macro instantiations first (gensym-order workaround). +import layers/logos_delivery +import waku/factory/waku +import waku/node/waku_node +import waku/api/requests/subscription +import waku/waku_core/[topics/content_topic, topics/pubsub_topic] +import waku/api/ffi/kernel_helpers + +template requireInitializedKernel( + ctx: ptr FFIContext[LogosDelivery], opName: string, onError: untyped +) = + if isNil(ctx): + let errMsg {.inject.} = opName & " failed: invalid context" + onError + elif isNil(ctx.myLib) or isNil(ctx.myLib[]): + let errMsg {.inject.} = opName & " failed: node is not initialized" + onError + +registerReqFFI(CreateWakuRequest, ctx: ptr FFIContext[LogosDelivery]): + proc(configJson: cstring): Future[Result[string, string]] {.async.} = + let waku = (await createWakuFromJson(configJson)).valueOr: + chronicles.error "CreateWakuRequest: createWakuFromJson failed", err = error + return err(error) + ctx.myLib[] = LogosDelivery.new(Waku, waku).valueOr: + chronicles.error "CreateWakuRequest: LogosDelivery.new(Waku) failed", err = error + return err(error) + return ok("") + +proc waku_new( + configJson: cstring, callback: FFICallback, userData: pointer +): pointer {.dynlib, exportc, cdecl.} = + initializeLibrary() + + if isNil(callback): + echo "error: missing callback in waku_new" + return nil + + var ctx = ffi.createFFIContext[LogosDelivery]().valueOr: + let msg = "Error in createFFIContext: " & $error + callback(RET_ERR, unsafeAddr msg[0], cast[csize_t](len(msg)), userData) + return nil + + ctx.userData = userData + + ffi.sendRequestToFFIThread( + ctx, CreateWakuRequest.ffiNewReq(callback, userData, configJson) + ).isOkOr: + let msg = "error in sendRequestToFFIThread: " & $error + callback(RET_ERR, unsafeAddr msg[0], cast[csize_t](len(msg)), userData) + ffi.destroyFFIContext(ctx).isOkOr: + chronicles.error "destroyFFIContext failed after sendRequestToFFIThread error", + err = $error + return nil + + return ctx + +proc waku_start( + ctx: ptr FFIContext[LogosDelivery], callback: FFICallBack, userData: pointer +) {.ffi.} = + requireInitializedKernel(ctx, "waku_start"): + return err(errMsg) + (await ctx.myLib[].start()).isOkOr: + chronicles.error "waku_start failed", err = error + return err("failed to start: " & error) + return ok("") + +proc waku_stop( + ctx: ptr FFIContext[LogosDelivery], callback: FFICallBack, userData: pointer +) {.ffi.} = + requireInitializedKernel(ctx, "waku_stop"): + return err(errMsg) + (await ctx.myLib[].stop()).isOkOr: + chronicles.error "waku_stop failed", err = error + return err("failed to stop: " & $error) + return ok("") + +proc waku_relay_subscribe_shard( + ctx: ptr FFIContext[LogosDelivery], + callback: FFICallBack, + userData: pointer, + pubsubTopic: cstring, +) {.ffi.} = + requireInitializedKernel(ctx, "waku_relay_subscribe_shard"): + return err(errMsg) + RequestRelaySubscribeShard.request( + ctx.myLib[].brokerCtx, PubsubTopic($pubsubTopic) + ).isOkOr: + chronicles.error "waku_relay_subscribe_shard failed", err = error + return err(error) + return ok("") + +proc waku_relay_unsubscribe_shard( + ctx: ptr FFIContext[LogosDelivery], + callback: FFICallBack, + userData: pointer, + pubsubTopic: cstring, +) {.ffi.} = + requireInitializedKernel(ctx, "waku_relay_unsubscribe_shard"): + return err(errMsg) + RequestRelayUnsubscribeShard.request( + ctx.myLib[].brokerCtx, PubsubTopic($pubsubTopic) + ).isOkOr: + chronicles.error "waku_relay_unsubscribe_shard failed", err = error + return err(error) + return ok("") + +proc waku_relay_subscribe_content_topic( + ctx: ptr FFIContext[LogosDelivery], + callback: FFICallBack, + userData: pointer, + contentTopic: cstring, + pubsubTopic: cstring, +) {.ffi.} = + ## Subscribe to a content topic. `pubsubTopic` is the optional shard: pass an + ## empty string to derive it via auto-sharding; under static/manual sharding + ## a non-empty shard must be supplied. + requireInitializedKernel(ctx, "waku_relay_subscribe_content_topic"): + return err(errMsg) + let shardOp = + if len(pubsubTopic) == 0: + none[PubsubTopic]() + else: + some(PubsubTopic($pubsubTopic)) + RequestRelaySubscribeContentTopic.request( + ctx.myLib[].brokerCtx, ContentTopic($contentTopic), shardOp + ).isOkOr: + chronicles.error "waku_relay_subscribe_content_topic failed", err = error + return err(error) + return ok("") + +proc waku_relay_unsubscribe_content_topic( + ctx: ptr FFIContext[LogosDelivery], + callback: FFICallBack, + userData: pointer, + contentTopic: cstring, + pubsubTopic: cstring, +) {.ffi.} = + ## Unsubscribe from a content topic. `pubsubTopic` is the optional shard, same + ## convention as `waku_relay_subscribe_content_topic`. + requireInitializedKernel(ctx, "waku_relay_unsubscribe_content_topic"): + return err(errMsg) + let shardOp = + if len(pubsubTopic) == 0: + none[PubsubTopic]() + else: + some(PubsubTopic($pubsubTopic)) + RequestRelayUnsubscribeContentTopic.request( + ctx.myLib[].brokerCtx, ContentTopic($contentTopic), shardOp + ).isOkOr: + chronicles.error "waku_relay_unsubscribe_content_topic failed", err = error + return err(error) + return ok("") + +proc waku_destroy( + ctx: ptr FFIContext[LogosDelivery], callback: FFICallBack, userData: pointer +): cint {.dynlib, exportc, cdecl.} = + initializeLibrary() + checkParams(ctx, callback, userData) + + ffi.destroyFFIContext(ctx).isOkOr: + let msg = "waku_destroy error: " & $error + callback(RET_ERR, unsafeAddr msg[0], cast[csize_t](len(msg)), userData) + return RET_ERR + + callback(RET_OK, nil, 0, userData) + return RET_OK diff --git a/waku/api/ffi/kernel_helpers.nim b/waku/api/ffi/kernel_helpers.nim new file mode 100644 index 000000000..17df1f5f0 --- /dev/null +++ b/waku/api/ffi/kernel_helpers.nim @@ -0,0 +1,72 @@ +{.push raises: [].} + +## FFI helpers for kernel construction. JSON conf parsing + Waku build, +## shared by `kernel_ffi.nim` (waku_new) and `messaging_ffi.nim` +## (messaging_client_new_with_conf). + +import std/[json, strutils, tables] +import chronos, chronicles, results, confutils, confutils/std/net +import waku/factory/waku +import waku/api/api +import tools/confutils/cli_args + +proc createWakuFromJson*( + configJson: cstring +): Future[Result[Waku, string]] {.async.} = + ## Parse a JSON `WakuNodeConf` blob (case-insensitive, unknown-fields-rejected) + ## and construct a `Waku`. Returns the new Waku ref or an error. + + var conf = defaultWakuNodeConf().valueOr: + return err("Failed creating default conf: " & error) + + var jsonNode: JsonNode + try: + jsonNode = parseJson($configJson) + except Exception: + let exceptionMsg = getCurrentExceptionMsg() + error "Failed to parse config JSON", + error = exceptionMsg, configJson = $configJson + return err( + "Failed to parse config JSON: " & exceptionMsg & " configJson string: " & + $configJson + ) + + var jsonFields: Table[string, (string, JsonNode)] + for key, value in jsonNode: + let lowerKey = key.toLowerAscii() + if jsonFields.hasKey(lowerKey): + error "Duplicate configuration option found when normalized to lowercase", + key = key + return err( + "Duplicate configuration option found when normalized to lowercase: '" & key & + "'" + ) + jsonFields[lowerKey] = (key, value) + + for confField, confValue in fieldPairs(conf): + let lowerField = confField.toLowerAscii() + if jsonFields.hasKey(lowerField): + let (jsonKey, jsonValue) = jsonFields[lowerField] + let formattedString = ($jsonValue).strip(chars = {'\"'}) + try: + confValue = parseCmdArg(typeof(confValue), formattedString) + except Exception: + return err( + "Failed to parse field '" & confField & "' from JSON key '" & jsonKey & "': " & + getCurrentExceptionMsg() & ". Value: " & formattedString + ) + jsonFields.del(lowerField) + + if jsonFields.len > 0: + var unknownKeys = newSeq[string]() + for _, (jsonKey, _) in pairs(jsonFields): + unknownKeys.add(jsonKey) + error "Unrecognized configuration option(s) found", option = unknownKeys + return err("Unrecognized configuration option(s) found: " & $unknownKeys) + + let waku = (await api.createNode(conf)).valueOr: + return err($error) + + return ok(waku) + +{.pop.} diff --git a/waku/api/requests.nim b/waku/api/requests.nim new file mode 100644 index 000000000..842872d1c --- /dev/null +++ b/waku/api/requests.nim @@ -0,0 +1,12 @@ +{.push raises: [].} + +import + ./requests/[ + relay, filter, lightpush, store, peers, subscription, protocols, health, node, + rln, + ] + +export + relay, filter, lightpush, store, peers, subscription, protocols, health, node, rln + +{.pop.} diff --git a/waku/api/requests/filter.nim b/waku/api/requests/filter.nim new file mode 100644 index 000000000..1ae8dc451 --- /dev/null +++ b/waku/api/requests/filter.nim @@ -0,0 +1,47 @@ +{.push raises: [].} + +## Waku API: Filter v2 broker request types. + +import std/options +import chronos +import brokers/[broker_context, request_broker] +import waku/waku_core/[topics/content_topic, topics/pubsub_topic, peers] +import waku/waku_filter_v2/common + +export FilterSubscribeErrorKind + +RequestBroker: + type RequestFilterSubscribe* = object + subscribed*: bool + subscribeError*: Option[FilterSubscribeErrorKind] + errorDesc*: string + + proc signature( + servicePeer: RemotePeerInfo, + pubsubTopic: PubsubTopic, + contentTopics: seq[ContentTopic], + ): Future[Result[RequestFilterSubscribe, string]] + +RequestBroker: + type RequestFilterUnsubscribe* = object + unsubscribed*: bool + subscribeError*: Option[FilterSubscribeErrorKind] + errorDesc*: string + + proc signature( + servicePeer: RemotePeerInfo, + pubsubTopic: PubsubTopic, + contentTopics: seq[ContentTopic], + ): Future[Result[RequestFilterUnsubscribe, string]] + +RequestBroker: + type RequestFilterPing* = object + pingOk*: bool + subscribeError*: Option[FilterSubscribeErrorKind] + errorDesc*: string + + proc signature( + servicePeer: RemotePeerInfo, timeout: Duration + ): Future[Result[RequestFilterPing, string]] + +{.pop.} diff --git a/waku/api/requests/health.nim b/waku/api/requests/health.nim new file mode 100644 index 000000000..2671f2774 --- /dev/null +++ b/waku/api/requests/health.nim @@ -0,0 +1,42 @@ +## Waku API: node health and connectivity broker request types. + +import brokers/request_broker +import + waku/node/health_monitor/[ + protocol_health, topic_health, health_report, connection_status + ] +import waku/waku_core/topics +import waku/common/waku_protocol + +export protocol_health, topic_health, connection_status + +# Overall node connectivity status. +RequestBroker(sync): + type RequestConnectionStatus* = object + connectionStatus*: ConnectionStatus + +# Health of a set of content topics. +RequestBroker(sync): + type RequestContentTopicsHealth* = object + contentTopicHealth*: seq[tuple[topic: ContentTopic, health: TopicHealth]] + + proc signature(topics: seq[ContentTopic]): Result[RequestContentTopicsHealth, string] + +# Consolidated node health report. +RequestBroker: + type RequestHealthReport* = object + healthReport*: HealthReport + +# Health of a set of shards (pubsub topics). +RequestBroker(sync): + type RequestShardTopicsHealth* = object + topicHealth*: seq[tuple[topic: PubsubTopic, health: TopicHealth]] + + proc signature(topics: seq[PubsubTopic]): Result[RequestShardTopicsHealth, string] + +# Health of a mounted protocol. +RequestBroker: + type RequestProtocolHealth* = object + healthStatus*: ProtocolHealth + + proc signature(protocol: WakuProtocol): Future[Result[RequestProtocolHealth, string]] diff --git a/waku/api/requests/lightpush.nim b/waku/api/requests/lightpush.nim new file mode 100644 index 000000000..e671b821b --- /dev/null +++ b/waku/api/requests/lightpush.nim @@ -0,0 +1,26 @@ +{.push raises: [].} + +## Waku API: Lightpush broker request types. + +import std/options +import chronos +import brokers/[broker_context, request_broker] +import waku/waku_core/[message/message, topics/pubsub_topic, peers] +import waku/waku_lightpush/rpc # LightPushStatusCode + +export LightPushStatusCode + +# Publish a WakuMessage on a pubsub topic via lightpush to the supplied peer. +RequestBroker: + type RequestLightpushPublish* = object + relayedPeerCount*: uint32 + publishError*: Option[LightPushStatusCode] + errorDesc*: string + + proc signature( + peer: RemotePeerInfo, + pubsubTopic: PubsubTopic, + wakuMessage: WakuMessage, + ): Future[Result[RequestLightpushPublish, string]] + +{.pop.} diff --git a/waku/requests/node_requests.nim b/waku/api/requests/node.nim similarity index 100% rename from waku/requests/node_requests.nim rename to waku/api/requests/node.nim diff --git a/waku/api/requests/peers.nim b/waku/api/requests/peers.nim new file mode 100644 index 000000000..230bb358f --- /dev/null +++ b/waku/api/requests/peers.nim @@ -0,0 +1,35 @@ +{.push raises: [].} + +## Waku API: Peer manager broker request types. + +import std/options +import libp2p/peerid +import brokers/[broker_context, request_broker] +import waku/waku_core/[topics/pubsub_topic, peers] + +# Select a single peer that supports the given protocol codec. +RequestBroker(sync): + type RequestSelectPeer* = object + peer*: Option[RemotePeerInfo] + + proc signature( + proto: string, shard: Option[PubsubTopic] + ): Result[RequestSelectPeer, string] + +# Select all peers that support the given protocol codec. +RequestBroker(sync): + type RequestSelectPeers* = object + peers*: seq[RemotePeerInfo] + + proc signature( + proto: string, shard: Option[PubsubTopic] + ): Result[RequestSelectPeers, string] + +# Check whether the given peerId is currently connected. +RequestBroker(sync): + type RequestIsPeerConnected* = object + connected*: bool + + proc signature(peerId: PeerId): Result[RequestIsPeerConnected, string] + +{.pop.} diff --git a/waku/api/requests/protocols.nim b/waku/api/requests/protocols.nim new file mode 100644 index 000000000..c4eb67e98 --- /dev/null +++ b/waku/api/requests/protocols.nim @@ -0,0 +1,18 @@ +{.push raises: [].} + +## Waku API: kernel node-introspection broker types. +## +## Reports which optional protocol clients are mounted. Pure declarations; the +## provider is wired by WakuNode.startProvidersAndListeners. + +import brokers/[broker_context, request_broker] + +# Which optional protocol clients are currently mounted on the node. +RequestBroker(sync): + type RequestProtocolMountStatus* = object + relayMounted*: bool + lightpushMounted*: bool + filterMounted*: bool + storeMounted*: bool + +{.pop.} diff --git a/waku/api/requests/relay.nim b/waku/api/requests/relay.nim new file mode 100644 index 000000000..a1887d6ec --- /dev/null +++ b/waku/api/requests/relay.nim @@ -0,0 +1,30 @@ +{.push raises: [].} + +## Waku API: Relay broker request types. + +import std/options +import chronos +import brokers/[broker_context, request_broker] +import waku/waku_core/[message/message, topics/pubsub_topic] +import waku/waku_relay/protocol + +export PublishOutcome + +# RequestRelayPublish status fields: +# - publishError: wakuRelay.publish failure mode. +# - rlnProofFailed: RLN proof step refused to attach a proof. +# - validationFailed: validateMessage rejected the message pre-publish. +# - errorDesc: error description. +RequestBroker: + type RequestRelayPublish* = object + relayedPeerCount*: uint32 + publishError*: Option[PublishOutcome] + rlnProofFailed*: bool + validationFailed*: bool + errorDesc*: string + + proc signature( + pubsubTopic: PubsubTopic, wakuMessage: WakuMessage + ): Future[Result[RequestRelayPublish, string]] + +{.pop.} diff --git a/waku/requests/rln_requests.nim b/waku/api/requests/rln.nim similarity index 100% rename from waku/requests/rln_requests.nim rename to waku/api/requests/rln.nim diff --git a/waku/api/requests/store.nim b/waku/api/requests/store.nim new file mode 100644 index 000000000..94110609c --- /dev/null +++ b/waku/api/requests/store.nim @@ -0,0 +1,22 @@ +{.push raises: [].} + +## Waku API: Store broker request types. + +import std/options +import chronos +import brokers/[broker_context, request_broker] +import waku/waku_store/common # StoreQueryRequest/Response, ErrorCode, StoreError, StoreQueryResult + +export StoreQueryRequest, StoreQueryResponse, ErrorCode + +RequestBroker: + type RequestStoreQueryToAny* = object + response*: StoreQueryResponse + queryError*: Option[ErrorCode] + errorDesc*: string + + proc signature( + request: StoreQueryRequest + ): Future[Result[RequestStoreQueryToAny, string]] + +{.pop.} diff --git a/waku/api/requests/subscription.nim b/waku/api/requests/subscription.nim new file mode 100644 index 000000000..74559a806 --- /dev/null +++ b/waku/api/requests/subscription.nim @@ -0,0 +1,110 @@ +{.push raises: [].} + +## Waku API: kernel subscription broker types. +## +## Protocol-explicit subscribe/unsubscribe/is-subscribed surface owned by +## WakuSubscriptionManager. Relay (gossipsub): shard ops + content-topic ops. +## Edge (managed filter): content-topic only. Content-topic ops carry an +## optional shard: derived under auto-sharding, supplied under static sharding. +## Providers installed by startWakuSubscriptionManager. + +import std/options +import brokers/[broker_context, request_broker] +import waku/waku_core/[topics/content_topic, topics/pubsub_topic] + +# ---- Relay (gossipsub) ---- + +RequestBroker(sync): + type RequestRelaySubscribeShard* = object + subscribed*: bool + + proc signature(shard: PubsubTopic): Result[RequestRelaySubscribeShard, string] + +RequestBroker(sync): + type RequestRelayUnsubscribeShard* = object + unsubscribed*: bool + + proc signature(shard: PubsubTopic): Result[RequestRelayUnsubscribeShard, string] + +RequestBroker(sync): + type RequestRelaySubscribeContentTopic* = object + subscribed*: bool + + proc signature( + contentTopic: ContentTopic, shard: Option[PubsubTopic] + ): Result[RequestRelaySubscribeContentTopic, string] + +RequestBroker(sync): + type RequestRelayUnsubscribeContentTopic* = object + unsubscribed*: bool + + proc signature( + contentTopic: ContentTopic, shard: Option[PubsubTopic] + ): Result[RequestRelayUnsubscribeContentTopic, string] + +# ---- Edge (managed filter; content-topic only) ---- + +RequestBroker(sync): + type RequestEdgeSubscribe* = object + subscribed*: bool + + proc signature( + contentTopic: ContentTopic, shard: Option[PubsubTopic] + ): Result[RequestEdgeSubscribe, string] + +RequestBroker(sync): + type RequestEdgeUnsubscribe* = object + unsubscribed*: bool + + proc signature( + contentTopic: ContentTopic, shard: Option[PubsubTopic] + ): Result[RequestEdgeUnsubscribe, string] + +# ---- Read ops ---- + +# Is the content topic subscribed on the relay surface? shard optional: +# derived under auto-sharding, supplied under static/manual sharding. +RequestBroker(sync): + type RequestIsRelaySubscribed* = object + subscribed*: bool + + proc signature( + contentTopic: ContentTopic, shard: Option[PubsubTopic] + ): Result[RequestIsRelaySubscribed, string] + +# Is the content topic subscribed on the edge surface? +RequestBroker(sync): + type RequestIsEdgeSubscribed* = object + subscribed*: bool + + proc signature( + contentTopic: ContentTopic, shard: Option[PubsubTopic] + ): Result[RequestIsEdgeSubscribed, string] + +# Is the content topic subscribed on the node's primary surface? Default +# multiplexing: relay if mounted, else edge. +RequestBroker(sync): + type RequestIsSubscribed* = object + subscribed*: bool + + proc signature( + contentTopic: ContentTopic, shard: Option[PubsubTopic] + ): Result[RequestIsSubscribed, string] + +# Snapshot of every relay-subscribed shard with its content-topic interest set. +RequestBroker(sync): + type RequestRelaySubscribedTopics* = object + topics*: seq[tuple[shard: PubsubTopic, contentTopics: seq[ContentTopic]]] + +# Snapshot of the node's primary-surface subscribed shards with their +# content-topic interest sets. Default multiplexing: relay if mounted, else edge. +RequestBroker(sync): + type RequestSubscribedTopics* = object + topics*: seq[tuple[shard: PubsubTopic, contentTopics: seq[ContentTopic]]] + +# Snapshot of every edge-subscribed shard with its content-topic interest set. +RequestBroker(sync): + type RequestEdgeSubscribedTopics* = object + topics*: seq[tuple[shard: PubsubTopic, contentTopics: seq[ContentTopic]]] + +{.pop.} diff --git a/waku/events/delivery_events.nim b/waku/events/delivery_events.nim deleted file mode 100644 index 5730335e0..000000000 --- a/waku/events/delivery_events.nim +++ /dev/null @@ -1,12 +0,0 @@ -import brokers/event_broker -import waku/waku_core/[message/message, message/digest] - -EventBroker: - type OnFilterSubscribeEvent* = object - pubsubTopic*: string - contentTopics*: seq[string] - -EventBroker: - type OnFilterUnSubscribeEvent* = object - pubsubTopic*: string - contentTopics*: seq[string] diff --git a/waku/events/events.nim b/waku/events/events.nim index 5a3c0c748..80b86a221 100644 --- a/waku/events/events.nim +++ b/waku/events/events.nim @@ -1,3 +1,2 @@ -import ./[message_events, delivery_events, health_events, peer_events, lifecycle_events] - -export message_events, delivery_events, health_events, peer_events, lifecycle_events +import ./peer_events +export peer_events diff --git a/waku/events/message_events.nim b/waku/events/message_events.nim deleted file mode 100644 index b45f91249..000000000 --- a/waku/events/message_events.nim +++ /dev/null @@ -1,34 +0,0 @@ -import brokers/event_broker -import waku/[api/types, waku_core/message, waku_core/topics] -export types - -EventBroker: - # Event emitted when a message is sent to the network - type MessageSentEvent* = object - requestId*: RequestId - messageHash*: string - -EventBroker: - # Event emitted when a message send operation fails - type MessageErrorEvent* = object - requestId*: RequestId - messageHash*: string - error*: string - -EventBroker: - # Confirmation that a message has been correctly delivered to some neighbouring nodes. - type MessagePropagatedEvent* = object - requestId*: RequestId - messageHash*: string - -EventBroker: - # Event emitted when a message is received via Waku - type MessageReceivedEvent* = object - messageHash*: string - message*: WakuMessage - -EventBroker: - # Internal event emitted when a message arrives from the network via any protocol - type MessageSeenEvent* = object - topic*: PubsubTopic - message*: WakuMessage diff --git a/waku/events/peer_events.nim b/waku/events/peer_events.nim index 7eed309b3..2ea1436d0 100644 --- a/waku/events/peer_events.nim +++ b/waku/events/peer_events.nim @@ -1,13 +1,13 @@ import brokers/event_broker import libp2p/switch -type WakuPeerEventKind* {.pure.} = enum +type EventWakuPeerKind* {.pure.} = enum EventConnected EventDisconnected EventIdentified EventMetadataUpdated EventBroker: - type WakuPeerEvent* = object + type EventWakuPeer* = object peerId*: PeerId - kind*: WakuPeerEventKind + kind*: EventWakuPeerKind diff --git a/waku/factory/builder.nim b/waku/factory/builder.nim index 4212cb92d..391dd5cea 100644 --- a/waku/factory/builder.nim +++ b/waku/factory/builder.nim @@ -16,6 +16,7 @@ import ../discovery/waku_discv5, ../waku_node, ../node/peer_manager, + ../node/subscription_manager, ../common/rate_limit/setting, ../common/utils/parse_size_units @@ -226,4 +227,6 @@ proc build*(builder: WakuNodeBuilder): Result[WakuNode, string] = except Exception: return err("failed to build WakuNode instance: " & getCurrentExceptionMsg()) + node.subscriptionManager = WakuSubscriptionManager.new(node) + ok(node) diff --git a/waku/factory/waku.nim b/waku/factory/waku.nim index 6a5567f8c..22bf0e669 100644 --- a/waku/factory/waku.nim +++ b/waku/factory/waku.nim @@ -29,25 +29,23 @@ import waku_relay/protocol, waku_enr/sharding, waku_enr/multiaddr, - api/types, common/logging, node/peer_manager, node/health_monitor, node/waku_metrics, - node/delivery_service/delivery_service, - node/delivery_service/subscription_manager, rest_api/message_cache, rest_api/endpoint/server, rest_api/endpoint/builder as rest_server_builder, discovery/waku_dnsdisc, discovery/waku_discv5, discovery/autonat_service, - requests/health_requests, + api/requests/health, factory/node_factory, factory/internal_config, factory/app_callbacks, persistency/persistency, ], + waku/node/subscription_manager, ./waku_conf, ./waku_state_info @@ -73,8 +71,6 @@ type Waku* = ref object healthMonitor*: NodeHealthMonitor - deliveryService*: DeliveryService - restServer*: WakuRestServerRef metricsServer*: MetricsHttpServerRef appCallbacks*: AppCallbacks @@ -215,10 +211,6 @@ proc new*( error "Failed setting up app callbacks", error = error return err("Failed setting up app callbacks: " & $error) - ## Delivery Monitor - let deliveryService = DeliveryService.new(wakuConf.p2pReliability, node).valueOr: - return err("could not create delivery service: " & $error) - var waku = Waku( stateInfo: WakuStateInfo.init(node), conf: wakuConf, @@ -226,7 +218,6 @@ proc new*( key: wakuConf.nodeKey, node: node, healthMonitor: healthMonitor, - deliveryService: deliveryService, appCallbacks: appCallbacks, restServer: restServer, brokerCtx: brokerCtx, @@ -428,6 +419,14 @@ proc startWaku*(waku: ptr Waku): Future[Result[void, string]] {.async: (raises: waku[].node.ports.discv5Udp = waku[].wakuDiscV5.udpPort.uint16 waku[].conf.discv5Conf.get().udpPort = waku[].wakuDiscV5.udpPort + ## Subscription engine: register the broker subscription surface and run the + ## core-mode auto-subscribe / edge-mode filter loops. Started here (after the + ## node is up and the discv5 subscription listener is registered) so a fresh + ## shard subscription's PubsubSub event reaches discv5 ENR advertisement. + if not waku[].node.subscriptionManager.isNil(): + waku[].node.subscriptionManager.startWakuSubscriptionManager().isOkOr: + return err("failed to start WakuSubscriptionManager: " & error) + ## Update waku data that is set dynamically on node start try: (await updateWaku(waku)).isOkOr: @@ -435,29 +434,10 @@ proc startWaku*(waku: ptr Waku): Future[Result[void, string]] {.async: (raises: except CatchableError: return err("Caught exception in startWaku: " & getCurrentExceptionMsg()) - ## Reliability - if not waku[].deliveryService.isNil(): - waku[].deliveryService.startDeliveryService().isOkOr: - return err("failed to start delivery service: " & $error) - ## Health Monitor waku[].healthMonitor.startHealthMonitor().isOkOr: return err("failed to start health monitor: " & $error) - ## Setup RequestConnectionStatus provider - - RequestConnectionStatus.setProvider( - globalBrokerContext(), - proc(): Result[RequestConnectionStatus, string] = - try: - let healthReport = waku[].healthMonitor.getSyncNodeHealthReport() - return - ok(RequestConnectionStatus(connectionStatus: healthReport.connectionStatus)) - except CatchableError: - err("Failed to read health report: " & getCurrentExceptionMsg()), - ).isOkOr: - error "Failed to set RequestConnectionStatus provider", error = error - ## Setup RequestProtocolHealth provider RequestProtocolHealth.setProvider( @@ -487,6 +467,20 @@ proc startWaku*(waku: ptr Waku): Future[Result[void, string]] {.async: (raises: ).isOkOr: error "Failed to set RequestHealthReport provider", error = error + ## Setup RequestConnectionStatus provider + + RequestConnectionStatus.setProvider( + globalBrokerContext(), + proc(): Result[RequestConnectionStatus, string] = + try: + let healthReport = waku[].healthMonitor.getSyncNodeHealthReport() + return + ok(RequestConnectionStatus(connectionStatus: healthReport.connectionStatus)) + except CatchableError: + err("Failed to read health report: " & getCurrentExceptionMsg()), + ).isOkOr: + error "Failed to set RequestConnectionStatus provider", error = error + if conf.restServerConf.isSome(): rest_server_builder.startRestServerProtocolSupport( waku[].restServer, @@ -538,9 +532,8 @@ proc stop*(waku: Waku): Future[Result[void, string]] {.async: (raises: []).} = if not waku.wakuDiscv5.isNil(): await waku.wakuDiscv5.stop() - if not waku.deliveryService.isNil(): - await waku.deliveryService.stopDeliveryService() - waku.deliveryService = nil + if not waku.node.isNil() and not waku.node.subscriptionManager.isNil(): + await waku.node.subscriptionManager.stopWakuSubscriptionManager() if not waku.node.isNil(): await waku.node.stop() @@ -551,8 +544,10 @@ proc stop*(waku: Waku): Future[Result[void, string]] {.async: (raises: []).} = if not waku.healthMonitor.isNil(): await waku.healthMonitor.stopHealthMonitor() - ## Clear RequestConnectionStatus provider + ## Clear health-tier providers (set in setup above). RequestConnectionStatus.clearProvider(waku.brokerCtx) + RequestHealthReport.clearProvider(waku.brokerCtx) + RequestProtocolHealth.clearProvider(waku.brokerCtx) if not waku.restServer.isNil(): await waku.restServer.stop() diff --git a/waku/node/delivery_service/delivery_service.nim b/waku/node/delivery_service/delivery_service.nim deleted file mode 100644 index f3d78d98e..000000000 --- a/waku/node/delivery_service/delivery_service.nim +++ /dev/null @@ -1,44 +0,0 @@ -## This module helps to ensure the correct transmission and reception of messages - -import results -import chronos, chronicles -import - ./recv_service, - ./send_service, - ./subscription_manager, - waku/[ - waku_core, waku_node, waku_store/client, waku_relay/protocol, waku_lightpush/client - ] - -type DeliveryService* = ref object - sendService*: SendService - recvService*: RecvService - subscriptionManager*: SubscriptionManager - -proc new*( - T: type DeliveryService, useP2PReliability: bool, w: WakuNode -): Result[T, string] = - ## storeClient is needed to give store visitility to DeliveryService - ## wakuRelay and wakuLightpushClient are needed to give a mechanism to SendService to re-publish - let subscriptionManager = SubscriptionManager.new(w) - let sendService = ?SendService.new(useP2PReliability, w, subscriptionManager) - let recvService = RecvService.new(w, subscriptionManager) - - return ok( - DeliveryService( - sendService: sendService, - recvService: recvService, - subscriptionManager: subscriptionManager, - ) - ) - -proc startDeliveryService*(self: DeliveryService): Result[void, string] = - ?self.subscriptionManager.startSubscriptionManager() - self.recvService.startRecvService() - self.sendService.startSendService() - return ok() - -proc stopDeliveryService*(self: DeliveryService) {.async.} = - await self.sendService.stopSendService() - await self.recvService.stopRecvService() - await self.subscriptionManager.stopSubscriptionManager() diff --git a/waku/node/delivery_service/not_delivered_storage/not_delivered_storage.nim b/waku/node/delivery_service/not_delivered_storage/not_delivered_storage.nim deleted file mode 100644 index b0f5f5828..000000000 --- a/waku/node/delivery_service/not_delivered_storage/not_delivered_storage.nim +++ /dev/null @@ -1,38 +0,0 @@ -## This module is aimed to keep track of the sent/published messages that are considered -## not being properly delivered. -## -## The archiving of such messages will happen in a local sqlite database. -## -## In the very first approach, we consider that a message is sent properly is it has been -## received by any store node. -## - -import results -import - ../../../common/databases/db_sqlite, - ../../../waku_core/message/message, - ../../../node/delivery_service/not_delivered_storage/migrations - -const NotDeliveredMessagesDbUrl = "not-delivered-messages.db" - -type NotDeliveredStorage* = ref object - database: SqliteDatabase - -type TrackedWakuMessage = object - msg: WakuMessage - numTrials: uint - ## for statistics purposes. Counts the number of times the node has tried to publish it - -proc new*(T: type NotDeliveredStorage): Result[T, string] = - let db = ?SqliteDatabase.new(NotDeliveredMessagesDbUrl) - - ?migrate(db) - - return ok(NotDeliveredStorage(database: db)) - -proc archiveMessage*( - self: NotDeliveredStorage, msg: WakuMessage -): Result[void, string] = - ## Archives a waku message so that we can keep track of it - ## even when the app restarts - return ok() diff --git a/waku/node/health_monitor/connection_status.nim b/waku/node/health_monitor/connection_status.nim index 68ec9d4be..953c91487 100644 --- a/waku/node/health_monitor/connection_status.nim +++ b/waku/node/health_monitor/connection_status.nim @@ -1,6 +1,11 @@ -import chronos, results, std/strutils, ../../api/types +import chronos, results, std/strutils -export ConnectionStatus +## Overall connectivity state for the node, plus its change-handler type. + +type ConnectionStatus* {.pure.} = enum + Disconnected + PartiallyConnected + Connected const HealthyThreshold* = 2 ## Minimum peers required per service protocol for a "Connected" status (excluding Relay). diff --git a/waku/node/health_monitor/node_health_monitor.nim b/waku/node/health_monitor/node_health_monitor.nim index c652f7cea..cabddf378 100644 --- a/waku/node/health_monitor/node_health_monitor.nim +++ b/waku/node/health_monitor/node_health_monitor.nim @@ -10,8 +10,7 @@ import waku/[ waku_relay, waku_rln_relay, - api/types, - events/health_events, + api/events/health, events/peer_events, node/waku_node, node/peer_manager, @@ -22,6 +21,7 @@ import node/health_monitor/connection_status, node/health_monitor/protocol_health, node/health_monitor/event_loop_monitor, + api/requests/health, requests/health_requests, ] @@ -48,7 +48,7 @@ type NodeHealthMonitor* = ref object ## latest known connectivity strength (e.g. connected peer count) metric for each protocol. ## if it doesn't make sense for the protocol in question, this is set to zero. relayObserver: PubSubObserver - peerEventListener: WakuPeerEventListener + peerEventListener: EventWakuPeerListener shardHealthListener: EventShardTopicHealthChangeListener eventLoopLagExceeded: bool ## set to true when the chronos event loop lag exceeds the severe threshold, @@ -680,9 +680,9 @@ proc startHealthMonitor*(hm: NodeHealthMonitor): Result[void, string] = ) hm.node.wakuRelay.addObserver(hm.relayObserver) - hm.peerEventListener = WakuPeerEvent.listen( + hm.peerEventListener = EventWakuPeer.listen( hm.node.brokerCtx, - proc(evt: WakuPeerEvent): Future[void] {.async: (raises: []), gcsafe.} = + proc(evt: EventWakuPeer): Future[void] {.async: (raises: []), gcsafe.} = ## Recompute health on any peer changing anything (join, leave, identify, metadata update) hm.healthUpdateEvent.fire(), ).valueOr: @@ -725,7 +725,7 @@ proc stopHealthMonitor*(hm: NodeHealthMonitor) {.async.} = if not isNil(hm.eventLoopMonitorFut): await hm.eventLoopMonitorFut.cancelAndWait() - await WakuPeerEvent.dropListener(hm.node.brokerCtx, hm.peerEventListener) + await EventWakuPeer.dropListener(hm.node.brokerCtx, hm.peerEventListener) await EventShardTopicHealthChange.dropListener( hm.node.brokerCtx, hm.shardHealthListener ) diff --git a/waku/node/kernel_api/filter.nim b/waku/node/kernel_api/filter.nim index 948035f14..5bee4fe54 100644 --- a/waku/node/kernel_api/filter.nim +++ b/waku/node/kernel_api/filter.nim @@ -27,7 +27,8 @@ import ../../waku_filter_v2/client as filter_client, ../../waku_filter_v2/subscriptions as filter_subscriptions, ../../common/rate_limit/setting, - ../peer_manager + ../peer_manager, + ../providers/filter as filter_providers logScope: topics = "waku node filter api" @@ -96,6 +97,9 @@ proc mountFilterClient*(node: WakuNode) {.async: (raises: []).} = except LPError: error "failed to mount wakuFilterClient", error = getCurrentExceptionMsg() + filter_providers.registerFilterProviders(node).isOkOr: + error "failed to register filter API providers", error = error + proc filterSubscribe*( node: WakuNode, pubsubTopic: Option[PubsubTopic], diff --git a/waku/node/kernel_api/lightpush.nim b/waku/node/kernel_api/lightpush.nim index ffe2afdac..bedb96262 100644 --- a/waku/node/kernel_api/lightpush.nim +++ b/waku/node/kernel_api/lightpush.nim @@ -29,7 +29,8 @@ import ../../waku_lightpush as lightpush_protocol, ../peer_manager, ../../common/rate_limit/setting, - ../../waku_rln_relay + ../../waku_rln_relay, + ../providers/lightpush as lightpush_providers logScope: topics = "waku node lightpush api" @@ -184,6 +185,8 @@ proc mountLightPushClient*(node: WakuNode) = if node.wakuLightpushClient.isNil(): node.wakuLightpushClient = WakuLightPushClient.new(node.peerManager, node.rng) + lightpush_providers.registerLightpushProviders(node).isOkOr: + error "failed to register lightpush API providers", error = error proc lightpushPublishHandler( node: WakuNode, diff --git a/waku/node/kernel_api/relay.nim b/waku/node/kernel_api/relay.nim index f1b80cf19..9ef1395d9 100644 --- a/waku/node/kernel_api/relay.nim +++ b/waku/node/kernel_api/relay.nim @@ -30,16 +30,16 @@ import waku_rln_relay, node/waku_node, node/peer_manager, - events/message_events, + api/events/message, + waku_lightpush/callbacks, + node/providers/relay as relay_providers, ] export waku_relay.WakuRelayHandler -declarePublicHistogram waku_histogram_message_size, - "message size histogram in kB", - buckets = [ - 0.0, 1.0, 3.0, 5.0, 15.0, 50.0, 75.0, 100.0, 125.0, 150.0, 500.0, 700.0, 1000.0, Inf - ] +# NOTE: `waku_node_messages` + `waku_histogram_message_size` are declared in +# `waku/node/waku_telemetry` (re-exported via `node/waku_node`) so both this +# handler and WakuSubscriptionManager observe the same Prometheus collectors. logScope: topics = "waku node relay api" @@ -268,6 +268,9 @@ proc mountRelay*( node.switch.mount(node.wakuRelay, protocolMatcher(WakuRelayCodec)) + relay_providers.registerRelayProviders(node).isOkOr: + error "failed to register relay API providers", error = error + info "relay mounted successfully" return ok() diff --git a/waku/node/kernel_api/store.nim b/waku/node/kernel_api/store.nim index fcf0dfc89..87fb4f779 100644 --- a/waku/node/kernel_api/store.nim +++ b/waku/node/kernel_api/store.nim @@ -26,7 +26,8 @@ import ../../waku_store/resume, ../peer_manager, ../../common/rate_limit/setting, - ../../waku_archive + ../../waku_archive, + ../providers/store as store_providers logScope: topics = "waku node store api" @@ -120,6 +121,9 @@ proc mountStoreClient*(node: WakuNode) = node.wakuStoreClient = store_client.WakuStoreClient.new(node.peerManager, node.rng) + store_providers.registerStoreProviders(node).isOkOr: + error "failed to register store API providers", error = error + proc query*( node: WakuNode, request: store_common.StoreQueryRequest, peer: RemotePeerInfo ): Future[store_common.WakuStoreResult[store_common.StoreQueryResponse]] {. diff --git a/waku/node/peer_manager/peer_manager.nim b/waku/node/peer_manager/peer_manager.nim index 6602c049b..2e8e16eee 100644 --- a/waku/node/peer_manager/peer_manager.nim +++ b/waku/node/peer_manager/peer_manager.nim @@ -774,7 +774,7 @@ proc refreshPeerMetadata(pm: PeerManager, peerId: PeerId) {.async.} = # TODO: should only trigger an event if metadata actually changed # should include the shard subscription delta in the event when # it is a MetadataUpdated event - WakuPeerEvent.emit(pm.brokerCtx, peerId, WakuPeerEventKind.EventMetadataUpdated) + EventWakuPeer.emit(pm.brokerCtx, peerId, EventWakuPeerKind.EventMetadataUpdated) return info "disconnecting from peer", peerId = peerId, reason = reason @@ -819,7 +819,7 @@ proc onPeerEvent(pm: PeerManager, peerId: PeerId, event: PeerEvent) {.async.} = asyncSpawn(pm.evictPeer(peerId)) peerStore.delete(peerId) - WakuPeerEvent.emit(pm.brokerCtx, peerId, WakuPeerEventKind.EventConnected) + EventWakuPeer.emit(pm.brokerCtx, peerId, EventWakuPeerKind.EventConnected) if not pm.onConnectionChange.isNil(): # we don't want to await for the callback to finish @@ -836,7 +836,7 @@ proc onPeerEvent(pm: PeerManager, peerId: PeerId, event: PeerEvent) {.async.} = pm.ipTable.del(ip) break - WakuPeerEvent.emit(pm.brokerCtx, peerId, WakuPeerEventKind.EventDisconnected) + EventWakuPeer.emit(pm.brokerCtx, peerId, EventWakuPeerKind.EventDisconnected) if not pm.onConnectionChange.isNil(): # we don't want to await for the callback to finish @@ -844,7 +844,7 @@ proc onPeerEvent(pm: PeerManager, peerId: PeerId, event: PeerEvent) {.async.} = of PeerEventKind.Identified: info "event identified", peerId = peerId - WakuPeerEvent.emit(pm.brokerCtx, peerId, WakuPeerEventKind.EventIdentified) + EventWakuPeer.emit(pm.brokerCtx, peerId, EventWakuPeerKind.EventIdentified) peerStore[ConnectionBook][peerId] = connectedness peerStore[DirectionBook][peerId] = direction diff --git a/waku/node/providers/filter.nim b/waku/node/providers/filter.nim new file mode 100644 index 000000000..b894d0d68 --- /dev/null +++ b/waku/node/providers/filter.nim @@ -0,0 +1,124 @@ +{.push raises: [].} + +## Filter broker provider wiring. Binds the kernel filter broker requests to +## the live filter client. + +import std/options +import chronos, results +import ../waku_node +import ../../waku_core +import ../../waku_filter_v2 +import ../../waku_filter_v2/client as filter_client +import ../../api/requests/filter + +# Collapse FilterSubscribeError (case object) to a description string. +proc filterErrDesc(e: FilterSubscribeError): string = + result = $e.kind + case e.kind + of FilterSubscribeErrorKind.PEER_DIAL_FAILURE: + result.add(": " & e.address) + of FilterSubscribeErrorKind.BAD_RESPONSE, + FilterSubscribeErrorKind.BAD_REQUEST, + FilterSubscribeErrorKind.NOT_FOUND, + FilterSubscribeErrorKind.TOO_MANY_REQUESTS, + FilterSubscribeErrorKind.SERVICE_UNAVAILABLE: + result.add(": " & e.cause) + else: + discard + +proc registerFilterProviders*(node: WakuNode): Result[void, string] = + ## Bind the filter broker providers to the live filter client. + RequestFilterSubscribe.setProvider( + node.brokerCtx, + proc( + servicePeer: RemotePeerInfo, + pubsubTopic: PubsubTopic, + contentTopics: seq[ContentTopic], + ): Future[Result[RequestFilterSubscribe, string]] {.async.} = + var res: FilterSubscribeResult + try: + res = + await node.wakuFilterClient.subscribe(servicePeer, pubsubTopic, contentTopics) + except CatchableError as e: + res = FilterSubscribeResult.err(FilterSubscribeError.badResponse(e.msg)) + if res.isOk(): + return ok( + RequestFilterSubscribe( + subscribed: true, + subscribeError: none(FilterSubscribeErrorKind), + errorDesc: "", + ) + ) + return ok( + RequestFilterSubscribe( + subscribed: false, + subscribeError: some(res.error.kind), + errorDesc: filterErrDesc(res.error), + ) + ), + ).isOkOr: + return err("registerFilterProviders: RequestFilterSubscribe: " & error) + + RequestFilterUnsubscribe.setProvider( + node.brokerCtx, + proc( + servicePeer: RemotePeerInfo, + pubsubTopic: PubsubTopic, + contentTopics: seq[ContentTopic], + ): Future[Result[RequestFilterUnsubscribe, string]] {.async.} = + var res: FilterSubscribeResult + try: + res = await node.wakuFilterClient.unsubscribe( + servicePeer, pubsubTopic, contentTopics + ) + except CatchableError as e: + res = FilterSubscribeResult.err(FilterSubscribeError.badResponse(e.msg)) + if res.isOk(): + return ok( + RequestFilterUnsubscribe( + unsubscribed: true, + subscribeError: none(FilterSubscribeErrorKind), + errorDesc: "", + ) + ) + return ok( + RequestFilterUnsubscribe( + unsubscribed: false, + subscribeError: some(res.error.kind), + errorDesc: filterErrDesc(res.error), + ) + ), + ).isOkOr: + return err("registerFilterProviders: RequestFilterUnsubscribe: " & error) + + RequestFilterPing.setProvider( + node.brokerCtx, + proc( + servicePeer: RemotePeerInfo, timeout: Duration + ): Future[Result[RequestFilterPing, string]] {.async.} = + var res: FilterSubscribeResult + try: + res = await node.wakuFilterClient.ping(servicePeer, timeout) + except CatchableError as e: + res = FilterSubscribeResult.err(FilterSubscribeError.badResponse(e.msg)) + if res.isOk(): + return ok( + RequestFilterPing( + pingOk: true, + subscribeError: none(FilterSubscribeErrorKind), + errorDesc: "", + ) + ) + return ok( + RequestFilterPing( + pingOk: false, + subscribeError: some(res.error.kind), + errorDesc: filterErrDesc(res.error), + ) + ), + ).isOkOr: + return err("registerFilterProviders: RequestFilterPing: " & error) + + ok() + +{.pop.} diff --git a/waku/node/providers/lightpush.nim b/waku/node/providers/lightpush.nim new file mode 100644 index 000000000..16a7137d1 --- /dev/null +++ b/waku/node/providers/lightpush.nim @@ -0,0 +1,54 @@ +{.push raises: [].} + +## Lightpush broker provider wiring. Binds the kernel lightpush broker request +## to the live lightpush client. + +import std/options +import chronos, results +import ../waku_node +import ../../waku_core +import ../../waku_lightpush/client as lightpush_client +import ../../waku_lightpush as lightpush_protocol +import ../../api/requests/lightpush + +proc registerLightpushProviders*(node: WakuNode): Result[void, string] = + ## Bind the lightpush broker provider to the live lightpush client. + RequestLightpushPublish.setProvider( + node.brokerCtx, + proc( + peer: RemotePeerInfo, + pubsubTopic: PubsubTopic, + wakuMessage: WakuMessage, + ): Future[Result[RequestLightpushPublish, string]] {.async.} = + try: + let res = + await node.wakuLightpushClient.publish(some(pubsubTopic), wakuMessage, peer) + if res.isOk(): + return ok( + RequestLightpushPublish( + relayedPeerCount: res.value, + publishError: none(LightPushStatusCode), + errorDesc: "", + ) + ) + return ok( + RequestLightpushPublish( + relayedPeerCount: 0'u32, + publishError: some(res.error.code), + errorDesc: res.error.desc.get(""), + ) + ) + except CatchableError as e: + return ok( + RequestLightpushPublish( + relayedPeerCount: 0'u32, + publishError: some(LightPushErrorCode.INTERNAL_SERVER_ERROR), + errorDesc: "lightpush.publish raised: " & e.msg, + ) + ), + ).isOkOr: + return err("registerLightpushProviders: RequestLightpushPublish: " & error) + + ok() + +{.pop.} diff --git a/waku/node/providers/relay.nim b/waku/node/providers/relay.nim new file mode 100644 index 000000000..cba4abfaa --- /dev/null +++ b/waku/node/providers/relay.nim @@ -0,0 +1,76 @@ +{.push raises: [].} + +## Relay broker provider wiring. Binds the kernel relay publish broker to the +## live relay protocol (RLN proof + validation + publish). + +import std/options +import chronos, results +import ../waku_node +import ../../waku_core +import ../../waku_relay +import ../../waku_rln_relay +import ../../waku_lightpush/callbacks +import ../../api/requests/relay as relay_api + +proc registerRelayProviders*(node: WakuNode): Result[void, string] = + ## Bind the relay publish broker provider to the live relay protocol. + relay_api.RequestRelayPublish.setProvider( + node.brokerCtx, + proc( + pubsubTopic: PubsubTopic, wakuMessage: WakuMessage + ): Future[Result[relay_api.RequestRelayPublish, string]] {.async.} = + # Publish-path exceptions propagate; the broker turns them into a request + # error, which the caller maps to a permanent failure. + let rlnPeer = + if node.wakuRlnRelay.isNil(): + none(WakuRLNRelay) + else: + some(node.wakuRlnRelay) + let msgWithProof = checkAndGenerateRLNProof(rlnPeer, wakuMessage).valueOr: + return ok( + relay_api.RequestRelayPublish( + relayedPeerCount: 0'u32, + publishError: none(PublishOutcome), + rlnProofFailed: true, + validationFailed: false, + errorDesc: error, + ) + ) + + (await node.wakuRelay.validateMessage(pubsubTopic, msgWithProof)).isOkOr: + return ok( + relay_api.RequestRelayPublish( + relayedPeerCount: 0'u32, + publishError: none(PublishOutcome), + rlnProofFailed: false, + validationFailed: true, + errorDesc: error, + ) + ) + + let res = await node.wakuRelay.publish(pubsubTopic, msgWithProof) + if res.isOk(): + return ok( + relay_api.RequestRelayPublish( + relayedPeerCount: res.value.uint32, + publishError: none(PublishOutcome), + rlnProofFailed: false, + validationFailed: false, + errorDesc: "", + ) + ) + return ok( + relay_api.RequestRelayPublish( + relayedPeerCount: 0'u32, + publishError: some(res.error), + rlnProofFailed: false, + validationFailed: false, + errorDesc: $res.error, + ) + ), + ).isOkOr: + return err("registerRelayProviders: RequestRelayPublish: " & error) + + ok() + +{.pop.} diff --git a/waku/node/providers/store.nim b/waku/node/providers/store.nim new file mode 100644 index 000000000..27292ef28 --- /dev/null +++ b/waku/node/providers/store.nim @@ -0,0 +1,56 @@ +{.push raises: [].} + +## Store broker provider wiring. Binds the kernel store broker request to the +## live store client. + +import std/options +import chronos, results +import ../waku_node +import ../../waku_store/client as store_client +import ../../waku_store/common as store_common +import ../../api/requests/store as store_api + +proc registerStoreProviders*(node: WakuNode): Result[void, string] = + ## Bind the store broker provider to the live store client. + store_api.RequestStoreQueryToAny.setProvider( + node.brokerCtx, + proc( + request: store_common.StoreQueryRequest + ): Future[Result[store_api.RequestStoreQueryToAny, string]] {.async.} = + var res: store_common.StoreQueryResult + try: + res = await node.wakuStoreClient.queryToAny(request) + except CatchableError: + res = store_common.StoreQueryResult.err( + store_common.StoreError(kind: store_common.ErrorCode.UNKNOWN) + ) + if res.isOk(): + return ok( + store_api.RequestStoreQueryToAny( + response: res.value, + queryError: none(store_common.ErrorCode), + errorDesc: "", + ) + ) + let storeErr = res.error + var desc = $storeErr.kind + case storeErr.kind + of store_common.ErrorCode.PEER_DIAL_FAILURE: + desc.add(": " & storeErr.address) + of store_common.ErrorCode.BAD_RESPONSE, store_common.ErrorCode.BAD_REQUEST: + desc.add(": " & storeErr.cause) + else: + discard + return ok( + store_api.RequestStoreQueryToAny( + response: default(store_common.StoreQueryResponse), + queryError: some(storeErr.kind), + errorDesc: desc, + ) + ), + ).isOkOr: + return err("registerStoreProviders: RequestStoreQueryToAny: " & error) + + ok() + +{.pop.} diff --git a/waku/node/subscription_manager.nim b/waku/node/subscription_manager.nim new file mode 100644 index 000000000..9034f0c45 --- /dev/null +++ b/waku/node/subscription_manager.nim @@ -0,0 +1,903 @@ +## Subscription engine: content-topic interest tracking, relay-mode pubsub +## subscription bookkeeping, edge-mode filter peer subscription maintenance. +## Type bodies live in ./waku_node.nim. + +import std/[sequtils, sets, tables, options], chronos, chronicles, results, metrics +import libp2p/[peerid, peerinfo] +import brokers/broker_context + +import + waku/[ + waku_core, + waku_core/topics, + waku_core/topics/sharding, + waku_node, + waku_relay, + waku_filter_v2/common as filter_common, + waku_filter_v2/protocol as filter_protocol, + waku_archive, + waku_store_sync, + api/events/health, + events/peer_events, + api/events/message, + api/requests/health, + requests/health_requests, + node/peer_manager, + node/health_monitor/topic_health, + node/health_monitor/connection_status, + ] +import waku/api/requests/filter as kernel_filter_api +import waku/api/requests/subscription + +func toTopicHealth*(peersCount: int): TopicHealth = + if peersCount >= HealthyThreshold: + TopicHealth.SUFFICIENTLY_HEALTHY + elif peersCount > 0: + TopicHealth.MINIMALLY_HEALTHY + else: + TopicHealth.UNHEALTHY + +proc isRelayMounted(self: WakuSubscriptionManager): bool = + not self.node.wakuRelay.isNil() + +proc isFilterMounted(self: WakuSubscriptionManager): bool = + not self.node.wakuFilterClient.isNil() + +iterator relaySubscribedTopics*( + self: WakuSubscriptionManager +): (PubsubTopic, HashSet[ContentTopic]) = + ## Iterate relay-subscribed content topics, batched per shard. Skips shards with no interest. + for pubsub, topics in self.relayContentTopicSubs.pairs: + if topics.len == 0: + continue + yield (pubsub, topics) + +iterator edgeSubscribedTopics*( + self: WakuSubscriptionManager +): (PubsubTopic, HashSet[ContentTopic]) = + ## Iterate edge-subscribed content topics, batched per shard. Skips shards with no interest. + for pubsub, topics in self.edgeContentTopicSubs.pairs: + if topics.len == 0: + continue + yield (pubsub, topics) + +proc edgeFilterPeerCount*(sm: WakuSubscriptionManager, shard: PubsubTopic): int = + sm.edgeFilterSubStates.withValue(shard, state): + return state.peers.len + return 0 + +proc new*(T: typedesc[WakuSubscriptionManager], node: WakuNode): T = + WakuSubscriptionManager( + node: node, + relayContentTopicSubs: initTable[PubsubTopic, HashSet[ContentTopic]](), + edgeContentTopicSubs: initTable[PubsubTopic, HashSet[ContentTopic]](), + directShardSubs: initHashSet[PubsubTopic](), + ) + +# Relay mesh subscription bookkeeping + +proc registerRelayHandler( + self: WakuSubscriptionManager, + shard: PubsubTopic, + appHandler: WakuRelayHandler = nil, +): bool = + ## Subscribe the relay mesh to shard with the single fan-out handler. Returns + ## true iff a fresh mesh subscription was created; false if already subscribed + ## (only the optional appHandler is re-recorded). The fan-out handler forwards + ## each message to filter, archive and store-sync, emits MessageSeenEvent, then + ## invokes the optional kernel-API app handler. + let node = self.node + let alreadySubscribed = node.wakuRelay.isSubscribed(shard) + + if not appHandler.isNil(): + if not alreadySubscribed or not node.legacyAppHandlers.hasKey(shard): + node.legacyAppHandlers[shard] = appHandler + else: + debug "Legacy appHandler already exists for active shard, ignoring new handler", + shard = shard + + if alreadySubscribed: + return false + + proc traceHandler(topic: PubsubTopic, msg: WakuMessage) {.async, gcsafe.} = + let msgSizeKB = msg.payload.len / 1000 + waku_node_messages.inc(labelValues = ["relay"]) + waku_histogram_message_size.observe(msgSizeKB) + + proc filterHandler(topic: PubsubTopic, msg: WakuMessage) {.async, gcsafe.} = + if node.wakuFilter.isNil(): + return + await node.wakuFilter.handleMessage(topic, msg) + + proc archiveHandler(topic: PubsubTopic, msg: WakuMessage) {.async, gcsafe.} = + if node.wakuArchive.isNil(): + return + await node.wakuArchive.handleMessage(topic, msg) + + proc syncHandler(topic: PubsubTopic, msg: WakuMessage) {.async, gcsafe.} = + if node.wakuStoreReconciliation.isNil(): + return + node.wakuStoreReconciliation.messageIngress(topic, msg) + + proc internalHandler(topic: PubsubTopic, msg: WakuMessage) {.async, gcsafe.} = + MessageSeenEvent.emit(node.brokerCtx, topic, msg) + + let uniqueTopicHandler = proc( + topic: PubsubTopic, msg: WakuMessage + ): Future[void] {.async, gcsafe.} = + await traceHandler(topic, msg) + await filterHandler(topic, msg) + await archiveHandler(topic, msg) + await syncHandler(topic, msg) + await internalHandler(topic, msg) + + # Invoke the kernel-API app handler if one is registered. + if node.legacyAppHandlers.hasKey(topic) and not node.legacyAppHandlers[topic].isNil(): + await node.legacyAppHandlers[topic](topic, msg) + + node.wakuRelay.subscribe(shard, uniqueTopicHandler) + return true + +proc meshSubscribe( + self: WakuSubscriptionManager, shard: PubsubTopic, handler: WakuRelayHandler = nil +) = + ## Idempotent relay-mesh subscribe. Emits PubsubSub only on a fresh mesh sub. + if self.registerRelayHandler(shard, handler): + self.node.topicSubscriptionQueue.emit((kind: SubscriptionKind.PubsubSub, topic: shard)) + +proc meshUnsubscribe(self: WakuSubscriptionManager, shard: PubsubTopic) = + ## Tear down the relay-mesh subscription for shard and drop its app handler. + ## Emits PubsubUnsub only if the mesh was actually subscribed. + if self.node.legacyAppHandlers.hasKey(shard): + self.node.legacyAppHandlers.del(shard) + if self.node.wakuRelay.isSubscribed(shard): + self.node.wakuRelay.unsubscribe(shard) + self.node.topicSubscriptionQueue.emit( + (kind: SubscriptionKind.PubsubUnsub, topic: shard) + ) + +proc held(self: WakuSubscriptionManager, shard: PubsubTopic): bool = + ## A shard's relay-mesh subscription is held while it has a direct shard + ## subscription or any relay content-topic interest. Edge interest does not + ## hold the mesh. + self.directShardSubs.contains(shard) or + self.relayContentTopicSubs.getOrDefault(shard).len > 0 + +proc resolveShard( + self: WakuSubscriptionManager, + topic: ContentTopic, + shardOp: Option[PubsubTopic], +): Result[PubsubTopic, string] = + ## Derive the shard for a content topic: use shardOp when given (required + ## under static/manual sharding), otherwise auto-shard. + let shardObj = ?deduceRelayShard(self.node, topic, shardOp) + return ok(PubsubTopic($shardObj)) + +# Relay content-topic interest + +proc addRelayContentTopicInterest( + self: WakuSubscriptionManager, shard: PubsubTopic, topic: ContentTopic +) = + if not self.relayContentTopicSubs.hasKey(shard): + self.relayContentTopicSubs[shard] = initHashSet[ContentTopic]() + self.relayContentTopicSubs.withValue(shard, cTopics): + cTopics[].incl(topic) + +proc removeRelayContentTopicInterest( + self: WakuSubscriptionManager, shard: PubsubTopic, topic: ContentTopic +) = + self.relayContentTopicSubs.withValue(shard, cTopics): + cTopics[].excl(topic) + if cTopics[].len == 0: + self.relayContentTopicSubs.del(shard) + +# Edge content-topic interest (drives the filter maintenance loop) + +proc addEdgeContentTopicInterest( + self: WakuSubscriptionManager, shard: PubsubTopic, topic: ContentTopic +) = + var changed = false + if not self.edgeContentTopicSubs.hasKey(shard): + self.edgeContentTopicSubs[shard] = initHashSet[ContentTopic]() + changed = true + self.edgeContentTopicSubs.withValue(shard, cTopics): + if not cTopics[].contains(topic): + cTopics[].incl(topic) + changed = true + if changed and not isNil(self.edgeFilterWakeup): + self.edgeFilterWakeup.fire() + +proc removeEdgeContentTopicInterest( + self: WakuSubscriptionManager, shard: PubsubTopic, topic: ContentTopic +) = + var changed = false + self.edgeContentTopicSubs.withValue(shard, cTopics): + if cTopics[].contains(topic): + cTopics[].excl(topic) + changed = true + if cTopics[].len == 0: + self.edgeContentTopicSubs.del(shard) + if changed and not isNil(self.edgeFilterWakeup): + self.edgeFilterWakeup.fire() + +proc isRelaySubscribed*( + self: WakuSubscriptionManager, shard: PubsubTopic, contentTopic: ContentTopic +): bool {.raises: [].} = + self.relayContentTopicSubs.withValue(shard, cTopics): + return cTopics[].contains(contentTopic) + return false + +proc isEdgeSubscribed*( + self: WakuSubscriptionManager, shard: PubsubTopic, contentTopic: ContentTopic +): bool {.raises: [].} = + self.edgeContentTopicSubs.withValue(shard, cTopics): + return cTopics[].contains(contentTopic) + return false + +# The four-operation subscription surface. +# subscribeShard/unsubscribeShard: direct (0/1) shard interest. +# subscribeContentTopic/unsubscribeContentTopic: per-content-topic interest. +# Content-topic ops take an optional shard: derived under auto-sharding, +# supplied under static/manual sharding. A shard's relay-mesh subscription is +# held while a direct shard subscription or any content-topic interest keeps it; +# the pubsub topic is torn down when nothing holds it. + +proc subscribeShard*( + self: WakuSubscriptionManager, + shard: PubsubTopic, + handler: WakuRelayHandler = nil, +): Result[void, string] = + if not self.isRelayMounted() and not self.isFilterMounted(): + return err("WakuSubscriptionManager requires either Relay or Filter Client.") + + self.directShardSubs.incl(shard) + if self.isRelayMounted(): + self.meshSubscribe(shard, handler) + + return ok() + +proc unsubscribeShard*( + self: WakuSubscriptionManager, shard: PubsubTopic +): Result[void, string] = + if not self.isRelayMounted() and not self.isFilterMounted(): + return err("WakuSubscriptionManager requires either Relay or Filter Client.") + + # Remove the direct interest only; the pubsub topic stays up if content-topic interest holds it. + self.directShardSubs.excl(shard) + if self.isRelayMounted() and not self.held(shard): + self.meshUnsubscribe(shard) + + return ok() + +# Relay content-topic subscription (gossipsub mesh) + +proc relaySubscribeContentTopic*( + self: WakuSubscriptionManager, + topic: ContentTopic, + shardOp: Option[PubsubTopic] = none[PubsubTopic](), +): Result[void, string] = + if not self.isRelayMounted(): + return err("relaySubscribeContentTopic requires Relay mounted.") + + let shard = ?self.resolveShard(topic, shardOp) + self.meshSubscribe(shard, nil) + self.addRelayContentTopicInterest(shard, topic) + return ok() + +proc relayUnsubscribeContentTopic*( + self: WakuSubscriptionManager, + topic: ContentTopic, + shardOp: Option[PubsubTopic] = none[PubsubTopic](), +): Result[void, string] = + if not self.isRelayMounted(): + return err("relayUnsubscribeContentTopic requires Relay mounted.") + + let shard = ?self.resolveShard(topic, shardOp) + self.removeRelayContentTopicInterest(shard, topic) + + # Tear the mesh down only when nothing holds it. + if not self.held(shard): + self.meshUnsubscribe(shard) + + return ok() + +# Edge content-topic subscription (filter; driver reconciles peers) + +proc edgeSubscribe*( + self: WakuSubscriptionManager, + topic: ContentTopic, + shardOp: Option[PubsubTopic] = none[PubsubTopic](), +): Result[void, string] = + if not self.isFilterMounted(): + return err("edgeSubscribe requires a Filter Client mounted.") + + let shard = ?self.resolveShard(topic, shardOp) + self.addEdgeContentTopicInterest(shard, topic) + return ok() + +proc edgeUnsubscribe*( + self: WakuSubscriptionManager, + topic: ContentTopic, + shardOp: Option[PubsubTopic] = none[PubsubTopic](), +): Result[void, string] = + if not self.isFilterMounted(): + return err("edgeUnsubscribe requires a Filter Client mounted.") + + let shard = ?self.resolveShard(topic, shardOp) + self.removeEdgeContentTopicInterest(shard, topic) + return ok() + +# Edge Filter driver + +const EdgeFilterSubscribeTimeout = chronos.seconds(15) + ## Timeout for a single filter subscribe/unsubscribe RPC to a service peer. +const EdgeFilterPingTimeout = chronos.seconds(5) + ## Timeout for a filter ping. +const EdgeFilterLoopInterval = chronos.seconds(30) + ## Interval for the edge filter maintenance loop. +const EdgeFilterSubLoopDebounce = chronos.seconds(1) + ## Debounce delay to coalesce wakeups into a single reconciliation pass. + +type EdgeDialTask = object + peer: RemotePeerInfo + shard: PubsubTopic + topics: seq[ContentTopic] + +proc updateShardHealth( + self: WakuSubscriptionManager, shard: PubsubTopic, state: var EdgeFilterSubState +) = + ## Recompute and emit health for a shard after its peer set changed. + let newHealth = toTopicHealth(state.peers.len) + if newHealth != state.currentHealth: + state.currentHealth = newHealth + EventShardTopicHealthChange.emit(self.node.brokerCtx, shard, newHealth) + +proc removePeer(self: WakuSubscriptionManager, shard: PubsubTopic, peerId: PeerId) = + ## Remove a peer from edgeFilterSubStates for the shard, update health, and + ## wake the sub loop to dial a replacement. Best-effort unsubscribe. + self.edgeFilterSubStates.withValue(shard, state): + var peer: RemotePeerInfo + var found = false + for p in state.peers: + if p.peerId == peerId: + peer = p + found = true + break + if not found: + return + + state.peers.keepItIf(it.peerId != peerId) + self.updateShardHealth(shard, state[]) + self.edgeFilterWakeup.fire() + + if self.isFilterMounted(): + self.edgeContentTopicSubs.withValue(shard, topics): + let ct = toSeq(topics[]) + if ct.len > 0: + let brokerCtx = self.node.brokerCtx + proc doUnsubscribe() {.async.} = + discard await kernel_filter_api.RequestFilterUnsubscribe.request( + brokerCtx, peer, shard, ct + ) + + asyncSpawn doUnsubscribe() + +type SendChunkedFilterRpcKind = enum + FilterSubscribe + FilterUnsubscribe + +proc sendChunkedFilterRpc( + self: WakuSubscriptionManager, + peer: RemotePeerInfo, + shard: PubsubTopic, + topics: seq[ContentTopic], + kind: SendChunkedFilterRpcKind, +): Future[bool] {.async.} = + ## Send a chunked filter subscribe or unsubscribe RPC. Returns true on + ## success. On failure the peer is removed and false returned. + try: + var i = 0 + while i < topics.len: + let chunk = + topics[i ..< min(i + filter_protocol.MaxContentTopicsPerRequest, topics.len)] + var failed = false + case kind + of FilterSubscribe: + let fut = kernel_filter_api.RequestFilterSubscribe.request( + self.node.brokerCtx, peer, shard, chunk + ) + if not (await fut.withTimeout(EdgeFilterSubscribeTimeout)) or fut.read().isErr(): + failed = true + of FilterUnsubscribe: + let fut = kernel_filter_api.RequestFilterUnsubscribe.request( + self.node.brokerCtx, peer, shard, chunk + ) + if not (await fut.withTimeout(EdgeFilterSubscribeTimeout)) or fut.read().isErr(): + failed = true + if failed: + trace "sendChunkedFilterRpc: chunk failed", + op = kind, shard = shard, peer = peer.peerId + self.removePeer(shard, peer.peerId) + return false + i += filter_protocol.MaxContentTopicsPerRequest + except CatchableError as exc: + debug "sendChunkedFilterRpc: failed", + op = kind, shard = shard, peer = peer.peerId, err = exc.msg + self.removePeer(shard, peer.peerId) + return false + return true + +proc syncFilterDeltas( + self: WakuSubscriptionManager, + peer: RemotePeerInfo, + shard: PubsubTopic, + added: seq[ContentTopic], + removed: seq[ContentTopic], +) {.async.} = + ## Push content topic changes (adds/removes) to an already-tracked peer. + if added.len > 0: + if not await self.sendChunkedFilterRpc(peer, shard, added, FilterSubscribe): + return + + if removed.len > 0: + discard await self.sendChunkedFilterRpc(peer, shard, removed, FilterUnsubscribe) + +proc dialFilterPeer( + self: WakuSubscriptionManager, + peer: RemotePeerInfo, + shard: PubsubTopic, + contentTopics: seq[ContentTopic], +) {.async.} = + ## Subscribe a new peer to all content topics on a shard and start tracking it. + self.edgeFilterSubStates.withValue(shard, state): + state.pendingPeers.incl(peer.peerId) + + try: + if not await self.sendChunkedFilterRpc(peer, shard, contentTopics, FilterSubscribe): + return + + self.edgeFilterSubStates.withValue(shard, state): + if state.peers.anyIt(it.peerId == peer.peerId): + trace "dialFilterPeer: peer already tracked, skipping duplicate", + shard = shard, peer = peer.peerId + return + + state.peers.add(peer) + self.updateShardHealth(shard, state[]) + trace "dialFilterPeer: successfully subscribed to all chunks", + shard = shard, peer = peer.peerId, totalPeers = state.peers.len + do: + trace "dialFilterPeer: shard removed while subscribing, discarding result", + shard = shard, peer = peer.peerId + finally: + self.edgeFilterSubStates.withValue(shard, state): + state.pendingPeers.excl(peer.peerId) + +proc pingFilterPeer( + self: WakuSubscriptionManager, peerId: PeerId, peer: RemotePeerInfo +): Future[(PeerId, bool)] {.async: (raises: []).} = + let req = ( + await kernel_filter_api.RequestFilterPing.request( + self.node.brokerCtx, peer, EdgeFilterPingTimeout + ) + ).valueOr: + return (peerId, false) + return (peerId, req.pingOk) + +proc edgeFilterMaintenanceLoop*(self: WakuSubscriptionManager) {.async.} = + ## Periodically pings all connected filter service peers. Peers that fail the ping are removed. + while true: + await sleepAsync(EdgeFilterLoopInterval) + + if not self.isFilterMounted(): + warn "filter client is nil within edge filter maintenance loop" + continue + + var connected = initTable[PeerId, RemotePeerInfo]() + for state in self.edgeFilterSubStates.values: + for peer in state.peers: + if self.node.peerManager.switch.peerStore.isConnected(peer.peerId): + connected[peer.peerId] = peer + + var alive = initHashSet[PeerId]() + + if connected.len > 0: + # Ping all connected peers concurrently; survivors go in `alive`. + var pingFuts: seq[Future[(PeerId, bool)]] + for peerId, peer in connected: + pingFuts.add(self.pingFilterPeer(peerId, peer)) + for f in pingFuts: + let (peerId, ok) = await f + if ok: + alive.incl(peerId) + + var changed = false + for shard, state in self.edgeFilterSubStates.mpairs: + let oldLen = state.peers.len + state.peers.keepItIf(it.peerId notin connected or alive.contains(it.peerId)) + + if state.peers.len < oldLen: + changed = true + self.updateShardHealth(shard, state) + trace "Edge Filter health degraded by Ping failure", + shard = shard, new = state.currentHealth + + if changed: + self.edgeFilterWakeup.fire() + +proc selectFilterCandidates( + self: WakuSubscriptionManager, shard: PubsubTopic, exclude: HashSet[PeerId], needed: int +): seq[RemotePeerInfo] = + ## Select filter service peer candidates for a shard. + + # Start with every filter server peer that can serve the shard + var allCandidates = self.node.peerManager.selectPeers( + filter_common.WakuFilterSubscribeCodec, some(shard) + ) + + # Remove all already used in this shard or being dialed for it + allCandidates.keepItIf(it.peerId notin exclude) + + # Collect peer IDs already tracked on other shards + var trackedOnOther = initHashSet[PeerId]() + for otherShard, otherState in self.edgeFilterSubStates.pairs: + if otherShard != shard: + for peer in otherState.peers: + trackedOnOther.incl(peer.peerId) + + # Prefer peers we already have a connection to first, preserving shuffle + var candidates = + allCandidates.filterIt(it.peerId in trackedOnOther) & + allCandidates.filterIt(it.peerId notin trackedOnOther) + + # We need to return 'needed' peers only + if candidates.len > needed: + candidates.setLen(needed) + return candidates + +proc edgeFilterSubLoop*(self: WakuSubscriptionManager) {.async.} = + ## Reconciles filter subscriptions with the desired state. + var lastSynced = initTable[PubsubTopic, HashSet[ContentTopic]]() + + while true: + await self.edgeFilterWakeup.wait() + await sleepAsync(EdgeFilterSubLoopDebounce) + self.edgeFilterWakeup.clear() + trace "edgeFilterSubLoop: woke up" + + if not self.isFilterMounted(): + trace "edgeFilterSubLoop: wakuFilterClient is nil, skipping" + continue + + let desired = self.edgeContentTopicSubs + + trace "edgeFilterSubLoop: desired state", numShards = desired.len + + let allShards = toHashSet(toSeq(desired.keys)) + toHashSet(toSeq(lastSynced.keys)) + + # Step 1: read state across all shards; build dial tasks and shards to delete. + + var dialTasks: seq[EdgeDialTask] + var shardsToDelete: seq[PubsubTopic] + + for shard in allShards: + let currTopics = desired.getOrDefault(shard) + let prevTopics = lastSynced.getOrDefault(shard) + + if shard notin self.edgeFilterSubStates: + self.edgeFilterSubStates[shard] = + EdgeFilterSubState(currentHealth: TopicHealth.UNHEALTHY) + + let addedTopics = toSeq(currTopics - prevTopics) + let removedTopics = toSeq(prevTopics - currTopics) + + self.edgeFilterSubStates.withValue(shard, state): + state.peers.keepItIf( + self.node.peerManager.switch.peerStore.isConnected(it.peerId) + ) + state.pending.keepItIf(not it.finished) + + if addedTopics.len > 0 or removedTopics.len > 0: + for peer in state.peers: + asyncSpawn self.syncFilterDeltas(peer, shard, addedTopics, removedTopics) + + if currTopics.len == 0: + shardsToDelete.add(shard) + else: + self.updateShardHealth(shard, state[]) + + let needed = max(0, HealthyThreshold - state.peers.len - state.pending.len) + + if needed > 0: + let tracked = state.peers.mapIt(it.peerId).toHashSet() + state.pendingPeers + let candidates = self.selectFilterCandidates(shard, tracked, needed) + let toDial = min(needed, candidates.len) + + trace "edgeFilterSubLoop: shard reconciliation", + shard = shard, + num_peers = state.peers.len, + num_pending = state.pending.len, + num_needed = needed, + num_available = candidates.len, + toDial = toDial + + for i in 0 ..< toDial: + dialTasks.add( + EdgeDialTask( + peer: candidates[i], shard: shard, topics: toSeq(currTopics) + ) + ) + + # Step 2: execute deferred shard deletion and dial tasks. + + for shard in shardsToDelete: + self.edgeFilterSubStates.withValue(shard, state): + for fut in state.pending: + if not fut.finished: + await fut.cancelAndWait() + self.edgeFilterSubStates.del(shard) + + for task in dialTasks: + let fut = self.dialFilterPeer(task.peer, task.shard, task.topics) + self.edgeFilterSubStates.withValue(task.shard, state): + state.pending.add(fut) + + lastSynced = desired + +proc startEdgeFilterLoops(self: WakuSubscriptionManager): Result[void, string] = + ## Start the edge filter orchestration loops. + ## Only valid in edge mode (relay nil, filter client present). + self.edgeFilterWakeup = newAsyncEvent() + + self.peerEventListener = EventWakuPeer.listen( + self.node.brokerCtx, + proc(evt: EventWakuPeer) {.async: (raises: []), gcsafe.} = + if evt.kind == EventWakuPeerKind.EventDisconnected or + evt.kind == EventWakuPeerKind.EventMetadataUpdated: + self.edgeFilterWakeup.fire() + , + ).valueOr: + return err("Failed to listen to peer events for edge filter: " & error) + + self.edgeFilterSubLoopFut = self.edgeFilterSubLoop() + self.edgeFilterMaintenanceLoopFut = self.edgeFilterMaintenanceLoop() + return ok() + +proc stopEdgeFilterLoops(self: WakuSubscriptionManager) {.async: (raises: []).} = + ## Stop the edge filter orchestration loops and clean up pending futures. + if not isNil(self.edgeFilterSubLoopFut): + await self.edgeFilterSubLoopFut.cancelAndWait() + self.edgeFilterSubLoopFut = nil + + if not isNil(self.edgeFilterMaintenanceLoopFut): + await self.edgeFilterMaintenanceLoopFut.cancelAndWait() + self.edgeFilterMaintenanceLoopFut = nil + + for shard, state in self.edgeFilterSubStates: + for fut in state.pending: + if not fut.finished: + await fut.cancelAndWait() + + await EventWakuPeer.dropListener(self.node.brokerCtx, self.peerEventListener) + +# WakuSubscriptionManager lifecycle. +# start/stopWakuSubscriptionManager orchestrate the relay and edge paths and +# register/clear broker providers. + +proc startWakuSubscriptionManager*(self: WakuSubscriptionManager): Result[void, string] = + RequestEdgeShardHealth.setProvider( + self.node.brokerCtx, + proc(shard: PubsubTopic): Result[RequestEdgeShardHealth, string] = + self.edgeFilterSubStates.withValue(shard, state): + return ok(RequestEdgeShardHealth(health: state.currentHealth)) + return ok(RequestEdgeShardHealth(health: TopicHealth.NOT_SUBSCRIBED)), + ).isOkOr: + error "Can't set provider for RequestEdgeShardHealth", error = error + + RequestEdgeFilterPeerCount.setProvider( + self.node.brokerCtx, + proc(): Result[RequestEdgeFilterPeerCount, string] = + var minPeers = high(int) + for state in self.edgeFilterSubStates.values: + minPeers = min(minPeers, state.peers.len) + if minPeers == high(int): + minPeers = 0 + return ok(RequestEdgeFilterPeerCount(peerCount: minPeers)), + ).isOkOr: + error "Can't set provider for RequestEdgeFilterPeerCount", error = error + + # The four-operation subscription surface on the broker. + RequestRelaySubscribeShard.setProvider( + self.node.brokerCtx, + proc(shard: PubsubTopic): Result[RequestRelaySubscribeShard, string] = + self.subscribeShard(shard).isOkOr: + return err(error) + return ok(RequestRelaySubscribeShard(subscribed: true)), + ).isOkOr: + error "Can't set provider for RequestRelaySubscribeShard", error = error + + RequestRelayUnsubscribeShard.setProvider( + self.node.brokerCtx, + proc(shard: PubsubTopic): Result[RequestRelayUnsubscribeShard, string] = + self.unsubscribeShard(shard).isOkOr: + return err(error) + return ok(RequestRelayUnsubscribeShard(unsubscribed: true)), + ).isOkOr: + error "Can't set provider for RequestRelayUnsubscribeShard", error = error + + RequestRelaySubscribeContentTopic.setProvider( + self.node.brokerCtx, + proc( + contentTopic: ContentTopic, shard: Option[PubsubTopic] + ): Result[RequestRelaySubscribeContentTopic, string] = + self.relaySubscribeContentTopic(contentTopic, shard).isOkOr: + return err(error) + return ok(RequestRelaySubscribeContentTopic(subscribed: true)), + ).isOkOr: + error "Can't set provider for RequestRelaySubscribeContentTopic", error = error + + RequestRelayUnsubscribeContentTopic.setProvider( + self.node.brokerCtx, + proc( + contentTopic: ContentTopic, shard: Option[PubsubTopic] + ): Result[RequestRelayUnsubscribeContentTopic, string] = + self.relayUnsubscribeContentTopic(contentTopic, shard).isOkOr: + return err(error) + return ok(RequestRelayUnsubscribeContentTopic(unsubscribed: true)), + ).isOkOr: + error "Can't set provider for RequestRelayUnsubscribeContentTopic", error = error + + RequestEdgeSubscribe.setProvider( + self.node.brokerCtx, + proc( + contentTopic: ContentTopic, shard: Option[PubsubTopic] + ): Result[RequestEdgeSubscribe, string] = + self.edgeSubscribe(contentTopic, shard).isOkOr: + return err(error) + return ok(RequestEdgeSubscribe(subscribed: true)), + ).isOkOr: + error "Can't set provider for RequestEdgeSubscribe", error = error + + RequestEdgeUnsubscribe.setProvider( + self.node.brokerCtx, + proc( + contentTopic: ContentTopic, shard: Option[PubsubTopic] + ): Result[RequestEdgeUnsubscribe, string] = + self.edgeUnsubscribe(contentTopic, shard).isOkOr: + return err(error) + return ok(RequestEdgeUnsubscribe(unsubscribed: true)), + ).isOkOr: + error "Can't set provider for RequestEdgeUnsubscribe", error = error + + RequestIsRelaySubscribed.setProvider( + self.node.brokerCtx, + proc( + contentTopic: ContentTopic, shard: Option[PubsubTopic] + ): Result[RequestIsRelaySubscribed, string] = + let resolved = ?self.resolveShard(contentTopic, shard) + return ok( + RequestIsRelaySubscribed(subscribed: self.isRelaySubscribed(resolved, contentTopic)) + ), + ).isOkOr: + error "Can't set provider for RequestIsRelaySubscribed", error = error + + RequestIsEdgeSubscribed.setProvider( + self.node.brokerCtx, + proc( + contentTopic: ContentTopic, shard: Option[PubsubTopic] + ): Result[RequestIsEdgeSubscribed, string] = + let resolved = ?self.resolveShard(contentTopic, shard) + return ok( + RequestIsEdgeSubscribed(subscribed: self.isEdgeSubscribed(resolved, contentTopic)) + ), + ).isOkOr: + error "Can't set provider for RequestIsEdgeSubscribed", error = error + + RequestIsSubscribed.setProvider( + self.node.brokerCtx, + proc( + contentTopic: ContentTopic, shard: Option[PubsubTopic] + ): Result[RequestIsSubscribed, string] = + let resolved = ?self.resolveShard(contentTopic, shard) + # Default multiplexing: relay if mounted, else edge. + return ok( + RequestIsSubscribed( + subscribed: + if self.isRelayMounted(): + self.isRelaySubscribed(resolved, contentTopic) + else: + self.isEdgeSubscribed(resolved, contentTopic) + ) + ), + ).isOkOr: + error "Can't set provider for RequestIsSubscribed", error = error + + RequestRelaySubscribedTopics.setProvider( + self.node.brokerCtx, + proc(): Result[RequestRelaySubscribedTopics, string] = + var topics: seq[tuple[shard: PubsubTopic, contentTopics: seq[ContentTopic]]] + for shard, cTopics in self.relaySubscribedTopics: + topics.add((shard: shard, contentTopics: toSeq(cTopics))) + return ok(RequestRelaySubscribedTopics(topics: topics)), + ).isOkOr: + error "Can't set provider for RequestRelaySubscribedTopics", error = error + + # Default multiplexing: relay if mounted, else edge. + RequestSubscribedTopics.setProvider( + self.node.brokerCtx, + proc(): Result[RequestSubscribedTopics, string] = + var topics: seq[tuple[shard: PubsubTopic, contentTopics: seq[ContentTopic]]] + if self.isRelayMounted(): + for shard, cTopics in self.relaySubscribedTopics: + topics.add((shard: shard, contentTopics: toSeq(cTopics))) + else: + for shard, cTopics in self.edgeSubscribedTopics: + topics.add((shard: shard, contentTopics: toSeq(cTopics))) + return ok(RequestSubscribedTopics(topics: topics)), + ).isOkOr: + error "Can't set provider for RequestSubscribedTopics", error = error + + RequestEdgeSubscribedTopics.setProvider( + self.node.brokerCtx, + proc(): Result[RequestEdgeSubscribedTopics, string] = + var topics: seq[tuple[shard: PubsubTopic, contentTopics: seq[ContentTopic]]] + for shard, cTopics in self.edgeSubscribedTopics: + topics.add((shard: shard, contentTopics: toSeq(cTopics))) + return ok(RequestEdgeSubscribedTopics(topics: topics)), + ).isOkOr: + error "Can't set provider for RequestEdgeSubscribedTopics", error = error + + # Fan out shard-health changes to per-content-topic health events. A content + # topic's health is its shard's health. Set up in both modes. + self.shardHealthListener = EventShardTopicHealthChange.listen( + self.node.brokerCtx, + proc(evt: EventShardTopicHealthChange) {.async: (raises: []), gcsafe.} = + let cTopics = + self.relayContentTopicSubs.getOrDefault(evt.topic) + + self.edgeContentTopicSubs.getOrDefault(evt.topic) + for ct in cTopics: + EventContentTopicHealthChange.emit(self.node.brokerCtx, ct, evt.health) + , + ).valueOr: + return err("Failed to listen to shard health events: " & error) + + if not self.isRelayMounted(): + return self.startEdgeFilterLoops() + + # Core mode: auto-subscribe relay to all autosharding shards. + if self.node.wakuAutoSharding.isSome(): + let autoSharding = self.node.wakuAutoSharding.get() + let clusterId = autoSharding.clusterId + let numShards = autoSharding.shardCountGenZero + + if numShards > 0: + for i in 0 ..< numShards: + let shardObj = RelayShard(clusterId: clusterId, shardId: uint16(i)) + self.subscribeShard(PubsubTopic($shardObj)).isOkOr: + error "Failed to auto-subscribe Relay to cluster shard: ", + shard = $shardObj, error = error + else: + info "WakuSubscriptionManager has no AutoSharding configured; skipping auto-subscribe." + + return ok() + +proc stopWakuSubscriptionManager*(self: WakuSubscriptionManager) {.async: (raises: []).} = + if not self.isRelayMounted(): + await self.stopEdgeFilterLoops() + await EventShardTopicHealthChange.dropListener( + self.node.brokerCtx, self.shardHealthListener + ) + RequestEdgeShardHealth.clearProvider(self.node.brokerCtx) + RequestEdgeFilterPeerCount.clearProvider(self.node.brokerCtx) + RequestRelaySubscribeShard.clearProvider(self.node.brokerCtx) + RequestRelayUnsubscribeShard.clearProvider(self.node.brokerCtx) + RequestRelaySubscribeContentTopic.clearProvider(self.node.brokerCtx) + RequestRelayUnsubscribeContentTopic.clearProvider(self.node.brokerCtx) + RequestEdgeSubscribe.clearProvider(self.node.brokerCtx) + RequestEdgeUnsubscribe.clearProvider(self.node.brokerCtx) + RequestIsRelaySubscribed.clearProvider(self.node.brokerCtx) + RequestIsEdgeSubscribed.clearProvider(self.node.brokerCtx) + RequestIsSubscribed.clearProvider(self.node.brokerCtx) + RequestRelaySubscribedTopics.clearProvider(self.node.brokerCtx) + RequestEdgeSubscribedTopics.clearProvider(self.node.brokerCtx) + RequestSubscribedTopics.clearProvider(self.node.brokerCtx) diff --git a/waku/node/waku_node.nim b/waku/node/waku_node.nim index 26a2b5a57..0710b74c0 100644 --- a/waku/node/waku_node.nim +++ b/waku/node/waku_node.nim @@ -52,22 +52,32 @@ import waku_enr, waku_peer_exchange, waku_rln_relay, + api/requests/relay as relay_api, + api/requests/lightpush as lightpush_api, + api/requests/store as store_api, + api/requests/filter as filter_api, + api/requests/peers as peers_api, + api/requests/protocols as protocols_api, common/rate_limit/setting, common/callbacks, common/nimchronos, waku_mix, - requests/node_requests, + api/requests/node, + api/requests/health, requests/health_requests, - events/health_events, - events/message_events, + api/events/health, + api/events/message, + events/peer_events, ], waku/discovery/waku_kademlia, waku/net/[bound_ports, net_config], ./peer_manager, ./health_monitor/health_status, - ./health_monitor/topic_health + ./health_monitor/topic_health, + ./waku_telemetry -declarePublicCounter waku_node_messages, "number of messages received", ["type"] +export waku_telemetry # waku_node_messages / waku_histogram_message_size, shared + # by kernel_api/relay and WakuSubscriptionManager. declarePublicGauge waku_version, "Waku version info (in git describe format)", ["version"] @@ -102,6 +112,40 @@ type enrUri*: string #multiaddrStrings*: seq[string] mixPubKey*: Option[string] + ## Subscription engine state. Procs live in `./subscription_manager.nim`; + ## the type bodies are here to avoid an import cycle. + + EdgeFilterSubState* = object + peers*: seq[RemotePeerInfo] + ## Filter service peers with confirmed subscriptions on this shard. + pending*: seq[Future[void]] ## In-flight dial futures for peers not yet confirmed. + pendingPeers*: HashSet[PeerId] ## PeerIds of peers currently being dialed. + currentHealth*: TopicHealth + ## Health derived from peers.len; updated on every peer set change. + + WakuSubscriptionManager* = ref object of RootObj + node*: WakuNode + relayContentTopicSubs*: Table[PubsubTopic, HashSet[ContentTopic]] + ## Per-shard content-topic interest for relay. A shard key is + ## present only while it has at least one content-topic interest. + edgeContentTopicSubs*: Table[PubsubTopic, HashSet[ContentTopic]] + ## Per-shard content-topic interest for edge. A shard key is + ## present only while it has at least one content-topic interest. + directShardSubs*: HashSet[PubsubTopic] + ## A shard's relay-mesh subscription is held while it is in this + ## set OR has non-empty content-topic interest. + edgeFilterSubStates*: Table[PubsubTopic, EdgeFilterSubState] + ## Per-shard filter subscription state for edge mode. + edgeFilterWakeup*: AsyncEvent + ## Set when the edge filter sub loop should re-reconcile. + edgeFilterSubLoopFut*: Future[void] + edgeFilterMaintenanceLoopFut*: Future[void] + peerEventListener*: EventWakuPeerListener + ## Listener for peer connect/disconnect events (edge filter wakeup). + shardHealthListener*: EventShardTopicHealthChangeListener + ## Listener on shard-health changes; fans them out to per-content-topic + ## health events for every content topic subscribed on the shard. + # NOTE based on Eth2Node in NBC eth2_network.nim WakuNode* = ref object peerManager*: PeerManager @@ -141,8 +185,9 @@ type kademliaDiscoveryLoop*: Future[void] wakuKademlia*: WakuKademlia ports*: BoundPorts + subscriptionManager*: WakuSubscriptionManager -proc deduceRelayShard( +proc deduceRelayShard*( node: WakuNode, contentTopic: ContentTopic, pubsubTopicOp: Option[PubsubTopic] = none[PubsubTopic](), @@ -481,6 +526,39 @@ proc updateAnnouncedAddrWithPrimaryIpAddr*(node: WakuNode): Result[void, string] return ok() +proc registerPeersProviders(node: WakuNode): Result[void, string] = + ## Bind the peers broker providers. + peers_api.RequestSelectPeer.setProvider( + node.brokerCtx, + proc( + proto: string, shard: Option[PubsubTopic] + ): Result[peers_api.RequestSelectPeer, string] {.gcsafe, raises: [].} = + ok(peers_api.RequestSelectPeer(peer: node.peerManager.selectPeer(proto, shard))), + ).isOkOr: + return err("registerPeersProviders: RequestSelectPeer: " & error) + + peers_api.RequestSelectPeers.setProvider( + node.brokerCtx, + proc( + proto: string, shard: Option[PubsubTopic] + ): Result[peers_api.RequestSelectPeers, string] {.gcsafe, raises: [].} = + ok(peers_api.RequestSelectPeers(peers: node.peerManager.selectPeers(proto, shard))), + ).isOkOr: + return err("registerPeersProviders: RequestSelectPeers: " & error) + + peers_api.RequestIsPeerConnected.setProvider( + node.brokerCtx, + proc(peerId: PeerId): Result[peers_api.RequestIsPeerConnected, string] {.gcsafe, raises: [].} = + ok( + peers_api.RequestIsPeerConnected( + connected: node.peerManager.switch.peerStore.isConnected(peerId) + ) + ), + ).isOkOr: + return err("registerPeersProviders: RequestIsPeerConnected: " & error) + + ok() + proc startProvidersAndListeners*(node: WakuNode) = RequestRelayShard.setProvider( node.brokerCtx, @@ -552,11 +630,40 @@ proc startProvidersAndListeners*(node: WakuNode) = ).isOkOr: error "Can't set provider for RequestContentTopicsHealth", error = error + # Which optional protocol clients are mounted; consumed by the messaging layer. + protocols_api.RequestProtocolMountStatus.setProvider( + node.brokerCtx, + proc(): Result[protocols_api.RequestProtocolMountStatus, string] = + ok( + protocols_api.RequestProtocolMountStatus( + relayMounted: not node.wakuRelay.isNil(), + lightpushMounted: not node.wakuLightpushClient.isNil(), + filterMounted: not node.wakuFilterClient.isNil(), + storeMounted: not node.wakuStoreClient.isNil(), + ) + ), + ).isOkOr: + error "Can't set provider for RequestProtocolMountStatus", error = error + + registerPeersProviders(node).isOkOr: + error "failed to register peers API providers", error = error + proc stopProvidersAndListeners*(node: WakuNode) = RequestRelayShard.clearProvider(node.brokerCtx) RequestContentTopicsHealth.clearProvider(node.brokerCtx) RequestShardTopicsHealth.clearProvider(node.brokerCtx) + relay_api.RequestRelayPublish.clearProvider(node.brokerCtx) + lightpush_api.RequestLightpushPublish.clearProvider(node.brokerCtx) + store_api.RequestStoreQueryToAny.clearProvider(node.brokerCtx) + filter_api.RequestFilterSubscribe.clearProvider(node.brokerCtx) + filter_api.RequestFilterUnsubscribe.clearProvider(node.brokerCtx) + filter_api.RequestFilterPing.clearProvider(node.brokerCtx) + peers_api.RequestSelectPeer.clearProvider(node.brokerCtx) + peers_api.RequestSelectPeers.clearProvider(node.brokerCtx) + peers_api.RequestIsPeerConnected.clearProvider(node.brokerCtx) + protocols_api.RequestProtocolMountStatus.clearProvider(node.brokerCtx) + proc start*(node: WakuNode) {.async.} = ## Starts a created Waku Node and ## all its mounted protocols. diff --git a/waku/node/waku_telemetry.nim b/waku/node/waku_telemetry.nim new file mode 100644 index 000000000..8209c5b01 --- /dev/null +++ b/waku/node/waku_telemetry.nim @@ -0,0 +1,17 @@ +{.push raises: [].} + +## Shared declarations for node telemetry metrics. Both relay-handler paths +## observe these collectors; declaring them once avoids a duplicate Prometheus +## registration. + +import metrics + +declarePublicCounter waku_node_messages, "number of messages received", ["type"] + +declarePublicHistogram waku_histogram_message_size, + "message size histogram in kB", + buckets = [ + 0.0, 1.0, 3.0, 5.0, 15.0, 50.0, 75.0, 100.0, 125.0, 150.0, 500.0, 700.0, 1000.0, Inf + ] + +{.pop.} diff --git a/waku/requests/health_requests.nim b/waku/requests/health_requests.nim index d48b3278f..08b1b7e9c 100644 --- a/waku/requests/health_requests.nim +++ b/waku/requests/health_requests.nim @@ -1,51 +1,17 @@ import brokers/request_broker - -import waku/api/types -import waku/node/health_monitor/[protocol_health, topic_health, health_report] +import waku/node/health_monitor/topic_health import waku/waku_core/topics -import waku/common/waku_protocol -export protocol_health, topic_health +export topic_health -# Get the overall node connectivity status -RequestBroker(sync): - type RequestConnectionStatus* = object - connectionStatus*: ConnectionStatus - -# Get the health status of a set of content topics -RequestBroker(sync): - type RequestContentTopicsHealth* = object - contentTopicHealth*: seq[tuple[topic: ContentTopic, health: TopicHealth]] - - proc signature(topics: seq[ContentTopic]): Result[RequestContentTopicsHealth, string] - -# Get a consolidated node health report -RequestBroker: - type RequestHealthReport* = object - healthReport*: HealthReport - -# Get the health status of a set of shards (pubsub topics) -RequestBroker(sync): - type RequestShardTopicsHealth* = object - topicHealth*: seq[tuple[topic: PubsubTopic, health: TopicHealth]] - - proc signature(topics: seq[PubsubTopic]): Result[RequestShardTopicsHealth, string] - -# Get the health status of a mounted protocol -RequestBroker: - type RequestProtocolHealth* = object - healthStatus*: ProtocolHealth - - proc signature(protocol: WakuProtocol): Future[Result[RequestProtocolHealth, string]] - -# Get edge filter health for a single shard (set by DeliveryService when edge mode is active) +# Edge filter health for a single shard, folded into RequestShardTopicsHealth by its provider. RequestBroker(sync): type RequestEdgeShardHealth* = object health*: TopicHealth proc signature(shard: PubsubTopic): Result[RequestEdgeShardHealth, string] -# Get edge filter confirmed peer count (set by DeliveryService when edge mode is active) +# Edge filter confirmed peer count. WakuSubscriptionManager sets it; health_monitor reads it. RequestBroker(sync): type RequestEdgeFilterPeerCount* = object peerCount*: int diff --git a/waku/requests/requests.nim b/waku/requests/requests.nim index 9225c0f3e..4218f121a 100644 --- a/waku/requests/requests.nim +++ b/waku/requests/requests.nim @@ -1,3 +1,2 @@ -import ./[health_requests, rln_requests, node_requests] - -export health_requests, rln_requests, node_requests +import ./health_requests +export health_requests diff --git a/waku/rest_api/endpoint/health/types.nim b/waku/rest_api/endpoint/health/types.nim index 88fa736a8..6902b8894 100644 --- a/waku/rest_api/endpoint/health/types.nim +++ b/waku/rest_api/endpoint/health/types.nim @@ -3,7 +3,7 @@ import results import chronicles, json_serialization, json_serialization/std/options import ../serdes -import waku/[waku_node, api/types] +import waku/waku_node #### Serialization and deserialization diff --git a/waku/waku_filter_v2/client.nim b/waku/waku_filter_v2/client.nim index 7798f41b7..c430d6bcb 100644 --- a/waku/waku_filter_v2/client.nim +++ b/waku/waku_filter_v2/client.nim @@ -12,7 +12,7 @@ import brokers/broker_context import - waku/[node/peer_manager, waku_core, events/delivery_events], + waku/[node/peer_manager, waku_core], ./common, ./protocol_metrics, ./rpc_codec, @@ -137,8 +137,6 @@ proc subscribe*( ?await wfc.sendSubscribeRequest(servicePeer, filterSubscribeRequest) - OnFilterSubscribeEvent.emit(wfc.brokerCtx, pubsubTopic, contentTopicSeq) - return ok() proc unsubscribe*( @@ -160,8 +158,6 @@ proc unsubscribe*( ?await wfc.sendSubscribeRequest(servicePeer, filterSubscribeRequest) - OnFilterUnSubscribeEvent.emit(wfc.brokerCtx, pubsubTopic, contentTopicSeq) - return ok() proc unsubscribeAll*( diff --git a/waku/waku_relay/protocol.nim b/waku/waku_relay/protocol.nim index d0b1ddb48..3e3e87b72 100644 --- a/waku/waku_relay/protocol.nim +++ b/waku/waku_relay/protocol.nim @@ -22,8 +22,8 @@ import import waku/waku_core, waku/node/health_monitor/topic_health, - waku/requests/health_requests, - waku/events/health_events, + waku/api/requests/health, + waku/api/events/health, ./message_id, waku/events/peer_events @@ -157,7 +157,7 @@ type ): Future[ValidationResult] {.gcsafe, raises: [Defect].} WakuRelay* = ref object of GossipSub brokerCtx: BrokerContext - peerEventListener: WakuPeerEventListener + peerEventListener: EventWakuPeerListener # seq of tuples: the first entry in the tuple contains the validators are called for every topic # the second entry contains the error messages to be returned when the validator fails wakuValidators: seq[tuple[handler: WakuValidatorHandler, errorMessage: string]] @@ -378,10 +378,10 @@ proc new*( w.initProtocolHandler() w.initRelayObservers() - w.peerEventListener = WakuPeerEvent.listen( + w.peerEventListener = EventWakuPeer.listen( w.brokerCtx, - proc(evt: WakuPeerEvent): Future[void] {.async: (raises: []), gcsafe.} = - if evt.kind == WakuPeerEventKind.EventDisconnected: + proc(evt: EventWakuPeer): Future[void] {.async: (raises: []), gcsafe.} = + if evt.kind == EventWakuPeerKind.EventDisconnected: w.topicHealthCheckAll = true w.topicHealthUpdateEvent.fire() , @@ -526,7 +526,7 @@ method stop*(w: WakuRelay) {.async: (raises: []).} = info "stop" await procCall GossipSub(w).stop() - await WakuPeerEvent.dropListener(w.brokerCtx, w.peerEventListener) + await EventWakuPeer.dropListener(w.brokerCtx, w.peerEventListener) if not w.topicHealthLoopHandle.isNil(): await w.topicHealthLoopHandle.cancelAndWait() diff --git a/waku/waku_rln_relay/rln_relay.nim b/waku/waku_rln_relay/rln_relay.nim index 7c36300b2..027c3a25f 100644 --- a/waku/waku_rln_relay/rln_relay.nim +++ b/waku/waku_rln_relay/rln_relay.nim @@ -30,7 +30,7 @@ import common/error_handling, waku_relay, # for WakuRelayHandler waku_core, - requests/rln_requests, + api/requests/rln, waku_keystore, ]