diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9ddf904ef..67a29fa31 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -89,12 +89,15 @@ jobs: path: | nimbledeps/ nimble.paths - key: ${{ runner.os }}-nimbledeps-nimble${{ env.NIMBLE_VERSION }}-${{ hashFiles('nimble.lock', 'BearSSL.mk', 'Nat.mk') }} + key: ${{ runner.os }}-nimbledeps-v2-nimble${{ env.NIMBLE_VERSION }}-${{ hashFiles('nimble.lock', 'BearSSL.mk', 'Nat.mk') }} - name: Install nimble deps if: steps.cache-nimbledeps.outputs.cache-hit != 'true' run: | - nimble setup --localdeps -y + # nim's source tree checksums differently across platforms, so its + # locked checksum is unreliable. --useSystemNim uses the CI nim and + # skips that check, while still verifying every other locked dep. + nimble setup --localdeps -y --useSystemNim make rebuild-nat-libs-nimbledeps make rebuild-bearssl-nimbledeps touch nimbledeps/.nimble-setup @@ -142,12 +145,15 @@ jobs: path: | nimbledeps/ nimble.paths - key: ${{ runner.os }}-nimbledeps-nimble${{ env.NIMBLE_VERSION }}-${{ hashFiles('nimble.lock', 'BearSSL.mk', 'Nat.mk') }} + key: ${{ runner.os }}-nimbledeps-v2-nimble${{ env.NIMBLE_VERSION }}-${{ hashFiles('nimble.lock', 'BearSSL.mk', 'Nat.mk') }} - name: Install nimble deps if: steps.cache-nimbledeps.outputs.cache-hit != 'true' run: | - nimble setup --localdeps -y + # nim's source tree checksums differently across platforms, so its + # locked checksum is unreliable. --useSystemNim uses the CI nim and + # skips that check, while still verifying every other locked dep. + nimble setup --localdeps -y --useSystemNim make rebuild-nat-libs-nimbledeps make rebuild-bearssl-nimbledeps touch nimbledeps/.nimble-setup @@ -207,12 +213,15 @@ jobs: path: | nimbledeps/ nimble.paths - key: ${{ runner.os }}-nimbledeps-nimble${{ env.NIMBLE_VERSION }}-${{ hashFiles('nimble.lock', 'BearSSL.mk', 'Nat.mk') }} + key: ${{ runner.os }}-nimbledeps-v2-nimble${{ env.NIMBLE_VERSION }}-${{ hashFiles('nimble.lock', 'BearSSL.mk', 'Nat.mk') }} - name: Install nimble deps if: steps.cache-nimbledeps.outputs.cache-hit != 'true' run: | - nimble setup --localdeps -y + # nim's source tree checksums differently across platforms, so its + # locked checksum is unreliable. --useSystemNim uses the CI nim and + # skips that check, while still verifying every other locked dep. + nimble setup --localdeps -y --useSystemNim make rebuild-nat-libs-nimbledeps make rebuild-bearssl-nimbledeps touch nimbledeps/.nimble-setup diff --git a/.github/workflows/container-image.yml b/.github/workflows/container-image.yml index 0ff427d87..b2066438b 100644 --- a/.github/workflows/container-image.yml +++ b/.github/workflows/container-image.yml @@ -15,6 +15,10 @@ env: NPROC: 2 MAKEFLAGS: "-j${NPROC}" NIMFLAGS: "--parallelBuild:${NPROC}" + # waku.nimble reads compile flags from NIM_PARAMS, not NIMFLAGS. Without + # -d:disableMarchNative here, config.nims applies -march=native and + # secp256k1 fails to compile. + NIM_PARAMS: "-d:disableMarchNative" NIM_VERSION: '2.2.4' NIMBLE_VERSION: '0.22.3' diff --git a/.github/workflows/windows-build.yml b/.github/workflows/windows-build.yml index 50f1602cd..4d955de23 100644 --- a/.github/workflows/windows-build.yml +++ b/.github/workflows/windows-build.yml @@ -24,6 +24,16 @@ jobs: MSYSTEM: MINGW64 steps: + - name: Configure Git to keep LF line endings + # Windows Git defaults to core.autocrlf=true. The LF→CRLF conversion + # changes the SHA1 of nimble's cloned deps, so they no longer match + # nimble.lock and nimble re-downloads them on every run (hanging the + # job). Disable autocrlf so clones match the Linux-computed checksums. + shell: pwsh + run: | + git config --global core.autocrlf false + git config --global core.eol lf + - name: Checkout code uses: actions/checkout@v4 @@ -52,6 +62,14 @@ jobs: mingw-w64-x86_64-clang mingw-w64-x86_64-nasm + - name: Configure Git in MSYS2 to keep LF line endings + # The step above only configures Git for Windows. nimble clones its deps + # from the MSYS2 shell, whose git reads a separate global config, so the + # same setting must be repeated here. + run: | + git config --global core.autocrlf false + git config --global core.eol lf + - name: Manually install nasm run: | bash scripts/install_nasm_in_windows.sh @@ -80,19 +98,16 @@ jobs: cd /tmp && nimble install "nimble@${{ env.NIMBLE_VERSION }}" -y echo "$HOME/.nimble/bin" >> $GITHUB_PATH - - name: Patch nimble.lock for Windows nim checksum - # nimble.exe uses Windows Git (core.autocrlf=true by default), which converts LF→CRLF - # on checkout. This changes the SHA1 of the nim package source tree relative to the - # Linux-computed checksum stored in nimble.lock. Patch the lock file with the - # Windows-computed checksum before nimble reads it. - run: | - sed -i 's/68bb85cbfb1832ce4db43943911b046c3af3caab/a092a045d3a427d127a5334a6e59c76faff54686/g' nimble.lock - - name: Install nimble deps if: steps.cache-nimbledeps.outputs.cache-hit != 'true' run: | export PATH="$GITHUB_WORKSPACE/.nim_runtime/bin:$HOME/.nimble/bin:$PATH" - nimble setup --localdeps -y + # nim's source tree checks out differently per platform (its own + # .gitattributes forces line endings), so its locked checksum never + # matches on Windows — even with autocrlf disabled. --useSystemNim + # uses the CI-installed nim and skips that check, while still + # verifying every other locked dependency. + nimble setup --localdeps -y --useSystemNim make rebuild-nat-libs-nimbledeps CC=gcc make rebuild-bearssl-nimbledeps CC=gcc touch nimbledeps/.nimble-setup diff --git a/Makefile b/Makefile index ea1bf66f0..344743c86 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,9 @@ export PATH := $(HOME)/.nimble/bin:$(PATH) # NIM binary location NIM_BINARY := $(shell which nim 2>/dev/null) NPH := $(HOME)/.nimble/bin/nph -NIMBLE := $(HOME)/.nimble/bin/nimble +# Resolve nimble via PATH (Windows has no $(HOME)/.nimble/bin); --useSystemNim +# reuses the nim on PATH so nimble never re-clones the locked nim. +NIMBLE := nimble --useSystemNim NIMBLEDEPS_STAMP := nimbledeps/.nimble-setup # Compilation parameters @@ -204,7 +206,7 @@ clean: | clean-librln testcommon: | build-deps build echo -e $(BUILD_MSG) "build/$@" && \ - nimble testcommon + $(NIMBLE) testcommon ########## ## Waku ## @@ -213,47 +215,54 @@ testcommon: | build-deps build testwaku: | build-deps build rln-deps librln echo -e $(BUILD_MSG) "build/$@" && \ - nimble test + $(NIMBLE) test +# Windows: build with nim directly — `nimble ` re-clones git deps every +# build and they intermittently hang on the MSYS2 runner. Flags mirror waku.nimble. wakunode2: | build-deps build deps librln +ifeq ($(detected_OS),Windows) echo -e $(BUILD_MSG) "build/$@" && \ - nimble wakunode2 + nim c --out:build/wakunode2 --mm:refc --cpu:amd64 $(NIM_PARAMS) -d:chronicles_log_level=TRACE apps/wakunode2/wakunode2.nim +else + echo -e $(BUILD_MSG) "build/$@" && \ + $(NIMBLE) wakunode2 +endif benchmarks: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ - nimble benchmarks + $(NIMBLE) benchmarks testwakunode2: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ - nimble testwakunode2 + $(NIMBLE) testwakunode2 example2: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ - nimble example2 + $(NIMBLE) example2 chat2: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ - nimble chat2 + $(NIMBLE) chat2 chat2mix: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ - nimble chat2mix + $(NIMBLE) chat2mix rln-db-inspector: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ - nimble rln_db_inspector + $(NIMBLE) rln_db_inspector chat2bridge: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ - nimble chat2bridge + $(NIMBLE) chat2bridge liteprotocoltester: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ - nimble liteprotocoltester + $(NIMBLE) liteprotocoltester lightpushwithmix: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ - nimble lightpushwithmix + $(NIMBLE) lightpushwithmix api_example: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ @@ -261,12 +270,12 @@ api_example: | build-deps build deps librln build/%: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$*" && \ - nimble buildone $* + $(NIMBLE) buildone $* compile-test: | build-deps build deps librln echo -e $(BUILD_MSG) "$(TEST_FILE)" "\"$(TEST_NAME)\"" && \ - nimble buildTest $(TEST_FILE) && \ - nimble execTest $(TEST_FILE) "\"$(TEST_NAME)\"" + $(NIMBLE) buildTest $(TEST_FILE) && \ + $(NIMBLE) execTest $(TEST_FILE) "\"$(TEST_NAME)\"" ################ ## Waku tools ## @@ -277,11 +286,11 @@ tools: networkmonitor wakucanary wakucanary: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ - nimble wakucanary + $(NIMBLE) wakucanary networkmonitor: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ - nimble networkmonitor + $(NIMBLE) networkmonitor ############ ## Format ## @@ -327,7 +336,7 @@ clean: docs: | build deps echo -e $(BUILD_MSG) "build/$@" && \ - nimble doc --run --index:on --project --out:.gh-pages waku/waku.nim waku.nims + $(NIMBLE) doc --run --index:on --project --out:.gh-pages waku/waku.nim waku.nims coverage: echo -e $(BUILD_MSG) "build/$@" && \ @@ -423,11 +432,16 @@ else ifeq ($(detected_OS),Linux) BUILD_COMMAND := $(BUILD_COMMAND)Linux endif +# Windows: build with nim directly (see wakunode2). Flags mirror waku.nimble. libwaku: | build-deps librln - nimble --verbose libwaku$(BUILD_COMMAND) waku.nimble +ifeq ($(detected_OS),Windows) + nim c --out:build/libwaku.dll --threads:on --app:lib --opt:speed --noMain --mm:refc --header -d:metrics --nimMainPrefix:libwaku --skipParentCfg:off -d:discv5_protocol_id=d5waku --cpu:amd64 $(NIM_PARAMS) library/libwaku.nim +else + $(NIMBLE) --verbose libwaku$(BUILD_COMMAND) waku.nimble +endif liblogosdelivery: | build-deps librln - nimble --verbose liblogosdelivery$(BUILD_COMMAND) waku.nimble + $(NIMBLE) --verbose liblogosdelivery$(BUILD_COMMAND) waku.nimble logosdelivery_example: | build liblogosdelivery @echo -e $(BUILD_MSG) "build/$@" @@ -502,7 +516,7 @@ endif build-libwaku-for-android-arch: ifneq ($(findstring /nix/store,$(LIBRLN_FILE)),) mkdir -p $(CURDIR)/build/android/$(ABIDIR)/ - CPU=$(CPU) ABIDIR=$(ABIDIR) ANDROID_ARCH=$(ANDROID_ARCH) ANDROID_COMPILER=$(ANDROID_COMPILER) ANDROID_TOOLCHAIN_DIR=$(ANDROID_TOOLCHAIN_DIR) nimble libWakuAndroid + CPU=$(CPU) ABIDIR=$(ABIDIR) ANDROID_ARCH=$(ANDROID_ARCH) ANDROID_COMPILER=$(ANDROID_COMPILER) ANDROID_TOOLCHAIN_DIR=$(ANDROID_TOOLCHAIN_DIR) $(NIMBLE) libWakuAndroid else ./scripts/build_rln_android.sh $(CURDIR)/build $(LIBRLN_BUILDDIR) $(LIBRLN_VERSION) $(CROSS_TARGET) $(ABIDIR) endif @@ -559,7 +573,7 @@ else endif build-libwaku-for-ios-arch: - IOS_SDK=$(IOS_SDK) IOS_ARCH=$(IOS_ARCH) IOS_SDK_PATH=$(IOS_SDK_PATH) nimble libWakuIOS + IOS_SDK=$(IOS_SDK) IOS_ARCH=$(IOS_ARCH) IOS_SDK_PATH=$(IOS_SDK_PATH) $(NIMBLE) libWakuIOS libwaku-ios-device: IOS_ARCH=arm64 libwaku-ios-device: IOS_SDK=iphoneos diff --git a/apps/liteprotocoltester/liteprotocoltester.nim b/apps/liteprotocoltester/liteprotocoltester.nim index 46c85e910..1877b8477 100644 --- a/apps/liteprotocoltester/liteprotocoltester.nim +++ b/apps/liteprotocoltester/liteprotocoltester.nim @@ -123,7 +123,7 @@ when isMainModule: error "Waku initialization failed", error = error quit(QuitFailure) - (waitFor startWaku(addr waku)).isOkOr: + (waitFor waku.start()).isOkOr: error "Starting waku failed", error = error quit(QuitFailure) diff --git a/apps/wakucanary/wakucanary.nim b/apps/wakucanary/wakucanary.nim index e7b1ff9aa..f9023c45c 100644 --- a/apps/wakucanary/wakucanary.nim +++ b/apps/wakucanary/wakucanary.nim @@ -13,6 +13,7 @@ import import ./certsgenerator, waku/[waku_enr, node/peer_manager, waku_core, waku_node, factory/builder], + waku/net/net_config, waku/waku_metadata/protocol, waku/common/callbacks diff --git a/apps/wakunode2/wakunode2.nim b/apps/wakunode2/wakunode2.nim index 484adf68f..be3a83f57 100644 --- a/apps/wakunode2/wakunode2.nim +++ b/apps/wakunode2/wakunode2.nim @@ -55,7 +55,7 @@ when isMainModule: error "Waku initialization failed", error = error quit(QuitFailure) - (waitFor startWaku(addr waku)).isOkOr: + (waitFor waku.start()).isOkOr: error "Starting waku failed", error = error quit(QuitFailure) diff --git a/channels/events.nim b/channels/events.nim index 904a34dc6..3e271976e 100644 --- a/channels/events.nim +++ b/channels/events.nim @@ -1,7 +1,7 @@ ## Reliable Channel event types emitted to API consumers. ## ## Lifecycle events for individual segments (sent / propagated / errored) -## are the same as the network-level ones the DeliveryService already +## are the same as the network-level ones the MessagingClient already ## emits — `requestId` is shared across layers — so we just re-export ## `waku/events/message_events` and avoid declaring duplicates. ## diff --git a/channels/reliable_channel.nim b/channels/reliable_channel.nim index e32b57e36..6aa7086e5 100644 --- a/channels/reliable_channel.nim +++ b/channels/reliable_channel.nim @@ -20,8 +20,7 @@ import bearssl/rand import stew/byteutils import libp2p/crypto/crypto as libp2p_crypto -import waku/api/api -import waku/factory/waku as waku_factory +import waku/api/types import waku/node/delivery_service/send_service import waku/waku_core/topics @@ -32,7 +31,7 @@ import ./rate_limit_manager/rate_limit_manager import ./encryption/encryption export - api, waku_factory, events, segmentation, scalable_data_sync, rate_limit_manager, + types, send_service, events, segmentation, scalable_data_sync, rate_limit_manager, encryption const LipWireReliableChannelVersion* = "RELIABLE-CHANNEL-API/1" @@ -47,9 +46,10 @@ type SendHandler* = proc(envelope: MessageEnvelope): Future[Result[RequestId, string]] {. async: (raises: [CatchableError]), gcsafe .} - ## Egress dispatch boundary. Defaults to `waku.send`; tests inject a - ## fake that records calls and returns canned `RequestId`s so the - ## send state machine can be exercised end-to-end without a network. + ## Egress dispatch boundary. Typically wraps `MessagingClient.send`; + ## tests inject a fake that records calls and returns canned + ## `RequestId`s so the send state machine can be exercised end-to-end + ## without a network. MessagePersistence {.pure.} = enum Persistent @@ -264,20 +264,20 @@ proc onReadyToSend( meta: LipWireReliableChannelVersion.toBytes(), ) - ## `waku.send` is not annotated `(raises: [])`, but this listener is. + ## `sendHandler` is not annotated `(raises: [])`, but this listener is. ## Convert any raise to a Result error so the state machine handles ## both failure modes (Result.err and exception) through one path. let sendRes = try: await self.sendHandler(envelope) except CatchableError as e: - Result[RequestId, string].err("waku send raised: " & e.msg) + Result[RequestId, string].err("messaging send raised: " & e.msg) let messagingReqId = sendRes.valueOr: MessageErrorEvent.emit( self.brokerCtx, MessageErrorEvent( - requestId: channelReqId, messageHash: "", error: "waku send failed: " & error + requestId: channelReqId, messageHash: "", error: "messaging send failed: " & error ), ) self.markSegmentFailed(channelReqId) @@ -374,7 +374,7 @@ proc onMessageReceived( proc new*( T: type ReliableChannel, - waku: Waku, + sendHandler: SendHandler, channelId: ChannelId, contentTopic: ContentTopic, senderId: SdsParticipantID, @@ -382,7 +382,6 @@ proc new*( sdsConfig: SdsConfig, rateConfig: RateLimitConfig, brokerCtx: BrokerContext = globalBrokerContext(), - sendHandler: SendHandler = nil, ): T = ## Pipeline handlers (segmentation/SDS/rate-limit) are constructed ## inside the channel rather than handed in by the caller — they are @@ -391,19 +390,11 @@ proc new*( ## `Decrypt` request brokers, so the channel keeps no per-instance ## encryption state either. ## - ## `sendHandler` defaults to `waku.send`; tests pass a fake to drive - ## the send state machine without touching the network. - let resolvedSendHandler = - if sendHandler.isNil(): - proc( - envelope: MessageEnvelope - ): Future[Result[RequestId, string]] {.async: (raises: [CatchableError]), gcsafe.} = - return await waku.send(envelope) - else: - sendHandler - + ## `sendHandler` is the egress dispatch. The owning `ReliableChannelManager` + ## typically constructs it as a closure over `MessagingClient.send`. Tests + ## pass a fake to drive the send state machine without touching the network. let chn = T( - sendHandler: resolvedSendHandler, + sendHandler: sendHandler, channelId: channelId, contentTopic: contentTopic, senderId: senderId, diff --git a/channels/reliable_channel_manager.nim b/channels/reliable_channel_manager.nim index 747f755b4..68ae82388 100644 --- a/channels/reliable_channel_manager.nim +++ b/channels/reliable_channel_manager.nim @@ -10,11 +10,10 @@ import results import chronos import stew/byteutils -import waku/api/api -import waku/api/api_conf +import brokers/broker_context + import waku/events/message_events as waku_message_events -import waku/factory/waku as waku_factory -import waku/node/delivery_service/delivery_service +import waku/messaging_client import waku/waku_core/topics import ./reliable_channel @@ -24,40 +23,43 @@ export reliable_channel type ReliableChannelManager* = ref object channels: Table[ChannelId, ReliableChannel] - waku: Waku - ## Owned by the manager. The channel layer reaches the messaging - ## API through `waku.send(envelope)`; constructing DeliveryTasks - ## directly would breach the layer boundary. + messagingClient: MessagingClient + ## Borrowed from the owning `Waku`. + sendHandler: SendHandler + ## Default egress dispatch for channels created through this manager. + ## Constructed at mount time as a closure over `MessagingClient.send` + ## so the channel layer itself stays callable-only. brokerCtx: BrokerContext proc new*( T: type ReliableChannelManager, - conf: WakuNodeConf, + messagingClient: MessagingClient, + sendHandler: SendHandler, brokerCtx: BrokerContext = globalBrokerContext(), -): Future[Result[T, string]] {.async.} = - ## TODO !! The proper ownership chain is: - ## ReliableChannelManager -> DeliveryService (MessagingClient) -> Waku (Kernel/Protocols) -> WakuNode, - ## and this will be implemented in the future. For now, `createNode` - ## is called here to get a Waku instance, and the WakuNode is immediately discarded. - ## This is a temporary workaround to get the API - - let waku = ?(await createNode(conf)) - - let manager = T( - channels: initTable[ChannelId, ReliableChannel](), waku: waku, brokerCtx: brokerCtx +): Result[T, string] = + if messagingClient.isNil(): + return err("messaging client is required") + if sendHandler.isNil(): + return err("sendHandler is required") + return ok( + T( + channels: initTable[ChannelId, ReliableChannel](), + messagingClient: messagingClient, + sendHandler: sendHandler, + brokerCtx: brokerCtx, + ) ) - return ok(manager) - proc start*(self: ReliableChannelManager): Result[void, string] = - ## Bring the owned DeliveryService up. Separated from `new` so callers - ## can register encryption providers / create channels before traffic - ## starts flowing. - self.waku.deliveryService.startDeliveryService() + ## Placeholder: per-channel listeners are installed in `ReliableChannel.new`, + ## so the manager has nothing to start at this layer. Kept for symmetry + ## with the `Waku` mount/start lifecycle and as a hook for future state. + discard + ok() proc stop*(self: ReliableChannelManager) {.async.} = - if not self.waku.isNil(): - await self.waku.deliveryService.stopDeliveryService() + ## Placeholder mirror of `start`. + discard proc createReliableChannel*( self: ReliableChannelManager, @@ -66,17 +68,17 @@ proc createReliableChannel*( senderId: SdsParticipantID, sendHandler: SendHandler = nil, ): Result[ChannelId, string] = - ## Spec entry point. The `DeliveryService` and `rng` the channel needs - ## are sourced from the owning `ReliableChannelManager` rather than - ## passed per call. Encryption is wired up through the `Encrypt`/ - ## `Decrypt` request brokers — the application installs its own - ## providers (or `setNoopEncryption()`) before traffic flows. + ## Spec entry point. The `sendHandler` and `rng` the channel needs are + ## sourced from the owning `ReliableChannelManager` rather than passed + ## per call. Encryption is wired up through the `Encrypt`/`Decrypt` + ## request brokers — the application installs its own providers + ## (or `setNoopEncryption()`) before traffic flows. ## ## Segmentation, SDS and rate-limit configs will eventually be read ## from the node's `NodeConfig`. Defaults for now. ## - ## `sendHandler` is left `nil` in production so the channel uses the - ## owned `waku.send`; tests pass a fake to bypass the network. + ## `sendHandler` defaults to the manager's default (constructed at mount + ## from `MessagingClient.send`); tests pass a fake to bypass the network. if self.channels.hasKey(channelId): return err("channel already exists: " & channelId) @@ -95,8 +97,14 @@ proc createReliableChannel*( epochPeriodSec: DefaultEpochPeriodSec, messagesPerEpoch: DefaultMessagesPerEpoch ) + let effectiveSendHandler = + if sendHandler.isNil(): + self.sendHandler + else: + sendHandler + let chn = ReliableChannel.new( - waku = self.waku, + sendHandler = effectiveSendHandler, channelId = channelId, contentTopic = contentTopic, senderId = senderId, @@ -104,7 +112,6 @@ proc createReliableChannel*( sdsConfig = sdsConfig, rateConfig = rateConfig, brokerCtx = self.brokerCtx, - sendHandler = sendHandler, ) self.channels[channelId] = chn @@ -137,5 +144,5 @@ proc send*( ## `ReliableChannel` installs its own `MessageReceivedEvent` listener ## in `ReliableChannel.new`, filters by spec marker and `contentTopic`, ## and routes to its private `onMessageReceived`. This keeps the lower -## layer (MessagingAPI/Waku) unaware of the existence of ReliableChannel +## layer (MessagingClient/Waku) unaware of the existence of ReliableChannel ## and keeps the manager out of per-channel event dispatch. diff --git a/examples/api_example/api_example.nim b/examples/api_example/api_example.nim index 2093a81c0..207e83429 100644 --- a/examples/api_example/api_example.nim +++ b/examples/api_example/api_example.nim @@ -82,8 +82,12 @@ when isMainModule: echo("Waku node created successfully!") + node.mountMessagingClient().isOkOr: + echo "Failed to mount messaging: ", error + quit(QuitFailure) + # Start the node - (waitFor startWaku(addr node)).isOkOr: + (waitFor node.start()).isOkOr: echo "Failed to start node: ", error quit(QuitFailure) diff --git a/examples/wakustealthcommitments/node_spec.nim b/examples/wakustealthcommitments/node_spec.nim index d85e83a5b..7751878ca 100644 --- a/examples/wakustealthcommitments/node_spec.nim +++ b/examples/wakustealthcommitments/node_spec.nim @@ -48,7 +48,7 @@ proc setup*(): Waku = error "Waku initialization failed", error = error quit(QuitFailure) - (waitFor startWaku(addr waku)).isOkOr: + (waitFor waku.start()).isOkOr: error "Starting waku failed", error = error quit(QuitFailure) diff --git a/liblogosdelivery/logos_delivery_api/node_api.nim b/liblogosdelivery/logos_delivery_api/node_api.nim index 2e30d1b43..042bdb3a8 100644 --- a/liblogosdelivery/logos_delivery_api/node_api.nim +++ b/liblogosdelivery/logos_delivery_api/node_api.nim @@ -172,7 +172,17 @@ proc logosdelivery_start_node( chronicles.error "ConnectionStatusChange.listen failed", err = $error return err("ConnectionStatusChange.listen failed: " & $error) - (await startWaku(addr ctx.myLib[])).isOkOr: + ctx.myLib[].mountMessagingClient().isOkOr: + let errMsg = $error + chronicles.error "mountMessagingClient failed", error = errMsg + return err("failed to mount messaging: " & errMsg) + + ctx.myLib[].mountReliableChannelManager().isOkOr: + let errMsg = $error + chronicles.error "mountReliableChannelManager failed", err = errMsg + return err("failed to mount reliable channel manager: " & errMsg) + + (await ctx.myLib[].start()).isOkOr: let errMsg = $error chronicles.error "START_NODE failed", err = errMsg return err("failed to start: " & errMsg) diff --git a/library/kernel_api/discovery_api.nim b/library/kernel_api/discovery_api.nim index f61b7bad1..882c91686 100644 --- a/library/kernel_api/discovery_api.nim +++ b/library/kernel_api/discovery_api.nim @@ -5,8 +5,7 @@ import waku/discovery/waku_dnsdisc, waku/discovery/waku_discv5, waku/waku_core/peers, - waku/node/waku_node, - waku/node/kernel_api, + waku/waku_node, library/declare_lib proc retrieveBootstrapNodes( diff --git a/library/kernel_api/node_lifecycle_api.nim b/library/kernel_api/node_lifecycle_api.nim index 8f3e99b24..55dd7cd55 100644 --- a/library/kernel_api/node_lifecycle_api.nim +++ b/library/kernel_api/node_lifecycle_api.nim @@ -71,7 +71,7 @@ registerReqFFI(CreateNodeRequest, ctx: ptr FFIContext[Waku]): proc waku_start( ctx: ptr FFIContext[Waku], callback: FFICallBack, userData: pointer ) {.ffi.} = - (await startWaku(ctx[].myLib)).isOkOr: + (await ctx.myLib[].start()).isOkOr: error "START_NODE failed", error = error return err("failed to start: " & $error) return ok("") diff --git a/library/kernel_api/protocols/filter_api.nim b/library/kernel_api/protocols/filter_api.nim index c4f99510a..866b70ca2 100644 --- a/library/kernel_api/protocols/filter_api.nim +++ b/library/kernel_api/protocols/filter_api.nim @@ -8,8 +8,7 @@ import waku/waku_filter_v2/common, waku/waku_core/subscription/push_handler, waku/node/peer_manager/peer_manager, - waku/node/waku_node, - waku/node/kernel_api, + waku/waku_node, waku/waku_core/topics/pubsub_topic, waku/waku_core/topics/content_topic, library/events/json_message_event, diff --git a/library/kernel_api/protocols/relay_api.nim b/library/kernel_api/protocols/relay_api.nim index b184d6011..4364a4170 100644 --- a/library/kernel_api/protocols/relay_api.nim +++ b/library/kernel_api/protocols/relay_api.nim @@ -7,7 +7,7 @@ import waku/waku_core/message, waku/waku_core/topics/pubsub_topic, waku/waku_core/topics, - waku/node/kernel_api/relay, + waku/node/waku_node/relay, waku/waku_relay/protocol, waku/node/peer_manager, library/events/json_message_event, diff --git a/nimble.lock b/nimble.lock index cd533001e..4bdb8bb82 100644 --- a/nimble.lock +++ b/nimble.lock @@ -328,8 +328,8 @@ } }, "brokers": { - "version": "#v2.0.1", - "vcsRevision": "2093ca4d50e581adda73fee7fd16231f990f4cbe", + "version": "#v3.1.1", + "vcsRevision": "a7316a35f1b62e3497ae8ee0fc1aace74df0beb2", "url": "https://github.com/NagyZoltanPeter/nim-brokers.git", "downloadMethod": "git", "dependencies": [ @@ -341,7 +341,7 @@ "cbor_serialization" ], "checksums": { - "sha1": "cc74c987af94537e9d44d1b0143aa417299040c5" + "sha1": "4447d7c1f9da14ae439afb23aee45116ce2ecb40" } }, "stint": { @@ -620,8 +620,8 @@ } }, "sds": { - "version": "#2e9a7683f0e180bf112135fae3a3803eed8490d4", - "vcsRevision": "2e9a7683f0e180bf112135fae3a3803eed8490d4", + "version": "#abdd40cc645f1b024c3ee99cced7e287c4e4c441", + "vcsRevision": "abdd40cc645f1b024c3ee99cced7e287c4e4c441", "url": "https://github.com/logos-messaging/nim-sds.git", "downloadMethod": "git", "dependencies": [ @@ -636,7 +636,7 @@ "taskpools" ], "checksums": { - "sha1": "d13f1bf8d1b90b27e9edfc063b043831242cda19" + "sha1": "61c4ae13c6896bfa70e662520e8660a78c7f438c" } }, "ffi": { diff --git a/nix/default.nix b/nix/default.nix index ec9e0542c..dfe537f24 100644 --- a/nix/default.nix +++ b/nix/default.nix @@ -30,7 +30,7 @@ let # while others use the repo root. Pass both so the compiler finds either layout. pathArgs = builtins.concatStringsSep " " - (builtins.concatMap (p: [ "--path:${p}" "--path:${p}/src" ]) + (builtins.concatMap (p: [ "--path:${p}" "--path:${p}/src" "--path:${p}/sds" ]) (builtins.attrValues otherDeps)); libExt = diff --git a/tests/api/test_api_health.nim b/tests/api/test_api_health.nim index d949db24f..62fe39b9e 100644 --- a/tests/api/test_api_health.nim +++ b/tests/api/test_api_health.nim @@ -103,7 +103,9 @@ suite "LM API health checking": client = (await createNode(conf)).valueOr: raiseAssert error - (await startWaku(addr client)).isOkOr: + client.mountMessagingClient().isOkOr: + raiseAssert error + (await client.start()).isOkOr: raiseAssert error asyncTeardown: @@ -281,7 +283,9 @@ suite "LM API health checking": edgeWaku = (await createNode(edgeConf)).valueOr: raiseAssert "Failed to create edge node: " & error - (await startWaku(addr edgeWaku)).isOkOr: + edgeWaku.mountMessagingClient().isOkOr: + raiseAssert "Failed to mount edge messaging: " & error + (await edgeWaku.start()).isOkOr: raiseAssert "Failed to start edge waku: " & error let relayReq = await RequestProtocolHealth.request( diff --git a/tests/api/test_api_receive.nim b/tests/api/test_api_receive.nim index d6aa954a4..85e522afc 100644 --- a/tests/api/test_api_receive.nim +++ b/tests/api/test_api_receive.nim @@ -6,6 +6,7 @@ import libp2p/[peerid, peerinfo, crypto/crypto] import brokers/broker_context import ../testlib/[common, wakucore, wakunode, testasync] import ../waku_archive/archive_utils +import waku/messaging_client import waku, @@ -16,7 +17,6 @@ import waku_relay/protocol, waku_archive, waku_archive/common as archive_common, - node/delivery_service/delivery_service, node/delivery_service/recv_service, ] import waku/factory/waku_conf @@ -147,7 +147,8 @@ suite "Messaging API, Receive Service (store recovery)": subscriber = (await createNode(createApiNodeConf(numShards))).expect( "Failed to create subscriber" ) - (await startWaku(addr subscriber)).expect("Failed to start subscriber") + subscriber.mountMessagingClient().expect("Failed to mount messaging") + (await subscriber.start()).expect("Failed to start subscriber") # publish after the subscriber exists but before it connects to the # store; the message reaches the archive but the subscriber doesn't @@ -185,7 +186,7 @@ suite "Messaging API, Receive Service (store recovery)": await eventManager.teardown() # trigger store check, should recover and deliver via MessageReceivedEvent - await subscriber.deliveryService.recvService.checkStore() + await subscriber.messagingClient.recvService.checkStore() let received = await eventManager.waitForEvents(TestTimeout) check received diff --git a/tests/api/test_api_send.nim b/tests/api/test_api_send.nim index 084119041..679d6a419 100644 --- a/tests/api/test_api_send.nim +++ b/tests/api/test_api_send.nim @@ -241,7 +241,9 @@ suite "Waku API - Send": lockNewGlobalBrokerContext: node = (await createNode(createApiNodeConf())).valueOr: raiseAssert error - (await startWaku(addr node)).isOkOr: + node.mountMessagingClient().isOkOr: + raiseAssert "Failed to mount messaging: " & error + (await node.start()).isOkOr: raiseAssert "Failed to start Waku node: " & error # node is not connected ! @@ -263,7 +265,9 @@ suite "Waku API - Send": lockNewGlobalBrokerContext: node = (await createNode(createApiNodeConf())).valueOr: raiseAssert error - (await startWaku(addr node)).isOkOr: + node.mountMessagingClient().isOkOr: + raiseAssert "Failed to mount messaging: " & error + (await node.start()).isOkOr: raiseAssert "Failed to start Waku node: " & error await node.node.connectToNodes( @@ -297,7 +301,9 @@ suite "Waku API - Send": lockNewGlobalBrokerContext: node = (await createNode(createApiNodeConf())).valueOr: raiseAssert error - (await startWaku(addr node)).isOkOr: + node.mountMessagingClient().isOkOr: + raiseAssert "Failed to mount messaging: " & error + (await node.start()).isOkOr: raiseAssert "Failed to start Waku node: " & error await node.node.connectToNodes(@[relayNode1PeerInfo]) @@ -327,7 +333,9 @@ suite "Waku API - Send": lockNewGlobalBrokerContext: node = (await createNode(createApiNodeConf())).valueOr: raiseAssert error - (await startWaku(addr node)).isOkOr: + node.mountMessagingClient().isOkOr: + raiseAssert "Failed to mount messaging: " & error + (await node.start()).isOkOr: raiseAssert "Failed to start Waku node: " & error await node.node.connectToNodes(@[lightpushNodePeerInfo]) @@ -357,7 +365,9 @@ suite "Waku API - Send": lockNewGlobalBrokerContext: node = (await createNode(createApiNodeConf())).valueOr: raiseAssert error - (await startWaku(addr node)).isOkOr: + node.mountMessagingClient().isOkOr: + raiseAssert "Failed to mount messaging: " & error + (await node.start()).isOkOr: raiseAssert "Failed to start Waku node: " & error await node.node.connectToNodes(@[lightpushNodePeerInfo, storeNodePeerInfo]) @@ -411,7 +421,9 @@ suite "Waku API - Send": lockNewGlobalBrokerContext: node = (await createNode(createApiNodeConf(cli_args.WakuMode.Edge))).valueOr: raiseAssert error - (await startWaku(addr node)).isOkOr: + node.mountMessagingClient().isOkOr: + raiseAssert "Failed to mount messaging: " & error + (await node.start()).isOkOr: raiseAssert "Failed to start Waku node: " & error await node.node.connectToNodes(@[fakeLightpushNodePeerInfo]) diff --git a/tests/api/test_api_subscription.nim b/tests/api/test_api_subscription.nim index 32d4e742f..7cb0e981a 100644 --- a/tests/api/test_api_subscription.nim +++ b/tests/api/test_api_subscription.nim @@ -5,6 +5,7 @@ import chronos, testutils/unittests, stew/byteutils import libp2p/[peerid, peerinfo, multiaddress, crypto/crypto] import brokers/broker_context import ../testlib/[common, wakucore, wakunode, testasync] +import waku/messaging_client import waku, @@ -13,14 +14,15 @@ import waku_core, events/message_events, waku_relay/protocol, - node/kernel_api/filter, - node/delivery_service/subscription_manager, + node/waku_node/filter, + node/subscription_manager, ] import waku/factory/waku_conf import tools/confutils/cli_args const TestTimeout = chronos.seconds(10) const NegativeTestTimeout = chronos.seconds(2) +const EdgeWaitTimeout = chronos.seconds(60) type ReceiveEventListenerManager = ref object brokerCtx: BrokerContext @@ -85,7 +87,8 @@ proc setupSubscriberNode(conf: WakuNodeConf): Future[Waku] {.async.} = var node: Waku lockNewGlobalBrokerContext: node = (await createNode(conf)).expect("Failed to create subscriber node") - (await startWaku(addr node)).expect("Failed to start subscriber node") + node.mountMessagingClient().expect("Failed to mount messaging") + (await node.start()).expect("Failed to start subscriber node") return node proc setupNetwork( @@ -161,20 +164,39 @@ proc getRelayShard(node: WakuNode, contentTopic: ContentTopic): PubsubTopic = return PubsubTopic($shardObj) proc waitForMesh(node: WakuNode, shard: PubsubTopic) {.async.} = - for _ in 0 ..< 50: + let deadline = Moment.now() + EdgeWaitTimeout + while Moment.now() < deadline: if node.wakuRelay.getNumPeersInMesh(shard).valueOr(0) > 0: return await sleepAsync(100.milliseconds) raise newException(ValueError, "GossipSub Mesh failed to stabilize on " & shard) proc waitForEdgeSubs(w: Waku, shard: PubsubTopic) {.async.} = - let sm = w.deliveryService.subscriptionManager - for _ in 0 ..< 50: - if sm.edgeFilterPeerCount(shard) > 0: + let deadline = Moment.now() + EdgeWaitTimeout + while Moment.now() < deadline: + if w.node.subscriptionManager.edgeFilterPeerCount(shard) > 0: return await sleepAsync(100.milliseconds) raise newException(ValueError, "Edge filter subscription failed on " & shard) +proc edgePeersReached(w: Waku, shard: PubsubTopic, n: int): Future[bool] {.async.} = + let deadline = Moment.now() + EdgeWaitTimeout + while Moment.now() < deadline: + if w.node.subscriptionManager.edgeFilterPeerCount(shard) >= n: + return true + await sleepAsync(100.milliseconds) + return false + +proc edgePeersDroppedBelow( + w: Waku, shard: PubsubTopic, n: int +): Future[bool] {.async.} = + let deadline = Moment.now() + EdgeWaitTimeout + while Moment.now() < deadline: + if w.node.subscriptionManager.edgeFilterPeerCount(shard) < n: + return true + await sleepAsync(100.milliseconds) + return false + proc publishToMesh( net: TestNetwork, contentTopic: ContentTopic, payload: seq[byte] ): Future[Result[int, string]] {.async.} = @@ -621,7 +643,8 @@ suite "Messaging API, SubscriptionManager": var subscriber: Waku lockNewGlobalBrokerContext: subscriber = (await createNode(conf)).expect("Failed to create edge subscriber") - (await startWaku(addr subscriber)).expect("Failed to start edge subscriber") + subscriber.mountMessagingClient().expect("Failed to mount messaging") + (await subscriber.start()).expect("Failed to start edge subscriber") # Connect edge subscriber to both filter servers so selectPeers finds both await subscriber.node.connectToNodes(@[publisherPeerInfo, meshBuddyPeerInfo]) @@ -632,12 +655,7 @@ suite "Messaging API, SubscriptionManager": (await subscriber.subscribe(testTopic)).expect("Failed to subscribe") # Wait for dialing both filter servers (HealthyThreshold = 2) - for _ in 0 ..< 100: - if subscriber.deliveryService.subscriptionManager.edgeFilterPeerCount(shard) >= 2: - break - await sleepAsync(100.milliseconds) - - check subscriber.deliveryService.subscriptionManager.edgeFilterPeerCount(shard) >= 2 + check await edgePeersReached(subscriber, shard, 2) # Verify message delivery with both servers alive await waitForMesh(publisher, shard) @@ -659,12 +677,8 @@ suite "Messaging API, SubscriptionManager": await subscriber.node.disconnectNode(meshBuddyPeerInfo) # Wait for the dead peer to be pruned - for _ in 0 ..< 50: - if subscriber.deliveryService.subscriptionManager.edgeFilterPeerCount(shard) < 2: - break - await sleepAsync(100.milliseconds) - - check subscriber.deliveryService.subscriptionManager.edgeFilterPeerCount(shard) >= 1 + check await edgePeersDroppedBelow(subscriber, shard, 2) + check subscriber.node.subscriptionManager.edgeFilterPeerCount(shard) >= 1 # Verify messages still arrive through the surviving filter server (publisher) eventManager = newReceiveEventListenerManager(subscriber.brokerCtx, 1) @@ -758,7 +772,8 @@ suite "Messaging API, SubscriptionManager": var subscriber: Waku lockNewGlobalBrokerContext: subscriber = (await createNode(conf)).expect("Failed to create edge subscriber") - (await startWaku(addr subscriber)).expect("Failed to start edge subscriber") + subscriber.mountMessagingClient().expect("Failed to mount messaging") + (await subscriber.start()).expect("Failed to start edge subscriber") await subscriber.node.connectToNodes( @[publisherPeerInfo, meshBuddyPeerInfo, sparePeerInfo] @@ -770,23 +785,13 @@ suite "Messaging API, SubscriptionManager": (await subscriber.subscribe(testTopic)).expect("Failed to subscribe") # Wait for 2 confirmed peers (HealthyThreshold). The 3rd is available but not dialed. - for _ in 0 ..< 100: - if subscriber.deliveryService.subscriptionManager.edgeFilterPeerCount(shard) >= 2: - break - await sleepAsync(100.milliseconds) - - require subscriber.deliveryService.subscriptionManager.edgeFilterPeerCount(shard) == - 2 + check await edgePeersReached(subscriber, shard, 2) + require subscriber.node.subscriptionManager.edgeFilterPeerCount(shard) == 2 await subscriber.node.disconnectNode(meshBuddyPeerInfo) # Wait for the sub loop to detect the loss and dial a replacement - for _ in 0 ..< 100: - if subscriber.deliveryService.subscriptionManager.edgeFilterPeerCount(shard) >= 2: - break - await sleepAsync(100.milliseconds) - - check subscriber.deliveryService.subscriptionManager.edgeFilterPeerCount(shard) >= 2 + check await edgePeersReached(subscriber, shard, 2) await waitForMesh(publisher, shard) diff --git a/tests/channels/test_reliable_channel_send_receive.nim b/tests/channels/test_reliable_channel_send_receive.nim index 5ea300eb3..dabc4497f 100644 --- a/tests/channels/test_reliable_channel_send_receive.nim +++ b/tests/channels/test_reliable_channel_send_receive.nim @@ -35,7 +35,7 @@ suite "Reliable Channel - ingress": ## Unit test for the receive side of the API: instead of standing ## up two libp2p nodes and a relay mesh, we drive the manager ## directly by emitting a `MessageReceivedEvent` (the exact event - ## the DeliveryService emits when a `WakuMessage` arrives off the + ## the MessagingClient emits when a `WakuMessage` arrives off the ## wire). The manager must: ## - drop traffic missing the Reliable Channel spec marker ## - dispatch the matching channel's `onMessageReceived` @@ -45,13 +45,15 @@ suite "Reliable Channel - ingress": contentTopic = ContentTopic("/reliable-channel/test/proto") let appPayload = "hello reliable channel".toBytes() + var waku: Waku var manager: ReliableChannelManager var brokerCtx: BrokerContext lockNewGlobalBrokerContext: brokerCtx = globalBrokerContext() - manager = (await ReliableChannelManager.new(createApiNodeConf())).expect( - "Failed to create manager" - ) + waku = (await createNode(createApiNodeConf())).expect("createNode") + waku.mountMessagingClient().expect("mountMessagingClient") + waku.mountReliableChannelManager().expect("mountReliableChannelManager") + manager = waku.reliableChannelManager ## Noop encryption providers so the Encrypt/Decrypt brokers have ## something to dispatch to; without this the channel falls back to @@ -95,7 +97,7 @@ suite "Reliable Channel - ingress": if arrived: check received.read() == appPayload - await manager.stop() + (await waku.stop()).expect("stop") asyncTest "manager drops unmarked WakuMessage": ## Mirror of the above: same content topic, but `meta` is empty @@ -105,13 +107,15 @@ suite "Reliable Channel - ingress": contentTopic = ContentTopic("/reliable-channel/test/proto") let appPayload = "foreign payload".toBytes() + var waku: Waku var manager: ReliableChannelManager var brokerCtx: BrokerContext lockNewGlobalBrokerContext: brokerCtx = globalBrokerContext() - manager = (await ReliableChannelManager.new(createApiNodeConf())).expect( - "Failed to create manager" - ) + waku = (await createNode(createApiNodeConf())).expect("createNode") + waku.mountMessagingClient().expect("mountMessagingClient") + waku.mountReliableChannelManager().expect("mountReliableChannelManager") + manager = waku.reliableChannelManager setNoopEncryption() @@ -146,7 +150,7 @@ suite "Reliable Channel - ingress": await sleepAsync(100.milliseconds) check not fired - await manager.stop() + (await waku.stop()).expect("stop") suite "Reliable Channel - send state machine": asyncTest "MessageSentEvent finalises the channelReqId as Sent": @@ -162,13 +166,15 @@ suite "Reliable Channel - send state machine": contentTopic = ContentTopic("/reliable-channel/test/sm-success") fakeMsgReqId = RequestId("fake-msg-req-1") + var waku: Waku var manager: ReliableChannelManager var brokerCtx: BrokerContext lockNewGlobalBrokerContext: brokerCtx = globalBrokerContext() - manager = (await ReliableChannelManager.new(createApiNodeConf())).expect( - "Failed to create manager" - ) + waku = (await createNode(createApiNodeConf())).expect("createNode") + waku.mountMessagingClient().expect("mountMessagingClient") + waku.mountReliableChannelManager().expect("mountReliableChannelManager") + manager = waku.reliableChannelManager setNoopEncryption() @@ -213,7 +219,7 @@ suite "Reliable Channel - send state machine": if finalised: check sentFut.read() == channelReqId - await manager.stop() + (await waku.stop()).expect("stop") asyncTest "two independent channelReqIds are finalised independently": ## Two `send()` calls -> two independent `channelReqId`s, each with @@ -227,13 +233,15 @@ suite "Reliable Channel - send state machine": channelId = ChannelId("sm-multi-channel") contentTopic = ContentTopic("/reliable-channel/test/sm-multi") + var waku: Waku var manager: ReliableChannelManager var brokerCtx: BrokerContext lockNewGlobalBrokerContext: brokerCtx = globalBrokerContext() - manager = (await ReliableChannelManager.new(createApiNodeConf())).expect( - "Failed to create manager" - ) + waku = (await createNode(createApiNodeConf())).expect("createNode") + waku.mountMessagingClient().expect("mountMessagingClient") + waku.mountReliableChannelManager().expect("mountReliableChannelManager") + manager = waku.reliableChannelManager setNoopEncryption() @@ -303,7 +311,7 @@ suite "Reliable Channel - send state machine": if erroredArrived: check erroredFut.read() == channelReqId2 - await manager.stop() + (await waku.stop()).expect("stop") asyncTest "TODO: channelReqId not pruned until ALL its segments are final": ## Placeholder for the multi-sibling prune rule. Today's @@ -326,13 +334,15 @@ suite "Reliable Channel - send state machine": channelId = ChannelId("sm-race-channel") contentTopic = ContentTopic("/reliable-channel/test/sm-race") + var waku: Waku var manager: ReliableChannelManager var brokerCtx: BrokerContext lockNewGlobalBrokerContext: brokerCtx = globalBrokerContext() - manager = (await ReliableChannelManager.new(createApiNodeConf())).expect( - "Failed to create manager" - ) + waku = (await createNode(createApiNodeConf())).expect("createNode") + waku.mountMessagingClient().expect("mountMessagingClient") + waku.mountReliableChannelManager().expect("mountReliableChannelManager") + manager = waku.reliableChannelManager setNoopEncryption() @@ -413,4 +423,4 @@ suite "Reliable Channel - send state machine": check channelReqId1 in finalisedReqIds check channelReqId2 in finalisedReqIds - await manager.stop() + (await waku.stop()).expect("stop") diff --git a/tests/factory/test_node_factory.nim b/tests/factory/test_node_factory.nim index 1fe242532..63dd730c2 100644 --- a/tests/factory/test_node_factory.nim +++ b/tests/factory/test_node_factory.nim @@ -11,7 +11,14 @@ import import tests/testlib/[wakunode, wakucore], - waku/[waku_node, waku_enr, net/auto_port, discovery/waku_discv5, node/waku_metrics], + waku/[ + waku_node, + net/net_config, + waku_enr, + net/auto_port, + discovery/waku_discv5, + node/waku_metrics, + ], waku/factory/[ node_factory, internal_config, diff --git a/tests/node/test_wakunode_filter.nim b/tests/node/test_wakunode_filter.nim index 2777b0124..b0dbaa198 100644 --- a/tests/node/test_wakunode_filter.nim +++ b/tests/node/test_wakunode_filter.nim @@ -11,8 +11,7 @@ import waku/[ waku_core, node/peer_manager, - node/waku_node, - node/kernel_api, + waku_node, waku_filter_v2, waku_filter_v2/client, waku_filter_v2/subscriptions, diff --git a/tests/node/test_wakunode_health_monitor.nim b/tests/node/test_wakunode_health_monitor.nim index 08f641a75..be779c586 100644 --- a/tests/node/test_wakunode_health_monitor.nim +++ b/tests/node/test_wakunode_health_monitor.nim @@ -15,18 +15,18 @@ import node/health_monitor/protocol_health, node/health_monitor/topic_health, node/health_monitor/node_health_monitor, - node/delivery_service/delivery_service, - node/delivery_service/subscription_manager, - node/kernel_api/relay, - node/kernel_api/store, - node/kernel_api/lightpush, - node/kernel_api/filter, + messaging_client, + node/waku_node/relay, + node/waku_node/store, + node/waku_node/lightpush, + node/waku_node/filter, events/health_events, events/peer_events, waku_archive, ] import ../testlib/[wakunode, wakucore], ../waku_archive/archive_utils +import waku/node/subscription_manager const MockDLow = 4 # Mocked GossipSub DLow value @@ -229,8 +229,8 @@ suite "Health Monitor - events": await nodeA.start() let ds = - DeliveryService.new(false, nodeA).expect("Failed to create DeliveryService") - ds.startDeliveryService().expect("Failed to start DeliveryService") + MessagingClient.new(false, nodeA).expect("Failed to create MessagingClient") + ds.start().expect("Failed to start MessagingClient") let monitorA = NodeHealthMonitor.new(nodeA) @@ -317,7 +317,7 @@ suite "Health Monitor - events": lastStatus == ConnectionStatus.Disconnected await monitorA.stopHealthMonitor() - await ds.stopDeliveryService() + await ds.stop() await nodeA.stop() asyncTest "Edge health driven by confirmed filter subscriptions": @@ -333,9 +333,9 @@ suite "Health Monitor - events": await nodeA.start() let ds = - DeliveryService.new(false, nodeA).expect("Failed to create DeliveryService") - ds.startDeliveryService().expect("Failed to start DeliveryService") - let subMgr = ds.subscriptionManager + MessagingClient.new(false, nodeA).expect("Failed to create MessagingClient") + ds.start().expect("Failed to start MessagingClient") + let subMgr = nodeA.subscriptionManager var nodeB: WakuNode lockNewGlobalBrokerContext: @@ -416,7 +416,7 @@ suite "Health Monitor - events": await EventShardTopicHealthChange.dropListener(nodeA.brokerCtx, shardHealthLis) check shardHealthOk == true - check subMgr.edgeFilterSubStates.len > 0 + check nodeA.subscriptionManager.edgeFilterSubStates.len > 0 healthSignal.clear() deadline = Moment.now() + TestConnectivityTimeLimit @@ -428,7 +428,7 @@ suite "Health Monitor - events": check lastStatus == ConnectionStatus.PartiallyConnected - await ds.stopDeliveryService() + await ds.stop() await monitorA.stopHealthMonitor() await nodeB.stop() await nodeA.stop() diff --git a/tests/node/test_wakunode_legacy_lightpush.nim b/tests/node/test_wakunode_legacy_lightpush.nim index 68c6cacde..cdd29b398 100644 --- a/tests/node/test_wakunode_legacy_lightpush.nim +++ b/tests/node/test_wakunode_legacy_lightpush.nim @@ -11,9 +11,7 @@ import waku/[ waku_core, node/peer_manager, - node/waku_node, - node/kernel_api, - node/kernel_api/lightpush, + waku_node, waku_lightpush_legacy, waku_lightpush_legacy/common, waku_lightpush_legacy/protocol_metrics, diff --git a/tests/node/test_wakunode_lightpush.nim b/tests/node/test_wakunode_lightpush.nim index b407327e3..4f5476701 100644 --- a/tests/node/test_wakunode_lightpush.nim +++ b/tests/node/test_wakunode_lightpush.nim @@ -8,15 +8,7 @@ import libp2p/crypto/crypto import - waku/[ - waku_core, - node/peer_manager, - node/waku_node, - node/kernel_api, - node/kernel_api/lightpush, - waku_lightpush, - waku_rln_relay, - ], + waku/[waku_core, node/peer_manager, waku_node, waku_lightpush, waku_rln_relay], ../testlib/[wakucore, wakunode, testasync, futures], ../resources/payloads, ../waku_rln_relay/[rln/waku_rln_relay_utils, utils_onchain] diff --git a/tests/node/test_wakunode_peer_exchange.nim b/tests/node/test_wakunode_peer_exchange.nim index e6649c455..ac263c92f 100644 --- a/tests/node/test_wakunode_peer_exchange.nim +++ b/tests/node/test_wakunode_peer_exchange.nim @@ -9,17 +9,12 @@ import libp2p/peerId, libp2p/crypto/crypto, eth/keys, - eth/p2p/discoveryv5/enr + eth/p2p/discoveryv5/enr, + brokers/broker_context import - waku/[ - waku_node, - node/kernel_api, - discovery/waku_discv5, - waku_peer_exchange, - node/peer_manager, - waku_core, - ], + waku/ + [waku_node, discovery/waku_discv5, waku_peer_exchange, node/peer_manager, waku_core], ../waku_peer_exchange/utils, ../testlib/[wakucore, wakunode, testasync] @@ -184,114 +179,115 @@ suite "Waku Peer Exchange": suite "Waku Peer Exchange with discv5": asyncTest "Node successfully exchanges px peers with real discv5": - ## Given (copied from test_waku_discv5.nim) - let - # todo: px flag - flags = CapabilitiesBitfield.init( - lightpush = false, filter = false, store = false, relay = true - ) - bindIp = parseIpAddress("0.0.0.0") - extIp = parseIpAddress("127.0.0.1") + lockNewGlobalBrokerContext: + ## Given (copied from test_waku_discv5.nim) + let + # todo: px flag + flags = CapabilitiesBitfield.init( + lightpush = false, filter = false, store = false, relay = true + ) + bindIp = parseIpAddress("0.0.0.0") + extIp = parseIpAddress("127.0.0.1") - nodeKey1 = generateSecp256k1Key() - nodeTcpPort1 = Port(64010) - nodeUdpPort1 = Port(9000) - node1 = newTestWakuNode( - nodeKey1, - bindIp, - nodeTcpPort1, - some(extIp), - wakuFlags = some(flags), - discv5UdpPort = some(nodeUdpPort1), + nodeKey1 = generateSecp256k1Key() + nodeTcpPort1 = Port(64010) + nodeUdpPort1 = Port(9000) + node1 = newTestWakuNode( + nodeKey1, + bindIp, + nodeTcpPort1, + some(extIp), + wakuFlags = some(flags), + discv5UdpPort = some(nodeUdpPort1), + ) + + nodeKey2 = generateSecp256k1Key() + nodeTcpPort2 = Port(64012) + nodeUdpPort2 = Port(9002) + node2 = newTestWakuNode( + nodeKey2, + bindIp, + nodeTcpPort2, + some(extIp), + wakuFlags = some(flags), + discv5UdpPort = some(nodeUdpPort2), + ) + + nodeKey3 = generateSecp256k1Key() + nodeTcpPort3 = Port(64014) + nodeUdpPort3 = Port(9004) + node3 = newTestWakuNode( + nodeKey3, + bindIp, + nodeTcpPort3, + some(extIp), + wakuFlags = some(flags), + discv5UdpPort = some(nodeUdpPort3), + ) + + # discv5 + let conf1 = WakuDiscoveryV5Config( + discv5Config: none(DiscoveryConfig), + address: bindIp, + port: nodeUdpPort1, + privateKey: keys.PrivateKey(nodeKey1.skkey), + bootstrapRecords: @[], + autoupdateRecord: true, ) - nodeKey2 = generateSecp256k1Key() - nodeTcpPort2 = Port(64012) - nodeUdpPort2 = Port(9002) - node2 = newTestWakuNode( - nodeKey2, - bindIp, - nodeTcpPort2, - some(extIp), - wakuFlags = some(flags), - discv5UdpPort = some(nodeUdpPort2), + let disc1 = + WakuDiscoveryV5.new(node1.rng, conf1, some(node1.enr), some(node1.peerManager)) + + let conf2 = WakuDiscoveryV5Config( + discv5Config: none(DiscoveryConfig), + address: bindIp, + port: nodeUdpPort2, + privateKey: keys.PrivateKey(nodeKey2.skkey), + bootstrapRecords: @[disc1.protocol.getRecord()], + autoupdateRecord: true, ) - nodeKey3 = generateSecp256k1Key() - nodeTcpPort3 = Port(64014) - nodeUdpPort3 = Port(9004) - node3 = newTestWakuNode( - nodeKey3, - bindIp, - nodeTcpPort3, - some(extIp), - wakuFlags = some(flags), - discv5UdpPort = some(nodeUdpPort3), + let disc2 = + WakuDiscoveryV5.new(node2.rng, conf2, some(node2.enr), some(node2.peerManager)) + + await allFutures(node1.start(), node2.start(), node3.start()) + let resultDisc1StartRes = await disc1.start() + assert resultDisc1StartRes.isOk(), resultDisc1StartRes.error + let resultDisc2StartRes = await disc2.start() + assert resultDisc2StartRes.isOk(), resultDisc2StartRes.error + + ## When + var attempts = 10 + while (disc1.protocol.nodesDiscovered < 1 or disc2.protocol.nodesDiscovered < 1) and + attempts > 0: + await sleepAsync(1.seconds) + attempts -= 1 + + # node2 can be connected, so will be returned by peer exchange + require ( + await node1.peerManager.connectPeer(node2.switch.peerInfo.toRemotePeerInfo()) ) - # discv5 - let conf1 = WakuDiscoveryV5Config( - discv5Config: none(DiscoveryConfig), - address: bindIp, - port: nodeUdpPort1, - privateKey: keys.PrivateKey(nodeKey1.skkey), - bootstrapRecords: @[], - autoupdateRecord: true, - ) + # Mount peer exchange + await node1.mountPeerExchange() + await node3.mountPeerExchange() + await node3.mountPeerExchangeClient() - let disc1 = - WakuDiscoveryV5.new(node1.rng, conf1, some(node1.enr), some(node1.peerManager)) + let dialResponse = + await node3.dialForPeerExchange(node1.switch.peerInfo.toRemotePeerInfo()) - let conf2 = WakuDiscoveryV5Config( - discv5Config: none(DiscoveryConfig), - address: bindIp, - port: nodeUdpPort2, - privateKey: keys.PrivateKey(nodeKey2.skkey), - bootstrapRecords: @[disc1.protocol.getRecord()], - autoupdateRecord: true, - ) + check dialResponse.isOk - let disc2 = - WakuDiscoveryV5.new(node2.rng, conf2, some(node2.enr), some(node2.peerManager)) + let + requestPeers = 1 + currentPeers = node3.peerManager.switch.peerStore.peers.len + let res = await node3.fetchPeerExchangePeers(1) + check res.tryGet() == 1 - await allFutures(node1.start(), node2.start(), node3.start()) - let resultDisc1StartRes = await disc1.start() - assert resultDisc1StartRes.isOk(), resultDisc1StartRes.error - let resultDisc2StartRes = await disc2.start() - assert resultDisc2StartRes.isOk(), resultDisc2StartRes.error + # Then node3 has received 1 peer from node1 + check: + node3.peerManager.switch.peerStore.peers.len == currentPeers + requestPeers - ## When - var attempts = 10 - while (disc1.protocol.nodesDiscovered < 1 or disc2.protocol.nodesDiscovered < 1) and - attempts > 0: - await sleepAsync(1.seconds) - attempts -= 1 - - # node2 can be connected, so will be returned by peer exchange - require ( - await node1.peerManager.connectPeer(node2.switch.peerInfo.toRemotePeerInfo()) - ) - - # Mount peer exchange - await node1.mountPeerExchange() - await node3.mountPeerExchange() - await node3.mountPeerExchangeClient() - - let dialResponse = - await node3.dialForPeerExchange(node1.switch.peerInfo.toRemotePeerInfo()) - - check dialResponse.isOk - - let - requestPeers = 1 - currentPeers = node3.peerManager.switch.peerStore.peers.len - let res = await node3.fetchPeerExchangePeers(1) - check res.tryGet() == 1 - - # Then node3 has received 1 peer from node1 - check: - node3.peerManager.switch.peerStore.peers.len == currentPeers + requestPeers - - await allFutures( - [node1.stop(), node2.stop(), node3.stop(), disc1.stop(), disc2.stop()] - ) + await allFutures( + [node1.stop(), node2.stop(), node3.stop(), disc1.stop(), disc2.stop()] + ) diff --git a/tests/node/test_wakunode_peer_manager.nim b/tests/node/test_wakunode_peer_manager.nim index ed58db7fe..b0c4354cf 100644 --- a/tests/node/test_wakunode_peer_manager.nim +++ b/tests/node/test_wakunode_peer_manager.nim @@ -16,8 +16,7 @@ import waku/[ waku_core, node/peer_manager, - node/waku_node, - node/kernel_api, + waku_node, discovery/waku_discv5, waku_filter_v2/common, waku_relay/protocol, diff --git a/tests/node/test_wakunode_relay_rln.nim b/tests/node/test_wakunode_relay_rln.nim index 3a2a8a67c..b78255ce9 100644 --- a/tests/node/test_wakunode_relay_rln.nim +++ b/tests/node/test_wakunode_relay_rln.nim @@ -13,11 +13,9 @@ from std/times import epochTime import ../../../waku/[ - node/waku_node, node/peer_manager, waku_core, waku_node, - node/kernel_api, common/error_handling, waku_rln_relay, waku_rln_relay/rln, diff --git a/tests/node/test_wakunode_sharding.nim b/tests/node/test_wakunode_sharding.nim index f482b6abc..88bf63efa 100644 --- a/tests/node/test_wakunode_sharding.nim +++ b/tests/node/test_wakunode_sharding.nim @@ -14,8 +14,7 @@ import waku/[ waku_core/topics/pubsub_topic, waku_core/topics/sharding, - node/waku_node, - node/kernel_api, + waku_node, common/paging, waku_core, waku_store/common, diff --git a/tests/node/test_wakunode_store.nim b/tests/node/test_wakunode_store.nim index 01deb2903..daa1db682 100644 --- a/tests/node/test_wakunode_store.nim +++ b/tests/node/test_wakunode_store.nim @@ -5,8 +5,7 @@ import std/[options, sequtils, sets], testutils/unittests, chronos, libp2p/crypt import waku/[ common/paging, - node/waku_node, - node/kernel_api, + waku_node, node/peer_manager, waku_core, waku_core/message/digest, diff --git a/tests/persistency/test_all.nim b/tests/persistency/test_all.nim index 194977692..5b0cfdbb5 100644 --- a/tests/persistency/test_all.nim +++ b/tests/persistency/test_all.nim @@ -5,5 +5,6 @@ import ./test_backend import ./test_lifecycle import ./test_facade import ./test_encoding +import ./test_sds_persistency import ./test_string_lookup import ./test_singleton diff --git a/tests/persistency/test_sds_persistency.nim b/tests/persistency/test_sds_persistency.nim new file mode 100644 index 000000000..ed14f904b --- /dev/null +++ b/tests/persistency/test_sds_persistency.nim @@ -0,0 +1,155 @@ +{.used.} + +## Behavioural tests for the SDS Persistence adapter (nim-sds 0.3.0 snapshot +## model). Importing `sds_persistency` also compile-checks the real adapter. +## +## Writes go through the fire-and-forget Job path (the Future resolves when +## the op is queued, not applied — Persistency v1), so every read-back polls +## until the row appears/disappears. + +import std/[options, os, times] +import chronos, results +import testutils/unittests +import waku/persistency/persistency +import waku/persistency/keys +import waku/persistency/sds_persistency + +proc tmpRoot(label: string): string = + let p = getTempDir() / ("sds_persistency_test_" & label & "_" & $epochTime().int) + removeDir(p) + p + +proc pollExists( + t: Job, category: string, k: Key, timeoutMs = 1000 +): Future[bool] {.async.} = + let deadline = epochTime() + (timeoutMs.float / 1000.0) + while epochTime() < deadline: + let r = await t.exists(category, k) + if r.isOk and r.get(): + return true + await sleepAsync(chronos.milliseconds(2)) + return false + +proc pollGone( + t: Job, category: string, k: Key, timeoutMs = 1000 +): Future[bool] {.async.} = + let deadline = epochTime() + (timeoutMs.float / 1000.0) + while epochTime() < deadline: + let r = await t.exists(category, k) + if r.isOk and not r.get(): + return true + await sleepAsync(chronos.milliseconds(2)) + return false + +proc mkMsg(channelId: SdsChannelID, msgId: SdsMessageID, lamport: int64): SdsMessage = + SdsMessage.init( + messageId = msgId, + lamportTimestamp = lamport, + causalHistory = @[], + channelId = channelId, + content = @[byte(1), byte(2)], + bloomFilter = @[], + ) + +suite "SDS persistency adapter (0.3.0 snapshot model)": + asyncTest "saveChannelMeta + updateHistory round-trip via loadChannel": + let root = tmpRoot("roundtrip") + defer: + removeDir(root) + let p = Persistency.instance(root).get() + defer: + Persistency.reset() + let job = p.openJob("sds").get() + let persistence = newSdsPersistence(job) + let channelId = "chan-1".SdsChannelID + + var meta = ChannelMeta.init() + meta.lamportTimestamp = 42 + check (await persistence.saveChannelMeta(channelId, meta)).isOk + check (await job.pollExists(CatMeta, toKey(channelId))) + + # append out of (lamport) order on purpose; loadChannel must sort. + var upd = HistoryUpdate.init() + upd.append = @[mkMsg(channelId, "m2", 2), mkMsg(channelId, "m1", 1)] + check (await persistence.updateHistory(channelId, upd)).isOk + check (await job.pollExists(CatLog, key(channelId, "m2"))) + + let data = (await persistence.loadChannel(channelId)).valueOr: + check false + return + check data.meta.lamportTimestamp == 42 + check data.messageHistory.len == 2 + check data.messageHistory[0].messageId == "m1" + check data.messageHistory[1].messageId == "m2" + + asyncTest "loadChannel on a fresh channel returns empty ChannelData": + let root = tmpRoot("empty") + defer: + removeDir(root) + let p = Persistency.instance(root).get() + defer: + Persistency.reset() + let job = p.openJob("sds").get() + let persistence = newSdsPersistence(job) + + let data = (await persistence.loadChannel("nope".SdsChannelID)).valueOr: + check false + return + check data.meta.lamportTimestamp == 0 + check data.messageHistory.len == 0 + + asyncTest "updateHistory evict removes a log row": + let root = tmpRoot("evict") + defer: + removeDir(root) + let p = Persistency.instance(root).get() + defer: + Persistency.reset() + let job = p.openJob("sds").get() + let persistence = newSdsPersistence(job) + let channelId = "c".SdsChannelID + + var upd = HistoryUpdate.init() + upd.append = @[mkMsg(channelId, "a", 1), mkMsg(channelId, "b", 2)] + check (await persistence.updateHistory(channelId, upd)).isOk + check (await job.pollExists(CatLog, key(channelId, "b"))) + + var ev = HistoryUpdate.init() + ev.evict = @["a".SdsMessageID] + check (await persistence.updateHistory(channelId, ev)).isOk + check (await job.pollGone(CatLog, key(channelId, "a"))) + + let data = (await persistence.loadChannel(channelId)).valueOr: + check false + return + check data.messageHistory.len == 1 + check data.messageHistory[0].messageId == "b" + + asyncTest "dropChannel wipes meta and log": + let root = tmpRoot("drop") + defer: + removeDir(root) + let p = Persistency.instance(root).get() + defer: + Persistency.reset() + let job = p.openJob("sds").get() + let persistence = newSdsPersistence(job) + let channelId = "d".SdsChannelID + + var meta = ChannelMeta.init() + meta.lamportTimestamp = 7 + check (await persistence.saveChannelMeta(channelId, meta)).isOk + var upd = HistoryUpdate.init() + upd.append = @[mkMsg(channelId, "x", 1)] + check (await persistence.updateHistory(channelId, upd)).isOk + check (await job.pollExists(CatMeta, toKey(channelId))) + check (await job.pollExists(CatLog, key(channelId, "x"))) + + check (await persistence.dropChannel(channelId)).isOk + check (await job.pollGone(CatMeta, toKey(channelId))) + + let data = (await persistence.loadChannel(channelId)).valueOr: + check false + return + check data.meta.lamportTimestamp == 0 + check data.messageHistory.len == 0 diff --git a/tests/test_waku_keepalive.nim b/tests/test_waku_keepalive.nim index 5d8402268..32cfb245d 100644 --- a/tests/test_waku_keepalive.nim +++ b/tests/test_waku_keepalive.nim @@ -9,7 +9,12 @@ import libp2p/stream/bufferstream, libp2p/stream/connection, libp2p/crypto/crypto -import waku/waku_core, waku/waku_node, ./testlib/wakucore, ./testlib/wakunode +import + waku/waku_core, + waku/waku_node, + waku/node/health_monitor, + ./testlib/wakucore, + ./testlib/wakunode suite "Waku Keepalive": asyncTest "handle ping keepalives": diff --git a/tests/testlib/wakunode.nim b/tests/testlib/wakunode.nim index 77c017d96..c23854f08 100644 --- a/tests/testlib/wakunode.nim +++ b/tests/testlib/wakunode.nim @@ -10,6 +10,7 @@ import import waku/[ waku_node, + net/net_config, waku_core/topics, node/peer_manager, waku_enr, diff --git a/tests/waku_discv5/test_waku_discv5.nim b/tests/waku_discv5/test_waku_discv5.nim index 936c01826..58d7d5bb8 100644 --- a/tests/waku_discv5/test_waku_discv5.nim +++ b/tests/waku_discv5/test_waku_discv5.nim @@ -21,8 +21,7 @@ import waku_enr/capabilities, factory/conf_builder/conf_builder, factory/waku, - node/waku_node, - node/kernel_api, + waku_node, node/peer_manager, ], ../testlib/[wakucore, testasync, assertions, futures, wakunode, testutils], @@ -431,7 +430,7 @@ suite "Waku Discovery v5": let waku0 = (await Waku.new(conf)).valueOr: raiseAssert error - (waitFor startWaku(addr waku0)).isOkOr: + (waitFor waku0.start()).isOkOr: raiseAssert error confBuilder.withNodeKey(crypto.PrivateKey.random(Secp256k1, myRng[])[]) @@ -445,7 +444,7 @@ suite "Waku Discovery v5": let waku1 = (await Waku.new(conf1)).valueOr: raiseAssert error - (waitFor startWaku(addr waku1)).isOkOr: + (waitFor waku1.start()).isOkOr: raiseAssert error await waku1.node.mountPeerExchange() @@ -461,7 +460,7 @@ suite "Waku Discovery v5": let waku2 = (await Waku.new(conf2)).valueOr: raiseAssert error - (waitFor startWaku(addr waku2)).isOkOr: + (waitFor waku2.start()).isOkOr: raiseAssert error # leave some time for discv5 to act diff --git a/tests/waku_filter_v2/test_waku_client.nim b/tests/waku_filter_v2/test_waku_client.nim index c57699d39..b34c22018 100644 --- a/tests/waku_filter_v2/test_waku_client.nim +++ b/tests/waku_filter_v2/test_waku_client.nim @@ -3,7 +3,8 @@ import std/[options, sequtils, json], testutils/unittests, results, chronos import - waku/node/[peer_manager, waku_node, kernel_api], + waku/node/peer_manager, + waku/waku_node, waku/waku_core, waku/waku_filter_v2/[common, client, subscriptions, protocol, rpc_codec], ../testlib/[wakucore, testasync, testutils, futures, sequtils, wakunode], diff --git a/tests/waku_peer_exchange/test_protocol.nim b/tests/waku_peer_exchange/test_protocol.nim index 74cdba110..29ec45d1e 100644 --- a/tests/waku_peer_exchange/test_protocol.nim +++ b/tests/waku_peer_exchange/test_protocol.nim @@ -5,7 +5,8 @@ import testutils/unittests, chronos, libp2p/[switch, peerId, crypto/crypto], - eth/[keys, p2p/discoveryv5/enr] + eth/[keys, p2p/discoveryv5/enr], + brokers/broker_context import waku/[ @@ -31,110 +32,113 @@ suite "Waku Peer Exchange": suite "request": asyncTest "Retrieve and provide peer exchange peers from discv5": - ## Given (copied from test_waku_discv5.nim) - let - # todo: px flag - flags = CapabilitiesBitfield.init( - lightpush = false, filter = false, store = false, relay = true - ) - bindIp = parseIpAddress("0.0.0.0") - extIp = parseIpAddress("127.0.0.1") + lockNewGlobalBrokerContext: + ## Given (copied from test_waku_discv5.nim) + let + # todo: px flag + flags = CapabilitiesBitfield.init( + lightpush = false, filter = false, store = false, relay = true + ) + bindIp = parseIpAddress("0.0.0.0") + extIp = parseIpAddress("127.0.0.1") - nodeKey1 = generateSecp256k1Key() - nodeTcpPort1 = Port(64010) - nodeUdpPort1 = Port(9000) - node1 = newTestWakuNode( - nodeKey1, - bindIp, - nodeTcpPort1, - some(extIp), - wakuFlags = some(flags), - discv5UdpPort = some(nodeUdpPort1), + nodeKey1 = generateSecp256k1Key() + nodeTcpPort1 = Port(64010) + nodeUdpPort1 = Port(9000) + node1 = newTestWakuNode( + nodeKey1, + bindIp, + nodeTcpPort1, + some(extIp), + wakuFlags = some(flags), + discv5UdpPort = some(nodeUdpPort1), + ) + + nodeKey2 = generateSecp256k1Key() + nodeTcpPort2 = Port(64012) + nodeUdpPort2 = Port(9002) + node2 = newTestWakuNode( + nodeKey2, + bindIp, + nodeTcpPort2, + some(extIp), + wakuFlags = some(flags), + discv5UdpPort = some(nodeUdpPort2), + ) + + nodeKey3 = generateSecp256k1Key() + nodeTcpPort3 = Port(64014) + nodeUdpPort3 = Port(9004) + node3 = newTestWakuNode( + nodeKey3, + bindIp, + nodeTcpPort3, + some(extIp), + wakuFlags = some(flags), + discv5UdpPort = some(nodeUdpPort3), + ) + + # discv5 + let conf1 = WakuDiscoveryV5Config( + discv5Config: none(DiscoveryConfig), + address: bindIp, + port: nodeUdpPort1, + privateKey: keys.PrivateKey(nodeKey1.skkey), + bootstrapRecords: @[], + autoupdateRecord: true, ) - nodeKey2 = generateSecp256k1Key() - nodeTcpPort2 = Port(64012) - nodeUdpPort2 = Port(9002) - node2 = newTestWakuNode( - nodeKey2, - bindIp, - nodeTcpPort2, - some(extIp), - wakuFlags = some(flags), - discv5UdpPort = some(nodeUdpPort2), + let disc1 = WakuDiscoveryV5.new( + node1.rng, conf1, some(node1.enr), some(node1.peerManager) ) - nodeKey3 = generateSecp256k1Key() - nodeTcpPort3 = Port(64014) - nodeUdpPort3 = Port(9004) - node3 = newTestWakuNode( - nodeKey3, - bindIp, - nodeTcpPort3, - some(extIp), - wakuFlags = some(flags), - discv5UdpPort = some(nodeUdpPort3), + let conf2 = WakuDiscoveryV5Config( + discv5Config: none(DiscoveryConfig), + address: bindIp, + port: nodeUdpPort2, + privateKey: keys.PrivateKey(nodeKey2.skkey), + bootstrapRecords: @[disc1.protocol.getRecord()], + autoupdateRecord: true, ) - # discv5 - let conf1 = WakuDiscoveryV5Config( - discv5Config: none(DiscoveryConfig), - address: bindIp, - port: nodeUdpPort1, - privateKey: keys.PrivateKey(nodeKey1.skkey), - bootstrapRecords: @[], - autoupdateRecord: true, - ) + let disc2 = WakuDiscoveryV5.new( + node2.rng, conf2, some(node2.enr), some(node2.peerManager) + ) - let disc1 = - WakuDiscoveryV5.new(node1.rng, conf1, some(node1.enr), some(node1.peerManager)) + await allFutures(node1.start(), node2.start(), node3.start()) + let resultDisc1StartRes = await disc1.start() + assert resultDisc1StartRes.isOk(), resultDisc1StartRes.error + let resultDisc2StartRes = await disc2.start() + assert resultDisc2StartRes.isOk(), resultDisc2StartRes.error - let conf2 = WakuDiscoveryV5Config( - discv5Config: none(DiscoveryConfig), - address: bindIp, - port: nodeUdpPort2, - privateKey: keys.PrivateKey(nodeKey2.skkey), - bootstrapRecords: @[disc1.protocol.getRecord()], - autoupdateRecord: true, - ) + ## When + var attempts = 10 + while (disc1.protocol.nodesDiscovered < 1 or disc2.protocol.nodesDiscovered < 1) and + attempts > 0: + await sleepAsync(1.seconds) + attempts -= 1 - let disc2 = - WakuDiscoveryV5.new(node2.rng, conf2, some(node2.enr), some(node2.peerManager)) + # node2 can be connected, so will be returned by peer exchange + require ( + await node1.peerManager.connectPeer(node2.switch.peerInfo.toRemotePeerInfo()) + ) - await allFutures(node1.start(), node2.start(), node3.start()) - let resultDisc1StartRes = await disc1.start() - assert resultDisc1StartRes.isOk(), resultDisc1StartRes.error - let resultDisc2StartRes = await disc2.start() - assert resultDisc2StartRes.isOk(), resultDisc2StartRes.error + # Mount peer exchange + await node1.mountPeerExchange() + await node3.mountPeerExchange() - ## When - var attempts = 10 - while (disc1.protocol.nodesDiscovered < 1 or disc2.protocol.nodesDiscovered < 1) and - attempts > 0: - await sleepAsync(1.seconds) - attempts -= 1 + let dialResponse = + await node3.dialForPeerExchange(node1.switch.peerInfo.toRemotePeerInfo()) + let response = dialResponse.get() - # node2 can be connected, so will be returned by peer exchange - require ( - await node1.peerManager.connectPeer(node2.switch.peerInfo.toRemotePeerInfo()) - ) + ## Then + check: + response.get().peerInfos.len == 1 + response.get().peerInfos[0].enr == disc2.protocol.localNode.record.raw - # Mount peer exchange - await node1.mountPeerExchange() - await node3.mountPeerExchange() - - let dialResponse = - await node3.dialForPeerExchange(node1.switch.peerInfo.toRemotePeerInfo()) - let response = dialResponse.get() - - ## Then - check: - response.get().peerInfos.len == 1 - response.get().peerInfos[0].enr == disc2.protocol.localNode.record.raw - - await allFutures( - [node1.stop(), node2.stop(), node3.stop(), disc1.stop(), disc2.stop()] - ) + await allFutures( + [node1.stop(), node2.stop(), node3.stop(), disc1.stop(), disc2.stop()] + ) asyncTest "Request returns some discovered peers": let diff --git a/tests/wakunode2/test_app.nim b/tests/wakunode2/test_app.nim index 7621ab1e7..8dc9e3582 100644 --- a/tests/wakunode2/test_app.nim +++ b/tests/wakunode2/test_app.nim @@ -46,7 +46,7 @@ suite "Wakunode2 - Waku initialization": var waku = (waitFor Waku.new(conf)).valueOr: raiseAssert error - (waitFor startWaku(addr waku)).isOkOr: + (waitFor waku.start()).isOkOr: raiseAssert error ## Then @@ -71,7 +71,7 @@ suite "Wakunode2 - Waku initialization": var waku = (waitFor Waku.new(conf)).valueOr: raiseAssert error - (waitFor startWaku(addr waku)).isOkOr: + (waitFor waku.start()).isOkOr: raiseAssert error ## Then @@ -128,7 +128,7 @@ suite "Wakunode2 - Waku initialization": (waitFor waku.stop()).isOkOr: raiseAssert error - (waitFor startWaku(addr waku)).isOkOr: + (waitFor waku.start()).isOkOr: raiseAssert error let portsJson = waku.stateInfo.getNodeInfoItem(NodeInfoId.MyBoundPorts) diff --git a/waku.nimble b/waku.nimble index 2b3c6ef7b..38548b5dc 100644 --- a/waku.nimble +++ b/waku.nimble @@ -59,19 +59,12 @@ requires "nim >= 2.2.4", "unittest2" # Packages not on nimble (use git URLs) + requires "https://github.com/logos-messaging/nim-ffi#v0.1.3" -requires "https://github.com/logos-messaging/nim-sds.git#2e9a7683f0e180bf112135fae3a3803eed8490d4" +requires "https://github.com/logos-messaging/nim-sds.git#abdd40cc645f1b024c3ee99cced7e287c4e4c441" -# brokers: pinned by URL+commit rather than the bare `brokers >= 2.0.1` -# form because the nim-lang/packages registry entry for `brokers` only -# carries metadata for the original v0.1.0 publication. Until that -# registry entry is refreshed, the local SAT solver enumerates "0.1.0" -# as the only available version and cannot satisfy `>= 2.0.1`. The URL -# pin below bypasses the registry and locks the exact commit of the -# v2.0.1 tag. Revert to the bare form once nim-lang/packages is -# updated. -requires "https://github.com/NagyZoltanPeter/nim-brokers.git#v2.0.1" +requires "https://github.com/NagyZoltanPeter/nim-brokers.git#v3.1.1" requires "https://github.com/vacp2p/nim-lsquic" requires "https://github.com/vacp2p/nim-jwt.git#057ec95eb5af0eea9c49bfe9025b3312c95dc5f2" diff --git a/waku/api/api.nim b/waku/api/api.nim index 1eee982fd..24049002b 100644 --- a/waku/api/api.nim +++ b/waku/api/api.nim @@ -1,9 +1,10 @@ import chronicles, chronos, results import waku/factory/waku +import waku/messaging_client import waku/[requests/health_requests, waku_core, waku_node] import waku/node/delivery_service/send_service -import waku/node/delivery_service/subscription_manager +import waku/node/subscription_manager import libp2p/peerid import ../../tools/confutils/cli_args import ./[api_conf, types] @@ -38,39 +39,15 @@ proc subscribe*( ): Future[Result[void, string]] {.async.} = ?checkApiAvailability(w) - return w.deliveryService.subscriptionManager.subscribe(contentTopic) + return w.node.subscriptionManager.subscribe(contentTopic) proc unsubscribe*(w: Waku, contentTopic: ContentTopic): Result[void, string] = ?checkApiAvailability(w) - return w.deliveryService.subscriptionManager.unsubscribe(contentTopic) + return w.node.subscriptionManager.unsubscribe(contentTopic) proc send*( w: Waku, envelope: MessageEnvelope ): Future[Result[RequestId, string]] {.async.} = ?checkApiAvailability(w) - - let isSubbed = w.deliveryService.subscriptionManager - .isSubscribed(envelope.contentTopic) - .valueOr(false) - if not isSubbed: - info "Auto-subscribing to topic on send", contentTopic = envelope.contentTopic - w.deliveryService.subscriptionManager.subscribe(envelope.contentTopic).isOkOr: - warn "Failed to auto-subscribe", error = error - return err("Failed to auto-subscribe before sending: " & error) - - let requestId = RequestId.new(w.rng) - - let deliveryTask = DeliveryTask.new(requestId, envelope, w.brokerCtx).valueOr: - return err("API send: Failed to create delivery task: " & error) - - info "API send: scheduling delivery task", - requestId = $requestId, - pubsubTopic = deliveryTask.pubsubTopic, - contentTopic = deliveryTask.msg.contentTopic, - msgHash = deliveryTask.msgHash.to0xHex(), - myPeerId = w.node.peerId() - - asyncSpawn w.deliveryService.sendService.send(deliveryTask) - - return ok(requestId) + return await w.messagingClient.send(envelope) diff --git a/waku/factory/builder.nim b/waku/factory/builder.nim index 4212cb92d..953d95a34 100644 --- a/waku/factory/builder.nim +++ b/waku/factory/builder.nim @@ -15,6 +15,7 @@ import ../waku_enr, ../discovery/waku_discv5, ../waku_node, + ../net/net_config, ../node/peer_manager, ../common/rate_limit/setting, ../common/utils/parse_size_units diff --git a/waku/factory/node_factory.nim b/waku/factory/node_factory.nim index 52b719b8f..2a2b6e5d9 100644 --- a/waku/factory/node_factory.nim +++ b/waku/factory/node_factory.nim @@ -17,6 +17,7 @@ import ./validator_signed, ../waku_enr/sharding, ../waku_node, + ../net/net_config, ../waku_core, ../waku_core/codecs, ../waku_rln_relay, diff --git a/waku/factory/waku.nim b/waku/factory/waku.nim index 6a5567f8c..edd02ade8 100644 --- a/waku/factory/waku.nim +++ b/waku/factory/waku.nim @@ -30,12 +30,13 @@ import waku_enr/sharding, waku_enr/multiaddr, api/types, + messaging_client, common/logging, node/peer_manager, node/health_monitor, + net/net_config, node/waku_metrics, - node/delivery_service/delivery_service, - node/delivery_service/subscription_manager, + node/subscription_manager, rest_api/message_cache, rest_api/endpoint/server, rest_api/endpoint/builder as rest_server_builder, @@ -48,6 +49,7 @@ import factory/app_callbacks, persistency/persistency, ], + channels/reliable_channel_manager, ./waku_conf, ./waku_state_info @@ -73,7 +75,9 @@ type Waku* = ref object healthMonitor*: NodeHealthMonitor - deliveryService*: DeliveryService + messagingClient*: MessagingClient + + reliableChannelManager*: ReliableChannelManager restServer*: WakuRestServerRef metricsServer*: MetricsHttpServerRef @@ -215,10 +219,6 @@ proc new*( error "Failed setting up app callbacks", error = error return err("Failed setting up app callbacks: " & $error) - ## Delivery Monitor - let deliveryService = DeliveryService.new(wakuConf.p2pReliability, node).valueOr: - return err("could not create delivery service: " & $error) - var waku = Waku( stateInfo: WakuStateInfo.init(node), conf: wakuConf, @@ -226,7 +226,6 @@ proc new*( key: wakuConf.nodeKey, node: node, healthMonitor: healthMonitor, - deliveryService: deliveryService, appCallbacks: appCallbacks, restServer: restServer, brokerCtx: brokerCtx, @@ -254,9 +253,9 @@ proc getPorts( return ok((tcpPort: tcpPort, websocketPort: websocketPort)) -proc getRunningNetConfig(waku: ptr Waku): Future[Result[NetConfig, string]] {.async.} = - let conf = waku[].conf - let (tcpPort, websocketPort) = getPorts(waku[].node.switch.peerInfo.listenAddrs).valueOr: +proc getRunningNetConfig(waku: Waku): Future[Result[NetConfig, string]] {.async.} = + let conf = waku.conf + let (tcpPort, websocketPort) = getPorts(waku.node.switch.peerInfo.listenAddrs).valueOr: return err("Could not retrieve ports: " & error) if tcpPort.isSome(): @@ -276,67 +275,67 @@ proc getRunningNetConfig(waku: ptr Waku): Future[Result[NetConfig, string]] {.as return ok(netConf) -proc updateEnr(waku: ptr Waku): Future[Result[void, string]] {.async.} = +proc updateEnr(waku: Waku): Future[Result[void, string]] {.async.} = let netConf: NetConfig = (await getRunningNetConfig(waku)).valueOr: return err("error calling updateNetConfig: " & $error) - let record = enrConfiguration(waku[].conf, netConf).valueOr: + let record = enrConfiguration(waku.conf, netConf).valueOr: return err("ENR setup failed: " & error) - if isClusterMismatched(record, waku[].conf.clusterId): + if isClusterMismatched(record, waku.conf.clusterId): return err("cluster-id mismatch configured shards") - waku[].node.enr = record + waku.node.enr = record # If TCP/WS was configured with port 0, node.announcedAddresses was built # pre-bind with a port value of 0. In any case, the resync is harmless. - waku[].node.announcedAddresses = netConf.announcedAddresses + waku.node.announcedAddresses = netConf.announcedAddresses return ok() -proc updateAddressInENR(waku: ptr Waku): Result[void, string] = - let addresses: seq[MultiAddress] = waku[].node.announcedAddresses +proc updateAddressInENR(waku: Waku): Result[void, string] = + let addresses: seq[MultiAddress] = waku.node.announcedAddresses let encodedAddrs = multiaddr.encodeMultiaddrs(addresses) ## First update the enr info contained in WakuNode - let keyBytes = waku[].key.getRawBytes().valueOr: + let keyBytes = waku.key.getRawBytes().valueOr: return err("failed to retrieve raw bytes from waku key: " & $error) let parsedPk = keys.PrivateKey.fromHex(keyBytes.toHex()).valueOr: return err("failed to parse the private key: " & $error) let enrFields = @[toFieldPair(MultiaddrEnrField, encodedAddrs)] - waku[].node.enr.update(parsedPk, extraFields = enrFields).isOkOr: + waku.node.enr.update(parsedPk, extraFields = enrFields).isOkOr: return err("failed to update multiaddress in ENR updateAddressInENR: " & $error) info "Waku node ENR updated successfully with new multiaddress", - enr = waku[].node.enr.toUri(), record = $(waku[].node.enr) + enr = waku.node.enr.toUri(), record = $(waku.node.enr) ## Now update the ENR infor in discv5 - if not waku[].wakuDiscv5.isNil(): - waku[].wakuDiscv5.protocol.localNode.record = waku[].node.enr - let enr = waku[].wakuDiscv5.protocol.localNode.record + if not waku.wakuDiscv5.isNil(): + waku.wakuDiscv5.protocol.localNode.record = waku.node.enr + let enr = waku.wakuDiscv5.protocol.localNode.record info "Waku discv5 ENR updated successfully with new multiaddress", enr = enr.toUri(), record = $(enr) return ok() -proc updateWaku(waku: ptr Waku): Future[Result[void, string]] {.async.} = +proc updateWaku(waku: Waku): Future[Result[void, string]] {.async.} = (await updateEnr(waku)).isOkOr: return err("error calling updateEnr: " & $error) - ?updateAnnouncedAddrWithPrimaryIpAddr(waku[].node) + ?updateAnnouncedAddrWithPrimaryIpAddr(waku.node) ?updateAddressInENR(waku) return ok() -proc startDnsDiscoveryRetryLoop(waku: ptr Waku): Future[void] {.async.} = +proc startDnsDiscoveryRetryLoop(waku: Waku): Future[void] {.async.} = while true: await sleepAsync(30.seconds) if waku.conf.dnsDiscoveryConf.isSome(): let dnsDiscoveryConf = waku.conf.dnsDiscoveryConf.get() - waku[].dynamicBootstrapNodes = ( + waku.dynamicBootstrapNodes = ( await waku_dnsdisc.retrieveDynamicBootstrapNodes( dnsDiscoveryConf.enrTreeUrl, dnsDiscoveryConf.nameServers ) @@ -344,35 +343,61 @@ proc startDnsDiscoveryRetryLoop(waku: ptr Waku): Future[void] {.async.} = error "Retrieving dynamic bootstrap nodes failed", error = error continue - if not waku[].wakuDiscv5.isNil(): - let dynamicBootstrapEnrs = waku[].dynamicBootstrapNodes - .filterIt(it.hasUdpPort()) - .mapIt(it.enr.get().toUri()) + if not waku.wakuDiscv5.isNil(): + let dynamicBootstrapEnrs = + waku.dynamicBootstrapNodes.filterIt(it.hasUdpPort()).mapIt(it.enr.get().toUri()) var discv5BootstrapEnrs: seq[enr.Record] # parse enrURIs from the configuration and add the resulting ENRs to the discv5BootstrapEnrs seq for enrUri in dynamicBootstrapEnrs: addBootstrapNode(enrUri, discv5BootstrapEnrs) - waku[].wakuDiscv5.updateBootstrapRecords( - waku[].wakuDiscv5.protocol.bootstrapRecords & discv5BootstrapEnrs + waku.wakuDiscv5.updateBootstrapRecords( + waku.wakuDiscv5.protocol.bootstrapRecords & discv5BootstrapEnrs ) info "Connecting to dynamic bootstrap peers" try: - await connectToNodes( - waku[].node, waku[].dynamicBootstrapNodes, "dynamic bootstrap" - ) + await connectToNodes(waku.node, waku.dynamicBootstrapNodes, "dynamic bootstrap") except CatchableError: error "failed to connect to dynamic bootstrap nodes: " & getCurrentExceptionMsg() return -proc startWaku*(waku: ptr Waku): Future[Result[void, string]] {.async: (raises: []).} = - if waku[].node.started: - warn "startWaku: waku node already started" +proc mountMessagingClient*(waku: Waku): Result[void, string] = + if not waku.messagingClient.isNil(): + return err("messaging client already mounted") + if waku.node.started: + return err("cannot mount messaging client on a started node") + waku.messagingClient = MessagingClient.new(waku.conf.p2pReliability, waku.node).valueOr: + return err("could not create messaging client: " & $error) + return ok() + +proc mountReliableChannelManager*(waku: Waku): Result[void, string] = + if not waku.reliableChannelManager.isNil(): + return err("reliable channel manager already mounted") + if waku.messagingClient.isNil(): + return err("reliable channel manager requires a mounted messaging client") + if waku.node.started: + return err("cannot mount reliable channel manager on a started node") + + let messagingClient = waku.messagingClient + let defaultSendHandler: SendHandler = proc( + envelope: MessageEnvelope + ): Future[Result[RequestId, string]] {.async: (raises: [CatchableError]), gcsafe.} = + return await messagingClient.send(envelope) + + waku.reliableChannelManager = ReliableChannelManager.new( + messagingClient, defaultSendHandler, waku.brokerCtx + ).valueOr: + return err("could not create reliable channel manager: " & $error) + return ok() + +proc start*(waku: Waku): Future[Result[void, string]] {.async: (raises: []).} = + if waku.node.started: + warn "start: waku node already started" return ok() info "Retrieve dynamic bootstrap nodes" - let conf = waku[].conf + let conf = waku.conf if conf.dnsDiscoveryConf.isSome(): let dnsDiscoveryConf = waku.conf.dnsDiscoveryConf.get() @@ -390,9 +415,9 @@ proc startWaku*(waku: ptr Waku): Future[Result[void, string]] {.async: (raises: error "Retrieving dynamic bootstrap nodes failed", error = dynamicBootstrapNodesRes.error # Start Dns Discovery retry loop - waku[].dnsRetryLoopHandle = waku.startDnsDiscoveryRetryLoop() + waku.dnsRetryLoopHandle = waku.startDnsDiscoveryRetryLoop() else: - waku[].dynamicBootstrapNodes = dynamicBootstrapNodesRes.get() + waku.dynamicBootstrapNodes = dynamicBootstrapNodesRes.get() ## Initialize persistency singleton instance - we don't need the instance itself here, ## but this ensures it's initialized before any store job starts. @@ -405,12 +430,12 @@ proc startWaku*(waku: ptr Waku): Future[Result[void, string]] {.async: (raises: let bound = getPorts(waku.node.switch.peerInfo.listenAddrs).valueOr: return err("failed to read bound ports from switch: " & $error) - waku[].node.ports.tcp = bound.tcpPort.get(Port(0)).uint16 - waku[].node.ports.webSocket = bound.websocketPort.get(Port(0)).uint16 + waku.node.ports.tcp = bound.tcpPort.get(Port(0)).uint16 + waku.node.ports.webSocket = bound.websocketPort.get(Port(0)).uint16 ## Discv5 if conf.discv5Conf.isSome(): - waku[].wakuDiscV5 = ( + waku.wakuDiscV5 = ( await waku_discv5.setupAndStartDiscv5( waku.node.enr, waku.node.peerManager, @@ -425,23 +450,21 @@ proc startWaku*(waku: ptr Waku): Future[Result[void, string]] {.async: (raises: ).valueOr: return err("failed to start waku discovery v5: " & error) - waku[].node.ports.discv5Udp = waku[].wakuDiscV5.udpPort.uint16 - waku[].conf.discv5Conf.get().udpPort = waku[].wakuDiscV5.udpPort + waku.node.ports.discv5Udp = waku.wakuDiscV5.udpPort.uint16 + waku.conf.discv5Conf.get().udpPort = waku.wakuDiscV5.udpPort ## Update waku data that is set dynamically on node start try: (await updateWaku(waku)).isOkOr: - return err("Error in startWaku: " & $error) + return err("Error in start: " & $error) except CatchableError: - return err("Caught exception in startWaku: " & getCurrentExceptionMsg()) + return err("Caught exception in start: " & getCurrentExceptionMsg()) - ## Reliability - if not waku[].deliveryService.isNil(): - waku[].deliveryService.startDeliveryService().isOkOr: - return err("failed to start delivery service: " & $error) + waku.node.subscriptionManager.subscribeAllAutoshards().isOkOr: + return err("failed to auto-subscribe autosharding shards: " & $error) ## Health Monitor - waku[].healthMonitor.startHealthMonitor().isOkOr: + waku.healthMonitor.startHealthMonitor().isOkOr: return err("failed to start health monitor: " & $error) ## Setup RequestConnectionStatus provider @@ -450,7 +473,7 @@ proc startWaku*(waku: ptr Waku): Future[Result[void, string]] {.async: (raises: globalBrokerContext(), proc(): Result[RequestConnectionStatus, string] = try: - let healthReport = waku[].healthMonitor.getSyncNodeHealthReport() + let healthReport = waku.healthMonitor.getSyncNodeHealthReport() return ok(RequestConnectionStatus(connectionStatus: healthReport.connectionStatus)) except CatchableError: @@ -467,7 +490,7 @@ proc startWaku*(waku: ptr Waku): Future[Result[void, string]] {.async: (raises: ): Future[Result[RequestProtocolHealth, string]] {.async.} = try: let protocolHealthStatus = - await waku[].healthMonitor.getProtocolHealthInfo(protocol) + await waku.healthMonitor.getProtocolHealthInfo(protocol) return ok(RequestProtocolHealth(healthStatus: protocolHealthStatus)) except CatchableError: return err("Failed to get protocol health: " & getCurrentExceptionMsg()), @@ -480,7 +503,7 @@ proc startWaku*(waku: ptr Waku): Future[Result[void, string]] {.async: (raises: globalBrokerContext(), proc(): Future[Result[RequestHealthReport, string]] {.async.} = try: - let report = await waku[].healthMonitor.getNodeHealthReport() + let report = await waku.healthMonitor.getNodeHealthReport() return ok(RequestHealthReport(healthReport: report)) except CatchableError: return err("Failed to get health report: " & getCurrentExceptionMsg()), @@ -489,9 +512,9 @@ proc startWaku*(waku: ptr Waku): Future[Result[void, string]] {.async: (raises: if conf.restServerConf.isSome(): rest_server_builder.startRestServerProtocolSupport( - waku[].restServer, - waku[].node, - waku[].wakuDiscv5, + waku.restServer, + waku.node, + waku.wakuDiscv5, conf.restServerConf.get(), conf.relay, conf.lightPush, @@ -509,21 +532,27 @@ proc startWaku*(waku: ptr Waku): Future[Result[void, string]] {.async: (raises: ) ).valueOr: return err("Starting monitoring and external interfaces failed: " & error) - waku[].metricsServer = server - waku[].node.ports.metrics = port.uint16 - waku[].conf.metricsServerConf.get().httpPort = port + waku.metricsServer = server + waku.node.ports.metrics = port.uint16 + waku.conf.metricsServerConf.get().httpPort = port except CatchableError: return err( "Caught exception starting monitoring and external interfaces failed: " & getCurrentExceptionMsg() ) - waku[].healthMonitor.setOverallHealth(HealthStatus.READY) + waku.healthMonitor.setOverallHealth(HealthStatus.READY) + + if not waku.messagingClient.isNil(): + waku.messagingClient.start().isOkOr: + return err("failed to start messaging client: " & $error) + + if not waku.reliableChannelManager.isNil(): + waku.reliableChannelManager.start().isOkOr: + return err("failed to start reliable channel manager: " & $error) return ok() proc stop*(waku: Waku): Future[Result[void, string]] {.async: (raises: []).} = - ## Waku shutdown - if not waku.node.started: warn "stop: attempting to stop node that isn't running" @@ -538,9 +567,11 @@ proc stop*(waku: Waku): Future[Result[void, string]] {.async: (raises: []).} = if not waku.wakuDiscv5.isNil(): await waku.wakuDiscv5.stop() - if not waku.deliveryService.isNil(): - await waku.deliveryService.stopDeliveryService() - waku.deliveryService = nil + if not waku.reliableChannelManager.isNil(): + await waku.reliableChannelManager.stop() + + if not waku.messagingClient.isNil(): + await waku.messagingClient.stop() if not waku.node.isNil(): await waku.node.stop() diff --git a/waku/messaging_client.nim b/waku/messaging_client.nim new file mode 100644 index 000000000..1fc4deb3c --- /dev/null +++ b/waku/messaging_client.nim @@ -0,0 +1,63 @@ +import results, chronos +import chronicles +import + ./api/types, + ./node/[ + waku_node, + subscription_manager, + delivery_service/recv_service, + delivery_service/send_service, + delivery_service/send_service/delivery_task, + ] + +type MessagingClient* = ref object + node: WakuNode + sendService*: SendService + recvService*: RecvService + started: bool + +proc new*( + T: type MessagingClient, useP2PReliability: bool, node: WakuNode +): Result[T, string] = + let sendService = ?SendService.new(useP2PReliability, node) + let recvService = RecvService.new(node) + ok(T(node: node, sendService: sendService, recvService: recvService)) + +proc start*(self: MessagingClient): Result[void, string] = + if self.started: + return ok() + self.recvService.startRecvService() + self.sendService.startSendService() + self.started = true + ok() + +proc stop*(self: MessagingClient) {.async.} = + if not self.started: + return + await self.sendService.stopSendService() + await self.recvService.stopRecvService() + self.started = false + +proc send*( + self: MessagingClient, envelope: MessageEnvelope +): Future[Result[RequestId, string]] {.async.} = + ## High-level messaging API send. Auto-subscribes to the content topic + ## (so the local node sees its own gossipsub broadcast), builds a + ## `DeliveryTask`, and hands it to the send service. Returns the request + ## id the caller can correlate with `MessageSentEvent` / `MessageErrorEvent`. + let isSubbed = + self.node.subscriptionManager.isSubscribed(envelope.contentTopic).valueOr(false) + if not isSubbed: + info "Auto-subscribing to topic on send", contentTopic = envelope.contentTopic + self.node.subscriptionManager.subscribe(envelope.contentTopic).isOkOr: + warn "Failed to auto-subscribe", error = error + return err("Failed to auto-subscribe before sending: " & error) + + let requestId = RequestId.new(self.node.rng) + + let deliveryTask = DeliveryTask.new(requestId, envelope, self.node.brokerCtx).valueOr: + return err("MessagingClient.send: Failed to create delivery task: " & error) + + asyncSpawn self.sendService.send(deliveryTask) + + return ok(requestId) diff --git a/waku/node/delivery_service/delivery_service.nim b/waku/node/delivery_service/delivery_service.nim deleted file mode 100644 index f3d78d98e..000000000 --- a/waku/node/delivery_service/delivery_service.nim +++ /dev/null @@ -1,44 +0,0 @@ -## This module helps to ensure the correct transmission and reception of messages - -import results -import chronos, chronicles -import - ./recv_service, - ./send_service, - ./subscription_manager, - waku/[ - waku_core, waku_node, waku_store/client, waku_relay/protocol, waku_lightpush/client - ] - -type DeliveryService* = ref object - sendService*: SendService - recvService*: RecvService - subscriptionManager*: SubscriptionManager - -proc new*( - T: type DeliveryService, useP2PReliability: bool, w: WakuNode -): Result[T, string] = - ## storeClient is needed to give store visitility to DeliveryService - ## wakuRelay and wakuLightpushClient are needed to give a mechanism to SendService to re-publish - let subscriptionManager = SubscriptionManager.new(w) - let sendService = ?SendService.new(useP2PReliability, w, subscriptionManager) - let recvService = RecvService.new(w, subscriptionManager) - - return ok( - DeliveryService( - sendService: sendService, - recvService: recvService, - subscriptionManager: subscriptionManager, - ) - ) - -proc startDeliveryService*(self: DeliveryService): Result[void, string] = - ?self.subscriptionManager.startSubscriptionManager() - self.recvService.startRecvService() - self.sendService.startSendService() - return ok() - -proc stopDeliveryService*(self: DeliveryService) {.async.} = - await self.sendService.stopSendService() - await self.recvService.stopRecvService() - await self.subscriptionManager.stopSubscriptionManager() diff --git a/waku/node/delivery_service/recv_service/recv_service.nim b/waku/node/delivery_service/recv_service/recv_service.nim index 899f80f71..500926cc7 100644 --- a/waku/node/delivery_service/recv_service/recv_service.nim +++ b/waku/node/delivery_service/recv_service/recv_service.nim @@ -4,17 +4,17 @@ import std/[tables, sequtils, options, sets] import chronos, chronicles, libp2p/utility -import ../[subscription_manager] import brokers/broker_context import waku/[ waku_core, + waku_core/topics, waku_store/client, waku_store/common, waku_filter_v2/client, - waku_core/topics, events/message_events, waku_node, + node/subscription_manager, ] const StoreCheckPeriod = chronos.minutes(5) ## How often to perform store queries @@ -38,7 +38,6 @@ type RecvService* = ref object of RootObj brokerCtx: BrokerContext node: WakuNode seenMsgListener: MessageSeenEventListener - subscriptionManager: SubscriptionManager recentReceivedMsgs: seq[RecvMessage] @@ -77,7 +76,9 @@ proc processIncomingMessage( ## or if the message is a duplicate (recently-seen). Otherwise, save it as ## recently-seen, emit a MessageReceivedEvent, and return true. - if not self.subscriptionManager.isSubscribed(pubsubTopic, message.contentTopic): + if not self.node.subscriptionManager.isContentSubscribed( + pubsubTopic, message.contentTopic + ): trace "skipping message as I am not subscribed", shard = pubsubTopic, contentTopic = message.contentTopic return false @@ -101,7 +102,7 @@ proc checkStore*(self: RecvService) {.async.} = self.endTimeToCheck = getNowInNanosecondTime() ## query store and deliver new recovered messages per subscribed topic - for pubsubTopic, contentTopics in self.subscriptionManager.subscribedTopics: + for pubsubTopic, contentTopics in self.node.subscriptionManager.subscribedContentTopics: let storeResp: StoreQueryResponse = ( await self.node.wakuStoreClient.queryToAny( StoreQueryRequest( @@ -146,7 +147,7 @@ proc msgChecker(self: RecvService) {.async.} = await sleepAsync(StoreCheckPeriod) await self.checkStore() -proc new*(T: typedesc[RecvService], node: WakuNode, s: SubscriptionManager): T = +proc new*(T: typedesc[RecvService], node: WakuNode): T = ## The storeClient will help to acquire any possible missed messages let now = getNowInNanosecondTime() @@ -154,7 +155,6 @@ proc new*(T: typedesc[RecvService], node: WakuNode, s: SubscriptionManager): T = node: node, startTimeToCheck: now, brokerCtx: node.brokerCtx, - subscriptionManager: s, recentReceivedMsgs: @[], ) diff --git a/waku/node/delivery_service/send_service/send_service.nim b/waku/node/delivery_service/send_service/send_service.nim index 88ec802cf..e60b26124 100644 --- a/waku/node/delivery_service/send_service/send_service.nim +++ b/waku/node/delivery_service/send_service/send_service.nim @@ -6,10 +6,10 @@ import chronos, chronicles, libp2p/utility import brokers/broker_context import ./[send_processor, relay_processor, lightpush_processor, delivery_task], - ../[subscription_manager], waku/[ waku_core, node/waku_node, + node/subscription_manager, node/peer_manager, waku_store/client, waku_store/common, @@ -58,7 +58,6 @@ type SendService* = ref object of RootObj node: WakuNode checkStoreForMessages: bool - subscriptionManager: SubscriptionManager proc setupSendProcessorChain( peerManager: PeerManager, @@ -96,10 +95,7 @@ proc setupSendProcessorChain( return ok(processors[0]) proc new*( - T: typedesc[SendService], - preferP2PReliability: bool, - w: WakuNode, - s: SubscriptionManager, + T: typedesc[SendService], preferP2PReliability: bool, w: WakuNode ): Result[T, string] = if w.wakuRelay.isNil() and w.wakuLightpushClient.isNil(): return err( @@ -120,7 +116,6 @@ proc new*( sendProcessor: sendProcessorChain, node: w, checkStoreForMessages: checkStoreForMessages, - subscriptionManager: s, ) return ok(sendService) @@ -263,7 +258,7 @@ proc send*(self: SendService, task: DeliveryTask) {.async.} = info "SendService.send: processing delivery task", requestId = task.requestId, msgHash = task.msgHash.to0xHex() - self.subscriptionManager.subscribe(task.msg.contentTopic).isOkOr: + self.node.subscriptionManager.subscribe(task.msg.contentTopic).isOkOr: error "SendService.send: failed to subscribe to content topic", contentTopic = task.msg.contentTopic, error = error diff --git a/waku/node/delivery_service/subscription_manager.nim b/waku/node/delivery_service/subscription_manager.nim deleted file mode 100644 index 393a61eae..000000000 --- a/waku/node/delivery_service/subscription_manager.nim +++ /dev/null @@ -1,596 +0,0 @@ -import std/[sequtils, sets, tables, options, strutils], chronos, chronicles, results -import libp2p/[peerid, peerinfo] -import brokers/broker_context - -import - waku/[ - waku_core, - waku_core/topics, - waku_core/topics/sharding, - waku_node, - waku_relay, - waku_filter_v2/common as filter_common, - waku_filter_v2/client as filter_client, - waku_filter_v2/protocol as filter_protocol, - events/health_events, - events/peer_events, - requests/health_requests, - node/peer_manager, - node/health_monitor/topic_health, - node/health_monitor/connection_status, - ] - -# --------------------------------------------------------------------------- -# Logos Messaging API SubscriptionManager -# -# Maps all topic subscription intent and centralizes all consistency -# maintenance of the pubsub and content topic subscription model across -# the various network drivers that handle topics (Edge/Filter and Core/Relay). -# --------------------------------------------------------------------------- - -type EdgeFilterSubState* = object - peers: seq[RemotePeerInfo] - ## Filter service peers with confirmed subscriptions on this shard. - pending: seq[Future[void]] ## In-flight dial futures for peers not yet confirmed. - pendingPeers: HashSet[PeerId] ## PeerIds of peers currently being dialed. - currentHealth: TopicHealth - ## Cached health derived from peers.len; updated on every peer set change. - -func toTopicHealth*(peersCount: int): TopicHealth = - if peersCount >= HealthyThreshold: - TopicHealth.SUFFICIENTLY_HEALTHY - elif peersCount > 0: - TopicHealth.MINIMALLY_HEALTHY - else: - TopicHealth.UNHEALTHY - -type SubscriptionManager* = ref object of RootObj - node: WakuNode - contentTopicSubs: Table[PubsubTopic, HashSet[ContentTopic]] - ## Map of Shard to ContentTopic needed because e.g. WakuRelay is PubsubTopic only. - ## A present key with an empty HashSet value means pubsubtopic already subscribed - ## (via subscribePubsubTopics()) but there's no specific content topic interest yet. - edgeFilterSubStates*: Table[PubsubTopic, EdgeFilterSubState] - ## Per-shard filter subscription state for edge mode. - edgeFilterWakeup: AsyncEvent - ## Signalled when the edge filter sub loop should re-reconcile. - edgeFilterSubLoopFut: Future[void] - edgeFilterHealthLoopFut: Future[void] - peerEventListener: WakuPeerEventListener - ## Listener for peer connect/disconnect events (edge filter wakeup). - -iterator subscribedTopics*( - self: SubscriptionManager -): (PubsubTopic, HashSet[ContentTopic]) = - ## Iterate over all subscribed content topics, batched per shard. - ## This is guaranteed to return a non-empty `topics` (content topics) list on iteration. - - for pubsub, topics in self.contentTopicSubs.pairs: - # We are iterating over subscribed content topics; if we are subscribed to - # a shard but have no subscription (interest) for any content topic in that - # shard, then avoid triggering an iteration that doesn't advance the intent - # to iterate over content topic subscriptions. - if topics.len == 0: - continue - yield (pubsub, topics) - -proc edgeFilterPeerCount*(sm: SubscriptionManager, shard: PubsubTopic): int = - sm.edgeFilterSubStates.withValue(shard, state): - return state.peers.len - return 0 - -proc new*(T: typedesc[SubscriptionManager], node: WakuNode): T = - SubscriptionManager( - node: node, contentTopicSubs: initTable[PubsubTopic, HashSet[ContentTopic]]() - ) - -proc addContentTopicInterest( - self: SubscriptionManager, shard: PubsubTopic, topic: ContentTopic -): Result[void, string] = - var changed = false - if not self.contentTopicSubs.hasKey(shard): - self.contentTopicSubs[shard] = initHashSet[ContentTopic]() - changed = true - - self.contentTopicSubs.withValue(shard, cTopics): - if not cTopics[].contains(topic): - cTopics[].incl(topic) - changed = true - - if changed and not isNil(self.edgeFilterWakeup): - self.edgeFilterWakeup.fire() - - return ok() - -proc removeContentTopicInterest( - self: SubscriptionManager, shard: PubsubTopic, topic: ContentTopic -): Result[void, string] = - var changed = false - self.contentTopicSubs.withValue(shard, cTopics): - if cTopics[].contains(topic): - cTopics[].excl(topic) - changed = true - - if cTopics[].len == 0 and isNil(self.node.wakuRelay): - self.contentTopicSubs.del(shard) # We're done with cTopics here - - if changed and not isNil(self.edgeFilterWakeup): - self.edgeFilterWakeup.fire() - - return ok() - -proc subscribePubsubTopics( - self: SubscriptionManager, shards: seq[PubsubTopic] -): Result[void, string] = - if isNil(self.node.wakuRelay): - return err("subscribePubsubTopics requires a Relay") - - var errors: seq[string] - - for shard in shards: - if not self.contentTopicSubs.hasKey(shard): - self.node.subscribe((kind: PubsubSub, topic: shard), nil).isOkOr: - errors.add("shard " & shard & ": " & error) - continue - - self.contentTopicSubs[shard] = initHashSet[ContentTopic]() - - if errors.len > 0: - return err("subscribeShard errors: " & errors.join("; ")) - - return ok() - -proc getShardForContentTopic( - self: SubscriptionManager, topic: ContentTopic -): Result[PubsubTopic, string] = - if self.node.wakuAutoSharding.isSome(): - let shardObj = ?self.node.wakuAutoSharding.get().getShard(topic) - return ok($shardObj) - - return err("SubscriptionManager requires AutoSharding") - -proc isSubscribed*( - self: SubscriptionManager, topic: ContentTopic -): Result[bool, string] = - let shard = ?self.getShardForContentTopic(topic) - return ok( - self.contentTopicSubs.hasKey(shard) and self.contentTopicSubs[shard].contains(topic) - ) - -proc isSubscribed*( - self: SubscriptionManager, shard: PubsubTopic, contentTopic: ContentTopic -): bool {.raises: [].} = - self.contentTopicSubs.withValue(shard, cTopics): - return cTopics[].contains(contentTopic) - return false - -proc subscribe*(self: SubscriptionManager, topic: ContentTopic): Result[void, string] = - if isNil(self.node.wakuRelay) and isNil(self.node.wakuFilterClient): - return err("SubscriptionManager requires either Relay or Filter Client.") - - let shard = ?self.getShardForContentTopic(topic) - - if not isNil(self.node.wakuRelay) and not self.contentTopicSubs.hasKey(shard): - ?self.subscribePubsubTopics(@[shard]) - - ?self.addContentTopicInterest(shard, topic) - - return ok() - -proc unsubscribe*( - self: SubscriptionManager, topic: ContentTopic -): Result[void, string] = - if isNil(self.node.wakuRelay) and isNil(self.node.wakuFilterClient): - return err("SubscriptionManager requires either Relay or Filter Client.") - - let shard = ?self.getShardForContentTopic(topic) - - if self.isSubscribed(shard, topic): - ?self.removeContentTopicInterest(shard, topic) - - return ok() - -# --------------------------------------------------------------------------- -# Edge Filter driver for the Logos Messaging API -# -# The SubscriptionManager absorbs natively the responsibility of using the -# Edge Filter protocol to effect subscriptions and message receipt for edge. -# --------------------------------------------------------------------------- - -const EdgeFilterSubscribeTimeout = chronos.seconds(15) - ## Timeout for a single filter subscribe/unsubscribe RPC to a service peer. -const EdgeFilterPingTimeout = chronos.seconds(5) - ## Timeout for a filter ping health check. -const EdgeFilterLoopInterval = chronos.seconds(30) - ## Interval for the edge filter health ping loop. -const EdgeFilterSubLoopDebounce = chronos.seconds(1) - ## Debounce delay to coalesce rapid-fire wakeups into a single reconciliation pass. - -type EdgeDialTask = object - peer: RemotePeerInfo - shard: PubsubTopic - topics: seq[ContentTopic] - -proc updateShardHealth( - self: SubscriptionManager, shard: PubsubTopic, state: var EdgeFilterSubState -) = - ## Recompute and emit health for a shard after its peer set changed. - let newHealth = toTopicHealth(state.peers.len) - if newHealth != state.currentHealth: - state.currentHealth = newHealth - EventShardTopicHealthChange.emit(self.node.brokerCtx, shard, newHealth) - -proc removePeer(self: SubscriptionManager, shard: PubsubTopic, peerId: PeerId) = - ## Remove a peer from edgeFilterSubStates for the given shard, - ## update health, and wake the sub loop to dial a replacement. - ## Best-effort unsubscribe so the service peer stops pushing to us. - self.edgeFilterSubStates.withValue(shard, state): - var peer: RemotePeerInfo - var found = false - for p in state.peers: - if p.peerId == peerId: - peer = p - found = true - break - if not found: - return - - state.peers.keepItIf(it.peerId != peerId) - self.updateShardHealth(shard, state[]) - self.edgeFilterWakeup.fire() - - if not self.node.wakuFilterClient.isNil(): - self.contentTopicSubs.withValue(shard, topics): - let ct = toSeq(topics[]) - if ct.len > 0: - proc doUnsubscribe() {.async.} = - discard await self.node.wakuFilterClient.unsubscribe(peer, shard, ct) - - asyncSpawn doUnsubscribe() - -type SendChunkedFilterRpcKind = enum - FilterSubscribe - FilterUnsubscribe - -proc sendChunkedFilterRpc( - self: SubscriptionManager, - peer: RemotePeerInfo, - shard: PubsubTopic, - topics: seq[ContentTopic], - kind: SendChunkedFilterRpcKind, -): Future[bool] {.async.} = - ## Send a chunked filter subscribe or unsubscribe RPC. Returns true on - ## success. On failure the peer is removed and false is returned. - try: - var i = 0 - while i < topics.len: - let chunk = - topics[i ..< min(i + filter_protocol.MaxContentTopicsPerRequest, topics.len)] - let fut = - case kind - of FilterSubscribe: - self.node.wakuFilterClient.subscribe(peer, shard, chunk) - of FilterUnsubscribe: - self.node.wakuFilterClient.unsubscribe(peer, shard, chunk) - if not (await fut.withTimeout(EdgeFilterSubscribeTimeout)) or fut.read().isErr(): - trace "sendChunkedFilterRpc: chunk failed", - op = kind, shard = shard, peer = peer.peerId - self.removePeer(shard, peer.peerId) - return false - i += filter_protocol.MaxContentTopicsPerRequest - except CatchableError as exc: - debug "sendChunkedFilterRpc: failed", - op = kind, shard = shard, peer = peer.peerId, err = exc.msg - self.removePeer(shard, peer.peerId) - return false - return true - -proc syncFilterDeltas( - self: SubscriptionManager, - peer: RemotePeerInfo, - shard: PubsubTopic, - added: seq[ContentTopic], - removed: seq[ContentTopic], -) {.async.} = - ## Push content topic changes (adds/removes) to an already-tracked peer. - if added.len > 0: - if not await self.sendChunkedFilterRpc(peer, shard, added, FilterSubscribe): - return - - if removed.len > 0: - discard await self.sendChunkedFilterRpc(peer, shard, removed, FilterUnsubscribe) - -proc dialFilterPeer( - self: SubscriptionManager, - peer: RemotePeerInfo, - shard: PubsubTopic, - contentTopics: seq[ContentTopic], -) {.async.} = - ## Subscribe a new peer to all content topics on a shard and start tracking it. - self.edgeFilterSubStates.withValue(shard, state): - state.pendingPeers.incl(peer.peerId) - - try: - if not await self.sendChunkedFilterRpc(peer, shard, contentTopics, FilterSubscribe): - return - - self.edgeFilterSubStates.withValue(shard, state): - if state.peers.anyIt(it.peerId == peer.peerId): - trace "dialFilterPeer: peer already tracked, skipping duplicate", - shard = shard, peer = peer.peerId - return - - state.peers.add(peer) - self.updateShardHealth(shard, state[]) - trace "dialFilterPeer: successfully subscribed to all chunks", - shard = shard, peer = peer.peerId, totalPeers = state.peers.len - do: - trace "dialFilterPeer: shard removed while subscribing, discarding result", - shard = shard, peer = peer.peerId - finally: - self.edgeFilterSubStates.withValue(shard, state): - state.pendingPeers.excl(peer.peerId) - -proc edgeFilterHealthLoop*(self: SubscriptionManager) {.async.} = - ## Periodically pings all connected filter service peers to verify they are - ## still alive at the application layer. Peers that fail the ping are removed. - while true: - await sleepAsync(EdgeFilterLoopInterval) - - if self.node.wakuFilterClient.isNil(): - warn "filter client is nil within edge filter health loop" - continue - - var connected = initTable[PeerId, RemotePeerInfo]() - for state in self.edgeFilterSubStates.values: - for peer in state.peers: - if self.node.peerManager.switch.peerStore.isConnected(peer.peerId): - connected[peer.peerId] = peer - - var alive = initHashSet[PeerId]() - - if connected.len > 0: - var pingTasks: seq[(PeerId, Future[FilterSubscribeResult])] - for peer in connected.values: - pingTasks.add( - (peer.peerId, self.node.wakuFilterClient.ping(peer, EdgeFilterPingTimeout)) - ) - - # extract future tasks from (PeerId, Future) tuples and await them - await allFutures(pingTasks.mapIt(it[1])) - - for (peerId, task) in pingTasks: - if task.read().isOk(): - alive.incl(peerId) - - var changed = false - for shard, state in self.edgeFilterSubStates.mpairs: - let oldLen = state.peers.len - state.peers.keepItIf(it.peerId notin connected or alive.contains(it.peerId)) - - if state.peers.len < oldLen: - changed = true - self.updateShardHealth(shard, state) - trace "Edge Filter health degraded by Ping failure", - shard = shard, new = state.currentHealth - - if changed: - self.edgeFilterWakeup.fire() - -proc selectFilterCandidates( - self: SubscriptionManager, shard: PubsubTopic, exclude: HashSet[PeerId], needed: int -): seq[RemotePeerInfo] = - ## Select filter service peer candidates for a shard. - - # Start with every filter server peer that can serve the shard - var allCandidates = self.node.peerManager.selectPeers( - filter_common.WakuFilterSubscribeCodec, some(shard) - ) - - # Remove all already used in this shard or being dialed for it - allCandidates.keepItIf(it.peerId notin exclude) - - # Collect peer IDs already tracked on other shards - var trackedOnOther = initHashSet[PeerId]() - for otherShard, otherState in self.edgeFilterSubStates.pairs: - if otherShard != shard: - for peer in otherState.peers: - trackedOnOther.incl(peer.peerId) - - # Prefer peers we already have a connection to first, preserving shuffle - var candidates = - allCandidates.filterIt(it.peerId in trackedOnOther) & - allCandidates.filterIt(it.peerId notin trackedOnOther) - - # We need to return 'needed' peers only - if candidates.len > needed: - candidates.setLen(needed) - return candidates - -proc edgeFilterSubLoop*(self: SubscriptionManager) {.async.} = - ## Reconciles filter subscriptions with the desired state from SubscriptionManager. - var lastSynced = initTable[PubsubTopic, HashSet[ContentTopic]]() - - while true: - await self.edgeFilterWakeup.wait() - await sleepAsync(EdgeFilterSubLoopDebounce) - self.edgeFilterWakeup.clear() - trace "edgeFilterSubLoop: woke up" - - if isNil(self.node.wakuFilterClient): - trace "edgeFilterSubLoop: wakuFilterClient is nil, skipping" - continue - - let desired = self.contentTopicSubs - - trace "edgeFilterSubLoop: desired state", numShards = desired.len - - let allShards = toHashSet(toSeq(desired.keys)) + toHashSet(toSeq(lastSynced.keys)) - - # Step 1: read state across all shards at once and - # create a list of peer dial tasks and shard tracking to delete. - - var dialTasks: seq[EdgeDialTask] - var shardsToDelete: seq[PubsubTopic] - - for shard in allShards: - let currTopics = desired.getOrDefault(shard) - let prevTopics = lastSynced.getOrDefault(shard) - - if shard notin self.edgeFilterSubStates: - self.edgeFilterSubStates[shard] = - EdgeFilterSubState(currentHealth: TopicHealth.UNHEALTHY) - - let addedTopics = toSeq(currTopics - prevTopics) - let removedTopics = toSeq(prevTopics - currTopics) - - self.edgeFilterSubStates.withValue(shard, state): - state.peers.keepItIf( - self.node.peerManager.switch.peerStore.isConnected(it.peerId) - ) - state.pending.keepItIf(not it.finished) - - if addedTopics.len > 0 or removedTopics.len > 0: - for peer in state.peers: - asyncSpawn self.syncFilterDeltas(peer, shard, addedTopics, removedTopics) - - if currTopics.len == 0: - shardsToDelete.add(shard) - else: - self.updateShardHealth(shard, state[]) - - let needed = max(0, HealthyThreshold - state.peers.len - state.pending.len) - - if needed > 0: - let tracked = state.peers.mapIt(it.peerId).toHashSet() + state.pendingPeers - let candidates = self.selectFilterCandidates(shard, tracked, needed) - let toDial = min(needed, candidates.len) - - trace "edgeFilterSubLoop: shard reconciliation", - shard = shard, - num_peers = state.peers.len, - num_pending = state.pending.len, - num_needed = needed, - num_available = candidates.len, - toDial = toDial - - for i in 0 ..< toDial: - dialTasks.add( - EdgeDialTask( - peer: candidates[i], shard: shard, topics: toSeq(currTopics) - ) - ) - - # Step 2: execute deferred shard tracking deletion and dial tasks. - - for shard in shardsToDelete: - self.edgeFilterSubStates.withValue(shard, state): - for fut in state.pending: - if not fut.finished: - await fut.cancelAndWait() - self.edgeFilterSubStates.del(shard) - - for task in dialTasks: - let fut = self.dialFilterPeer(task.peer, task.shard, task.topics) - self.edgeFilterSubStates.withValue(task.shard, state): - state.pending.add(fut) - - lastSynced = desired - -proc startEdgeFilterLoops(self: SubscriptionManager): Result[void, string] = - ## Start the edge filter orchestration loops. - ## Caller must ensure this is only called in edge mode (relay nil, filter client present). - self.edgeFilterWakeup = newAsyncEvent() - - self.peerEventListener = WakuPeerEvent.listen( - self.node.brokerCtx, - proc(evt: WakuPeerEvent) {.async: (raises: []), gcsafe.} = - if evt.kind == WakuPeerEventKind.EventDisconnected or - evt.kind == WakuPeerEventKind.EventMetadataUpdated: - self.edgeFilterWakeup.fire() - , - ).valueOr: - return err("Failed to listen to peer events for edge filter: " & error) - - self.edgeFilterSubLoopFut = self.edgeFilterSubLoop() - self.edgeFilterHealthLoopFut = self.edgeFilterHealthLoop() - return ok() - -proc stopEdgeFilterLoops(self: SubscriptionManager) {.async: (raises: []).} = - ## Stop the edge filter orchestration loops and clean up pending futures. - if not isNil(self.edgeFilterSubLoopFut): - await self.edgeFilterSubLoopFut.cancelAndWait() - self.edgeFilterSubLoopFut = nil - - if not isNil(self.edgeFilterHealthLoopFut): - await self.edgeFilterHealthLoopFut.cancelAndWait() - self.edgeFilterHealthLoopFut = nil - - for shard, state in self.edgeFilterSubStates: - for fut in state.pending: - if not fut.finished: - await fut.cancelAndWait() - - await WakuPeerEvent.dropListener(self.node.brokerCtx, self.peerEventListener) - -# --------------------------------------------------------------------------- -# SubscriptionManager Lifecycle (calls Edge behavior above) -# -# startSubscriptionManager and stopSubscriptionManager orchestrate both the -# core (relay) and edge (filter) paths, and register/clear broker providers. -# --------------------------------------------------------------------------- - -proc startSubscriptionManager*(self: SubscriptionManager): Result[void, string] = - # Register edge filter broker providers. The shard/content health providers - # in WakuNode query these via the broker as a fallback when relay health is - # not available. If edge mode is not active, these providers simply return - # NOT_SUBSCRIBED / strength 0, which is harmless. - RequestEdgeShardHealth.setProvider( - self.node.brokerCtx, - proc(shard: PubsubTopic): Result[RequestEdgeShardHealth, string] = - self.edgeFilterSubStates.withValue(shard, state): - return ok(RequestEdgeShardHealth(health: state.currentHealth)) - return ok(RequestEdgeShardHealth(health: TopicHealth.NOT_SUBSCRIBED)), - ).isOkOr: - error "Can't set provider for RequestEdgeShardHealth", error = error - - RequestEdgeFilterPeerCount.setProvider( - self.node.brokerCtx, - proc(): Result[RequestEdgeFilterPeerCount, string] = - var minPeers = high(int) - for state in self.edgeFilterSubStates.values: - minPeers = min(minPeers, state.peers.len) - if minPeers == high(int): - minPeers = 0 - return ok(RequestEdgeFilterPeerCount(peerCount: minPeers)), - ).isOkOr: - error "Can't set provider for RequestEdgeFilterPeerCount", error = error - - if self.node.wakuRelay.isNil(): - return self.startEdgeFilterLoops() - - # Core mode: auto-subscribe relay to all shards in autosharding. - if self.node.wakuAutoSharding.isSome(): - let autoSharding = self.node.wakuAutoSharding.get() - let clusterId = autoSharding.clusterId - let numShards = autoSharding.shardCountGenZero - - if numShards > 0: - var clusterPubsubTopics = newSeqOfCap[PubsubTopic](numShards) - - for i in 0 ..< numShards: - let shardObj = RelayShard(clusterId: clusterId, shardId: uint16(i)) - clusterPubsubTopics.add(PubsubTopic($shardObj)) - - self.subscribePubsubTopics(clusterPubsubTopics).isOkOr: - error "Failed to auto-subscribe Relay to cluster shards: ", error = error - else: - info "SubscriptionManager has no AutoSharding configured; skipping auto-subscribe." - - return ok() - -proc stopSubscriptionManager*(self: SubscriptionManager) {.async: (raises: []).} = - if self.node.wakuRelay.isNil(): - await self.stopEdgeFilterLoops() - RequestEdgeShardHealth.clearProvider(self.node.brokerCtx) - RequestEdgeFilterPeerCount.clearProvider(self.node.brokerCtx) diff --git a/waku/node/edge_filter_sub_state.nim b/waku/node/edge_filter_sub_state.nim new file mode 100644 index 000000000..9863f876a --- /dev/null +++ b/waku/node/edge_filter_sub_state.nim @@ -0,0 +1,13 @@ +{.push raises: [].} + +import std/sets +import chronos, libp2p/peerid +import ../waku_core, ./health_monitor/topic_health + +type EdgeFilterSubState* = object + peers*: seq[RemotePeerInfo] + pending*: seq[Future[void]] + pendingPeers*: HashSet[PeerId] + currentHealth*: TopicHealth + +{.pop.} diff --git a/waku/node/health_monitor/node_health_monitor.nim b/waku/node/health_monitor/node_health_monitor.nim index c652f7cea..e5c941191 100644 --- a/waku/node/health_monitor/node_health_monitor.nim +++ b/waku/node/health_monitor/node_health_monitor.nim @@ -14,8 +14,9 @@ import events/health_events, events/peer_events, node/waku_node, + node/node_telemetry, node/peer_manager, - node/kernel_api, + node/waku_node/ping, node/health_monitor/online_monitor, node/health_monitor/health_status, node/health_monitor/health_report, diff --git a/waku/node/kernel_api.nim b/waku/node/kernel_api.nim deleted file mode 100644 index 9d19acb07..000000000 --- a/waku/node/kernel_api.nim +++ /dev/null @@ -1,9 +0,0 @@ -import - ./kernel_api/filter as filter_api, - ./kernel_api/lightpush as lightpush_api, - ./kernel_api/store as store_api, - ./kernel_api/relay as relay_api, - ./kernel_api/peer_exchange as peer_exchange_api, - ./kernel_api/ping as ping_api - -export filter_api, lightpush_api, store_api, relay_api, peer_exchange_api, ping_api diff --git a/waku/node/node_telemetry.nim b/waku/node/node_telemetry.nim new file mode 100644 index 000000000..cd214969c --- /dev/null +++ b/waku/node/node_telemetry.nim @@ -0,0 +1,27 @@ +{.push raises: [].} + +import metrics + +declarePublicGauge waku_version, + "Waku version info (in git describe format)", ["version"] + +declarePublicCounter waku_node_errors, "number of wakunode errors", ["type"] + +declarePublicGauge waku_lightpush_peers, "number of lightpush peers" + +declarePublicGauge waku_filter_peers, "number of filter peers" + +declarePublicGauge waku_store_peers, "number of store peers" + +declarePublicGauge waku_px_peers, + "number of peers (in the node's peerManager) supporting the peer exchange protocol" + +declarePublicCounter waku_node_messages, "number of messages received", ["type"] + +declarePublicHistogram waku_histogram_message_size, + "message size histogram in kB", + buckets = [ + 0.0, 1.0, 3.0, 5.0, 15.0, 50.0, 75.0, 100.0, 125.0, 150.0, 500.0, 700.0, 1000.0, Inf + ] + +{.pop.} diff --git a/waku/node/shard_subscription.nim b/waku/node/shard_subscription.nim new file mode 100644 index 000000000..9801fb32d --- /dev/null +++ b/waku/node/shard_subscription.nim @@ -0,0 +1,11 @@ +{.push raises: [].} + +import std/sets +import ../waku_core + +type ShardSubscription* = object + contentTopics*: HashSet[ContentTopic] + directShardSub*: bool + ## shard subscribed directly (PubsubSub), independent of content-topic interest + +{.pop.} diff --git a/waku/node/subscription_manager.nim b/waku/node/subscription_manager.nim new file mode 100644 index 000000000..8bcb7bb46 --- /dev/null +++ b/waku/node/subscription_manager.nim @@ -0,0 +1,711 @@ +import std/[sequtils, sets, tables, options], chronos, chronicles, metrics, results +import libp2p/[peerid, peerinfo] +import brokers/broker_context + +import + waku/[ + waku_core, + waku_core/topics/sharding, + node/waku_node, + node/node_telemetry, + waku_relay, + waku_archive, + waku_store_sync, + waku_filter_v2/common as filter_common, + waku_filter_v2/client as filter_client, + waku_filter_v2/protocol as filter_protocol, + events/health_events, + events/message_events, + events/peer_events, + requests/health_requests, + node/peer_manager, + node/health_monitor/topic_health, + node/health_monitor/connection_status, + ] + +{.push raises: [].} + +proc registerRelayHandler( + node: WakuNode, shard: PubsubTopic, appHandler: WakuRelayHandler = nil +): bool = + ## Returns true iff we did a new (and only) subscription for this shard in GossipSub. + let alreadySubscribed = node.wakuRelay.isSubscribed(shard) + + if not appHandler.isNil(): + if not alreadySubscribed or not node.legacyAppHandlers.hasKey(shard): + node.legacyAppHandlers[shard] = appHandler + else: + debug "Legacy appHandler already exists for active shard, ignoring new handler", + shard + + if alreadySubscribed: + return false + + proc traceHandler(topic: PubsubTopic, msg: WakuMessage) {.async, gcsafe.} = + let msgSizeKB = msg.payload.len / 1000 + + waku_node_messages.inc(labelValues = ["relay"]) + waku_histogram_message_size.observe(msgSizeKB) + + proc filterHandler(topic: PubsubTopic, msg: WakuMessage) {.async, gcsafe.} = + if node.wakuFilter.isNil(): + return + + await node.wakuFilter.handleMessage(topic, msg) + + proc archiveHandler(topic: PubsubTopic, msg: WakuMessage) {.async, gcsafe.} = + if node.wakuArchive.isNil(): + return + + await node.wakuArchive.handleMessage(topic, msg) + + proc syncHandler(topic: PubsubTopic, msg: WakuMessage) {.async, gcsafe.} = + if node.wakuStoreReconciliation.isNil(): + return + + node.wakuStoreReconciliation.messageIngress(topic, msg) + + proc internalHandler(topic: PubsubTopic, msg: WakuMessage) {.async, gcsafe.} = + MessageSeenEvent.emit(node.brokerCtx, topic, msg) + + let uniqueTopicHandler = proc( + topic: PubsubTopic, msg: WakuMessage + ): Future[void] {.async, gcsafe.} = + await traceHandler(topic, msg) + await filterHandler(topic, msg) + await archiveHandler(topic, msg) + await syncHandler(topic, msg) + await internalHandler(topic, msg) + + if node.legacyAppHandlers.hasKey(topic) and not node.legacyAppHandlers[topic].isNil(): + await node.legacyAppHandlers[topic](topic, msg) + + node.wakuRelay.subscribe(shard, uniqueTopicHandler) + return true + +proc unregisterRelayHandler(node: WakuNode, shard: PubsubTopic): bool = + ## Returns true iff we had a subscription for this shard in GossipSub and it was removed. + if node.legacyAppHandlers.hasKey(shard): + node.legacyAppHandlers.del(shard) + + if node.wakuRelay.isSubscribed(shard): + node.wakuRelay.unsubscribe(shard) + return true + return false + +proc doRelaySubscribe( + node: WakuNode, shard: PubsubTopic, appHandler: WakuRelayHandler = nil +): bool = + ## Subscribes the node to a shard. + ## Returns true if we actually subscribed (transitioned from unsubscribed to subscribed). + ## Emit the shard subscription event if we actually subscribed. + let installed = node.registerRelayHandler(shard, appHandler) + if installed: + node.topicSubscriptionQueue.emit((kind: PubsubSub, topic: shard)) + return installed + +proc doRelayUnsubscribe(node: WakuNode, shard: PubsubTopic): bool = + ## Unsubscribes the node from a shard. + ## Returns true if we actually unsubscribed (transitioned from subscribed to unsubscribed). + ## Emit the shard unsubscription event if we actually unsubscribed. + let unsubscribed = node.unregisterRelayHandler(shard) + if unsubscribed: + node.topicSubscriptionQueue.emit((kind: PubsubUnsub, topic: shard)) + return unsubscribed + +proc new*(T: type SubscriptionManager, node: WakuNode): T = + T( + node: node, + shards: initTable[PubsubTopic, ShardSubscription](), + edgeFilterSubStates: initTable[PubsubTopic, EdgeFilterSubState](), + edgeFilterWakeup: newAsyncEvent(), + ) + +func wanted(entry: ShardSubscription): bool = + ## True if the shard has content-topic interest or a direct subscription. + return entry.contentTopics.len > 0 or entry.directShardSub + +proc isContentSubscribed*( + self: SubscriptionManager, shard: PubsubTopic, contentTopic: ContentTopic +): bool = + self.shards.withValue(shard, sub): + return contentTopic in sub.contentTopics + return false + +iterator subscribedContentTopics*( + self: SubscriptionManager +): (PubsubTopic, HashSet[ContentTopic]) = + ## Yields each shard with its non-empty content-topic set. + for shard, sub in self.shards.pairs: + if sub.contentTopics.len > 0: + yield (shard, sub.contentTopics) + +func toTopicHealth*(peersCount: int): TopicHealth = + if peersCount >= HealthyThreshold: + return TopicHealth.SUFFICIENTLY_HEALTHY + elif peersCount > 0: + return TopicHealth.MINIMALLY_HEALTHY + else: + return TopicHealth.UNHEALTHY + +proc edgeFilterPeerCount*(self: SubscriptionManager, shard: PubsubTopic): int = + self.edgeFilterSubStates.withValue(shard, state): + return state.peers.len + return 0 + +proc getShardForContentTopic( + self: SubscriptionManager, topic: ContentTopic +): Result[PubsubTopic, string] = + if self.node.wakuAutoSharding.isSome(): + let shardObj = ?self.node.wakuAutoSharding.get().getShard(topic) + return ok($shardObj) + + return err("autosharding is not configured; pass an explicit shard") + +proc subscribeShard*( + self: SubscriptionManager, shard: PubsubTopic, handler: WakuRelayHandler = nil +): Result[void, string] = + ## Subscribes to the shard directly and joins the relay mesh. + var added = false + self.shards.withValue(shard, entry): + if not entry.directShardSub: + entry.directShardSub = true + added = true + do: + self.shards[shard] = ShardSubscription( + contentTopics: initHashSet[ContentTopic](), directShardSub: true + ) + added = true + if added: + self.edgeFilterWakeup.fire() + if not isNil(self.node.wakuRelay): + discard self.node.doRelaySubscribe(shard, handler) + return ok() + +proc unsubscribeShard*( + self: SubscriptionManager, shard: PubsubTopic +): Result[void, string] = + ## Drops the direct shard subscription; unsubscribes the mesh if no content topic wants it. + var removed = false + var shardEmpty = false + self.shards.withValue(shard, entry): + if entry.directShardSub: + entry.directShardSub = false + removed = true + shardEmpty = not entry[].wanted() + if removed: + self.edgeFilterWakeup.fire() + if shardEmpty: + self.shards.del(shard) + if not isNil(self.node.wakuRelay): + discard self.node.doRelayUnsubscribe(shard) + return ok() + +proc subscribe*( + self: SubscriptionManager, + shard: PubsubTopic, + contentTopic: ContentTopic, + handler: WakuRelayHandler = nil, +): Result[void, string] = + ## Adds content-topic interest on the shard and joins the relay mesh. + var added = false + self.shards.withValue(shard, entry): + if contentTopic notin entry.contentTopics: + entry.contentTopics.incl(contentTopic) + added = true + do: + var entry = ShardSubscription(contentTopics: initHashSet[ContentTopic]()) + entry.contentTopics.incl(contentTopic) + self.shards[shard] = entry + added = true + if added: + self.edgeFilterWakeup.fire() + if not isNil(self.node.wakuRelay): + discard self.node.doRelaySubscribe(shard, handler) + return ok() + +proc unsubscribe*( + self: SubscriptionManager, shard: PubsubTopic, contentTopic: ContentTopic +): Result[void, string] = + ## Drops content-topic interest on the shard; unsubscribes the mesh if nothing else wants it. + var removed = false + var shardEmpty = false + self.shards.withValue(shard, entry): + if contentTopic in entry.contentTopics: + entry.contentTopics.excl(contentTopic) + removed = true + shardEmpty = not entry[].wanted() + if removed: + self.edgeFilterWakeup.fire() + if shardEmpty: + self.shards.del(shard) + if not isNil(self.node.wakuRelay): + discard self.node.doRelayUnsubscribe(shard) + return ok() + +proc subscribe*(self: SubscriptionManager, topic: ContentTopic): Result[void, string] = + ## Subscribes to a content topic, resolving its shard via autosharding. + let shard = ?self.getShardForContentTopic(topic) + return self.subscribe(shard, topic) + +proc unsubscribe*( + self: SubscriptionManager, topic: ContentTopic +): Result[void, string] = + ## Unsubscribes from a content topic, resolving its shard via autosharding. + let shard = ?self.getShardForContentTopic(topic) + return self.unsubscribe(shard, topic) + +proc unsubscribeAll*( + self: SubscriptionManager, shard: PubsubTopic +): Result[void, string] = + ## Drops every content topic on the shard, then the direct subscription. + var snapshot: seq[ContentTopic] + self.shards.withValue(shard, sub): + snapshot = toSeq(sub.contentTopics) + for contentTopic in snapshot: + ?self.unsubscribe(shard, contentTopic) + return self.unsubscribeShard(shard) + +proc isSubscribed*( + self: SubscriptionManager, topic: ContentTopic +): Result[bool, string] = + let shard = ?self.getShardForContentTopic(topic) + return ok(self.isContentSubscribed(shard, topic)) + +proc subscribeAllAutoshards*(self: SubscriptionManager): Result[void, string] = + ## Subscribes the relay to every shard in the configured autosharding cluster. + if self.node.wakuRelay.isNil() or self.node.wakuAutoSharding.isNone(): + return ok() + + let autoSharding = self.node.wakuAutoSharding.get() + let numShards = autoSharding.shardCountGenZero + if numShards == 0: + return ok() + + for i in 0'u32 ..< numShards: + let shardObj = RelayShard(clusterId: autoSharding.clusterId, shardId: uint16(i)) + self.subscribeShard(PubsubTopic($shardObj)).isOkOr: + error "failed to auto-subscribe relay to cluster shard", + shard = $shardObj, error = error + + ok() + +{.pop.} + +const EdgeFilterSubscribeTimeout = chronos.seconds(15) + ## Timeout for a single filter subscribe/unsubscribe RPC to a service peer. +const EdgeFilterPingTimeout = chronos.seconds(5) + ## Timeout for a filter ping health check. +const EdgeFilterLoopInterval = chronos.seconds(30) + ## Interval for the edge filter health ping loop. +const EdgeFilterSubLoopDebounce = chronos.seconds(1) + ## Debounce delay to coalesce rapid-fire wakeups into a single reconciliation pass. + +type EdgeFilterSubscribeTask = object + peer: RemotePeerInfo + shard: PubsubTopic + topics: seq[ContentTopic] + +proc updateShardHealth( + self: SubscriptionManager, shard: PubsubTopic, state: var EdgeFilterSubState +) = + ## Recompute and emit health for a shard after its peer set changed. + let newHealth = toTopicHealth(state.peers.len) + if newHealth != state.currentHealth: + state.currentHealth = newHealth + EventShardTopicHealthChange.emit(self.node.brokerCtx, shard, newHealth) + +proc removePeer(self: SubscriptionManager, shard: PubsubTopic, peerId: PeerId) = + ## Remove a peer from edgeFilterSubStates for the given shard, + ## update health, and wake the sub loop to filter-subscribe a replacement. + ## Best-effort unsubscribe so the service peer stops pushing to us. + self.edgeFilterSubStates.withValue(shard, state): + var idx = -1 + for i, p in state.peers: + if p.peerId == peerId: + idx = i + break + if idx < 0: + return + + let peer = state.peers[idx] + state.peers.del(idx) + self.updateShardHealth(shard, state[]) + self.edgeFilterWakeup.fire() + + if not self.node.wakuFilterClient.isNil(): + self.shards.withValue(shard, sub): + let ct = toSeq(sub.contentTopics) + if ct.len > 0: + proc doUnsubscribe() {.async.} = + discard await self.node.wakuFilterClient.unsubscribe(peer, shard, ct) + + asyncSpawn doUnsubscribe() + +type SendChunkedFilterRpcKind = enum + FilterSubscribe + FilterUnsubscribe + +proc sendChunkedFilterRpc( + self: SubscriptionManager, + peer: RemotePeerInfo, + shard: PubsubTopic, + topics: seq[ContentTopic], + kind: SendChunkedFilterRpcKind, +): Future[bool] {.async.} = + ## Send a chunked filter subscribe or unsubscribe RPC. Returns true on + ## success. On failure the peer is removed and false is returned. + try: + var i = 0 + while i < topics.len: + let chunk = + topics[i ..< min(i + filter_protocol.MaxContentTopicsPerRequest, topics.len)] + let fut = + case kind + of FilterSubscribe: + self.node.wakuFilterClient.subscribe(peer, shard, chunk) + of FilterUnsubscribe: + self.node.wakuFilterClient.unsubscribe(peer, shard, chunk) + if not (await fut.withTimeout(EdgeFilterSubscribeTimeout)) or fut.read().isErr(): + trace "sendChunkedFilterRpc: chunk failed", + op = kind, shard = shard, peer = peer.peerId + self.removePeer(shard, peer.peerId) + return false + i += filter_protocol.MaxContentTopicsPerRequest + except CatchableError as exc: + debug "sendChunkedFilterRpc: failed", + op = kind, shard = shard, peer = peer.peerId, err = exc.msg + self.removePeer(shard, peer.peerId) + return false + return true + +proc syncFilterDeltas( + self: SubscriptionManager, + peer: RemotePeerInfo, + shard: PubsubTopic, + added: seq[ContentTopic], + removed: seq[ContentTopic], +) {.async.} = + ## Push content topic changes (adds/removes) to an already-tracked peer. + if added.len > 0: + if not await self.sendChunkedFilterRpc(peer, shard, added, FilterSubscribe): + return + + if removed.len > 0: + discard await self.sendChunkedFilterRpc(peer, shard, removed, FilterUnsubscribe) + +proc subscribeFilterPeer( + self: SubscriptionManager, + peer: RemotePeerInfo, + shard: PubsubTopic, + contentTopics: seq[ContentTopic], +) {.async.} = + ## Filter-subscribe to a service peer for all content topics on a shard and + ## start tracking it (note that the filter client dials the peer if not connected). + self.edgeFilterSubStates.withValue(shard, state): + state.pendingPeers.incl(peer.peerId) + + try: + if not await self.sendChunkedFilterRpc(peer, shard, contentTopics, FilterSubscribe): + return + + self.edgeFilterSubStates.withValue(shard, state): + if state.peers.anyIt(it.peerId == peer.peerId): + trace "subscribeFilterPeer: peer already tracked, skipping duplicate", + shard = shard, peer = peer.peerId + return + + state.peers.add(peer) + self.updateShardHealth(shard, state[]) + trace "subscribeFilterPeer: successfully subscribed to all chunks", + shard = shard, peer = peer.peerId, totalPeers = state.peers.len + do: + trace "subscribeFilterPeer: shard removed while subscribing, discarding result", + shard = shard, peer = peer.peerId + finally: + self.edgeFilterSubStates.withValue(shard, state): + state.pendingPeers.excl(peer.peerId) + +proc edgeFilterConnectionLoop(self: SubscriptionManager) {.async.} = + ## Periodically pings all tracked filter service peers to verify they are + ## still alive at the application layer. Peers that fail the ping are removed. + while true: + await sleepAsync(EdgeFilterLoopInterval) + + if self.node.wakuFilterClient.isNil(): + warn "filter client is nil within edge filter connection loop" + continue + + var connected = initTable[PeerId, RemotePeerInfo]() + for state in self.edgeFilterSubStates.values: + for peer in state.peers: + if self.node.peerManager.switch.peerStore.isConnected(peer.peerId): + connected[peer.peerId] = peer + + var alive = initHashSet[PeerId]() + + if connected.len > 0: + var pingTasks: seq[(PeerId, Future[FilterSubscribeResult])] + for peer in connected.values: + pingTasks.add( + (peer.peerId, self.node.wakuFilterClient.ping(peer, EdgeFilterPingTimeout)) + ) + + await allFutures(pingTasks.mapIt(it[1])) + + for (peerId, task) in pingTasks: + if task.read().isOk(): + alive.incl(peerId) + + var changed = false + for shard, state in self.edgeFilterSubStates.mpairs: + let oldLen = state.peers.len + state.peers.keepItIf(it.peerId notin connected or alive.contains(it.peerId)) + + if state.peers.len < oldLen: + changed = true + self.updateShardHealth(shard, state) + trace "Edge Filter health degraded by Ping failure", + shard = shard, new = state.currentHealth + + if changed: + self.edgeFilterWakeup.fire() + +proc selectFilterCandidates( + self: SubscriptionManager, shard: PubsubTopic, exclude: HashSet[PeerId], needed: int +): seq[RemotePeerInfo] = + ## Select filter service peer candidates for a shard. + + # Start with every filter server peer that can serve the shard + var allCandidates = self.node.peerManager.selectPeers( + filter_common.WakuFilterSubscribeCodec, some(shard) + ) + + # Remove all already used in this shard or being filter-subscribed for it + allCandidates.keepItIf(it.peerId notin exclude) + + # Collect peer IDs already tracked on other shards + var trackedOnOther = initHashSet[PeerId]() + for otherShard, otherState in self.edgeFilterSubStates.pairs: + if otherShard != shard: + for peer in otherState.peers: + trackedOnOther.incl(peer.peerId) + + # Prefer peers we already have a connection to first, preserving shuffle + var candidates = + allCandidates.filterIt(it.peerId in trackedOnOther) & + allCandidates.filterIt(it.peerId notin trackedOnOther) + + # We need to return 'needed' peers only + if candidates.len > needed: + candidates.setLen(needed) + return candidates + +proc edgeFilterSubLoop(self: SubscriptionManager) {.async.} = + ## Reconciles filter subscriptions with the desired state from SubscriptionManager. + var lastSynced = initTable[PubsubTopic, HashSet[ContentTopic]]() + + while true: + await self.edgeFilterWakeup.wait() + await sleepAsync(EdgeFilterSubLoopDebounce) + self.edgeFilterWakeup.clear() + trace "edgeFilterSubLoop: woke up" + + if isNil(self.node.wakuFilterClient): + trace "edgeFilterSubLoop: wakuFilterClient is nil, skipping" + continue + + var newSynced = initTable[PubsubTopic, HashSet[ContentTopic]]() + var allShards: HashSet[PubsubTopic] + for shard, sub in self.shards.pairs: + if sub.contentTopics.len > 0: + newSynced[shard] = sub.contentTopics + allShards.incl(shard) + for shard in lastSynced.keys: + allShards.incl(shard) + + trace "edgeFilterSubLoop: desired state", numShards = newSynced.len + + # Step 1: read state across all shards at once and + # create a list of peer filter-subscribe tasks and shard tracking to delete. + + var subscribeTasks: seq[EdgeFilterSubscribeTask] + var shardsToDelete: seq[PubsubTopic] + + for shard in allShards: + # Compute added/removed deltas via direct iteration; no HashSet copies. + var addedTopics: seq[ContentTopic] + var removedTopics: seq[ContentTopic] + newSynced.withValue(shard, curr): + lastSynced.withValue(shard, prev): + for t in curr[]: + if t notin prev[]: + addedTopics.add(t) + for t in prev[]: + if t notin curr[]: + removedTopics.add(t) + do: + for t in curr[]: + addedTopics.add(t) + do: + lastSynced.withValue(shard, prev): + for t in prev[]: + removedTopics.add(t) + + discard self.edgeFilterSubStates.mgetOrPut( + shard, EdgeFilterSubState(currentHealth: TopicHealth.UNHEALTHY) + ) + + self.edgeFilterSubStates.withValue(shard, state): + state.peers.keepItIf( + self.node.peerManager.switch.peerStore.isConnected(it.peerId) + ) + state.pending.keepItIf(not it.finished) + + if addedTopics.len > 0 or removedTopics.len > 0: + for peer in state.peers: + asyncSpawn self.syncFilterDeltas(peer, shard, addedTopics, removedTopics) + + if shard notin newSynced: + shardsToDelete.add(shard) + else: + self.updateShardHealth(shard, state[]) + + let needed = max(0, HealthyThreshold - state.peers.len - state.pending.len) + + if needed > 0: + var tracked: HashSet[PeerId] + for p in state.peers: + tracked.incl(p.peerId) + for p in state.pendingPeers: + tracked.incl(p) + let candidates = self.selectFilterCandidates(shard, tracked, needed) + let toSubscribe = min(needed, candidates.len) + + trace "edgeFilterSubLoop: shard reconciliation", + shard = shard, + num_peers = state.peers.len, + num_pending = state.pending.len, + num_needed = needed, + num_available = candidates.len, + toSubscribe = toSubscribe + + var subscribeTopics: seq[ContentTopic] + newSynced.withValue(shard, curr): + subscribeTopics = toSeq(curr[]) + + for i in 0 ..< toSubscribe: + subscribeTasks.add( + EdgeFilterSubscribeTask( + peer: candidates[i], shard: shard, topics: subscribeTopics + ) + ) + + # Step 2: execute deferred shard tracking deletion and filter-subscribe tasks. + + for shard in shardsToDelete: + self.edgeFilterSubStates.withValue(shard, state): + for fut in state.pending: + if not fut.finished: + await fut.cancelAndWait() + self.edgeFilterSubStates.del(shard) + + for task in subscribeTasks: + let fut = self.subscribeFilterPeer(task.peer, task.shard, task.topics) + self.edgeFilterSubStates.withValue(task.shard, state): + state.pending.add(fut) + + lastSynced = newSynced + +proc startEdgeFilterLoops(self: SubscriptionManager): Result[void, string] = + ## Start the edge filter orchestration loops. + ## Caller must ensure this is only called in edge mode (relay nil, filter client present). + self.peerEventListener = WakuPeerEvent.listen( + self.node.brokerCtx, + proc(evt: WakuPeerEvent) {.async: (raises: []), gcsafe.} = + if evt.kind == WakuPeerEventKind.EventDisconnected: + # We know a peer is gone, so if it was a service filter peer for this + # edge node, remove it from the list of service filter peers for each + # shard it served and re-evaluate shard health for the affected shards. + for shard, state in self.edgeFilterSubStates.mpairs: + let oldLen = state.peers.len + state.peers.keepItIf(it.peerId != evt.peerId) + if state.peers.len < oldLen: + self.updateShardHealth(shard, state) + self.edgeFilterWakeup.fire() + elif evt.kind == WakuPeerEventKind.EventMetadataUpdated: + self.edgeFilterWakeup.fire(), + ).valueOr: + return err("Failed to listen to peer events for edge filter: " & error) + + self.edgeFilterSubLoopFut = self.edgeFilterSubLoop() + self.edgeFilterConnectionLoopFut = self.edgeFilterConnectionLoop() + return ok() + +proc stopEdgeFilterLoops(self: SubscriptionManager) {.async: (raises: []).} = + ## Stop the edge filter orchestration loops and clean up pending futures. + if not isNil(self.edgeFilterSubLoopFut): + await self.edgeFilterSubLoopFut.cancelAndWait() + self.edgeFilterSubLoopFut = nil + + if not isNil(self.edgeFilterConnectionLoopFut): + await self.edgeFilterConnectionLoopFut.cancelAndWait() + self.edgeFilterConnectionLoopFut = nil + + for shard, state in self.edgeFilterSubStates: + for fut in state.pending: + if not fut.finished: + await fut.cancelAndWait() + + await WakuPeerEvent.dropListener(self.node.brokerCtx, self.peerEventListener) + +proc start*(self: SubscriptionManager): Result[void, string] = + let edgeShardHealthRes = RequestEdgeShardHealth.setProvider( + self.node.brokerCtx, + proc(shard: PubsubTopic): Result[RequestEdgeShardHealth, string] = + self.edgeFilterSubStates.withValue(shard, state): + return ok(RequestEdgeShardHealth(health: state.currentHealth)) + return ok(RequestEdgeShardHealth(health: TopicHealth.NOT_SUBSCRIBED)), + ) + self.ownsEdgeShardHealthProvider = edgeShardHealthRes.isOk() + if edgeShardHealthRes.isErr(): + error "Can't set provider for RequestEdgeShardHealth", + error = edgeShardHealthRes.error + + let edgeFilterPeerCountRes = RequestEdgeFilterPeerCount.setProvider( + self.node.brokerCtx, + proc(): Result[RequestEdgeFilterPeerCount, string] = + var minPeers = high(int) + for state in self.edgeFilterSubStates.values: + minPeers = min(minPeers, state.peers.len) + if minPeers == high(int): + minPeers = 0 + return ok(RequestEdgeFilterPeerCount(peerCount: minPeers)), + ) + self.ownsEdgeFilterPeerCountProvider = edgeFilterPeerCountRes.isOk() + if edgeFilterPeerCountRes.isErr(): + error "Can't set provider for RequestEdgeFilterPeerCount", + error = edgeFilterPeerCountRes.error + + # Start Edge workers only when we are in Edge mode (relay not mounted) + # AND the filter client is mounted (otherwise the loops have nothing + # to talk to and just spam "filter client is nil" warnings). + if self.node.wakuRelay.isNil() and not self.node.wakuFilterClient.isNil(): + return self.startEdgeFilterLoops() + + return ok() + +proc stop*(self: SubscriptionManager) {.async: (raises: []).} = + # Stop Edge workers if we started them in `start` (Edge mode + filter client). + if self.node.wakuRelay.isNil() and not self.node.wakuFilterClient.isNil(): + await self.stopEdgeFilterLoops() + + # Only clear providers we actually registered: another SubscriptionManager + # sharing this brokerCtx may have won the race, and clearing its provider + # would leave the broker silently provider-less. + if self.ownsEdgeShardHealthProvider: + RequestEdgeShardHealth.clearProvider(self.node.brokerCtx) + self.ownsEdgeShardHealthProvider = false + if self.ownsEdgeFilterPeerCountProvider: + RequestEdgeFilterPeerCount.clearProvider(self.node.brokerCtx) + self.ownsEdgeFilterPeerCountProvider = false diff --git a/waku/node/waku_metrics.nim b/waku/node/waku_metrics.nim index af74b1532..bb4c10fff 100644 --- a/waku/node/waku_metrics.nim +++ b/waku/node/waku_metrics.nim @@ -4,6 +4,7 @@ import chronicles, chronos, metrics, metrics/chronos_httpserver import waku/[net/auto_port, waku_rln_relay/protocol_metrics as rln_metrics, utils/collector], ./peer_manager, + ./node_telemetry, ./waku_node const LogInterval = 10.minutes diff --git a/waku/node/waku_node.nim b/waku/node/waku_node.nim index 26a2b5a57..6a8826d2a 100644 --- a/waku/node/waku_node.nim +++ b/waku/node/waku_node.nim @@ -60,23 +60,18 @@ import requests/health_requests, events/health_events, events/message_events, + events/peer_events, ], waku/discovery/waku_kademlia, waku/net/[bound_ports, net_config], ./peer_manager, ./health_monitor/health_status, - ./health_monitor/topic_health + ./health_monitor/topic_health, + ./node_telemetry, + ./shard_subscription, + ./edge_filter_sub_state -declarePublicCounter waku_node_messages, "number of messages received", ["type"] - -declarePublicGauge waku_version, - "Waku version info (in git describe format)", ["version"] -declarePublicCounter waku_node_errors, "number of wakunode errors", ["type"] -declarePublicGauge waku_lightpush_peers, "number of lightpush peers" -declarePublicGauge waku_filter_peers, "number of filter peers" -declarePublicGauge waku_store_peers, "number of store peers" -declarePublicGauge waku_px_peers, - "number of peers (in the node's peerManager) supporting the peer exchange protocol" +export shard_subscription, edge_filter_sub_state logScope: topics = "waku node" @@ -94,7 +89,6 @@ const clientId* = "Nimbus Waku v2 node" const WakuNodeVersionString* = "version / git commit hash: " & git_version -# key and crypto modules different type # TODO: Move to application instance (e.g., `WakuNode2`) WakuInfo* = object # NOTE One for simplicity, can extend later as needed @@ -137,11 +131,25 @@ type rateLimitSettings*: ProtocolRateLimitSettings legacyAppHandlers*: Table[PubsubTopic, WakuRelayHandler] ## Kernel API Relay appHandlers (if any) + subscriptionManager*: SubscriptionManager wakuMix*: WakuMix kademliaDiscoveryLoop*: Future[void] wakuKademlia*: WakuKademlia ports*: BoundPorts + SubscriptionManager* = ref object of RootObj + node*: WakuNode + shards*: Table[PubsubTopic, ShardSubscription] + edgeFilterSubStates*: Table[PubsubTopic, EdgeFilterSubState] + edgeFilterWakeup*: AsyncEvent + edgeFilterSubLoopFut*: Future[void] + edgeFilterConnectionLoopFut*: Future[void] + peerEventListener*: WakuPeerEventListener + ownsEdgeShardHealthProvider*: bool + ownsEdgeFilterPeerCountProvider*: bool + +import ./subscription_manager + proc deduceRelayShard( node: WakuNode, contentTopic: ContentTopic, @@ -230,6 +238,8 @@ proc new*( peerManager.setShardGetter(node.getShardsGetter(@[])) + node.subscriptionManager = SubscriptionManager.new(node) + return node proc peerInfo*(node: WakuNode): PeerInfo = @@ -600,6 +610,9 @@ proc start*(node: WakuNode) {.async.} = node.startProvidersAndListeners() + node.subscriptionManager.start().isOkOr: + error "failed to start subscription manager", error = error + if not zeroPortPresent: updateAnnouncedAddrWithPrimaryIpAddr(node).isOkOr: error "failed update announced addr", error = $error @@ -611,6 +624,8 @@ proc start*(node: WakuNode) {.async.} = proc stop*(node: WakuNode) {.async.} = ## By stopping the switch we are stopping all the underlying mounted protocols + await node.subscriptionManager.stop() + node.stopProvidersAndListeners() ## NOTE: This will dispatch gossipsub stop to the WakuRelay.stop method override diff --git a/waku/node/kernel_api/filter.nim b/waku/node/waku_node/filter.nim similarity index 99% rename from waku/node/kernel_api/filter.nim rename to waku/node/waku_node/filter.nim index 948035f14..0db4875b0 100644 --- a/waku/node/kernel_api/filter.nim +++ b/waku/node/waku_node/filter.nim @@ -21,6 +21,7 @@ import import ../waku_node, + ../node_telemetry, ../../waku_core, ../../waku_core/topics/sharding, ../../waku_filter_v2, diff --git a/waku/node/kernel_api/lightpush.nim b/waku/node/waku_node/lightpush.nim similarity index 100% rename from waku/node/kernel_api/lightpush.nim rename to waku/node/waku_node/lightpush.nim diff --git a/waku/node/kernel_api/peer_exchange.nim b/waku/node/waku_node/peer_exchange.nim similarity index 99% rename from waku/node/kernel_api/peer_exchange.nim rename to waku/node/waku_node/peer_exchange.nim index a4bec727b..1cb6bd3bb 100644 --- a/waku/node/kernel_api/peer_exchange.nim +++ b/waku/node/waku_node/peer_exchange.nim @@ -19,6 +19,7 @@ import import ../waku_node, + ../node_telemetry, ../../waku_peer_exchange, ../../waku_core, ../peer_manager, diff --git a/waku/node/kernel_api/ping.nim b/waku/node/waku_node/ping.nim similarity index 100% rename from waku/node/kernel_api/ping.nim rename to waku/node/waku_node/ping.nim diff --git a/waku/node/kernel_api/relay.nim b/waku/node/waku_node/relay.nim similarity index 63% rename from waku/node/kernel_api/relay.nim rename to waku/node/waku_node/relay.nim index f1b80cf19..30fc22ec3 100644 --- a/waku/node/kernel_api/relay.nim +++ b/waku/node/waku_node/relay.nim @@ -29,90 +29,18 @@ import waku_store_sync, waku_rln_relay, node/waku_node, + node/subscription_manager, node/peer_manager, events/message_events, ] export waku_relay.WakuRelayHandler -declarePublicHistogram waku_histogram_message_size, - "message size histogram in kB", - buckets = [ - 0.0, 1.0, 3.0, 5.0, 15.0, 50.0, 75.0, 100.0, 125.0, 150.0, 500.0, 700.0, 1000.0, Inf - ] - logScope: topics = "waku node relay api" ## Waku relay -proc registerRelayHandler( - node: WakuNode, topic: PubsubTopic, appHandler: WakuRelayHandler = nil -): bool = - ## Registers the only handler for the given topic. - ## Notice that this handler internally calls other handlers, such as filter, - ## archive, etc, plus the handler provided by the application. - ## Returns `true` if a mesh subscription was created or `false` if the relay - ## was already subscribed to the topic. - - let alreadySubscribed = node.wakuRelay.isSubscribed(topic) - - if not appHandler.isNil(): - if not alreadySubscribed or not node.legacyAppHandlers.hasKey(topic): - node.legacyAppHandlers[topic] = appHandler - else: - debug "Legacy appHandler already exists for active PubsubTopic, ignoring new handler", - topic = topic - - if alreadySubscribed: - return false - - proc traceHandler(topic: PubsubTopic, msg: WakuMessage) {.async, gcsafe.} = - let msgSizeKB = msg.payload.len / 1000 - - waku_node_messages.inc(labelValues = ["relay"]) - waku_histogram_message_size.observe(msgSizeKB) - - proc filterHandler(topic: PubsubTopic, msg: WakuMessage) {.async, gcsafe.} = - if node.wakuFilter.isNil(): - return - - await node.wakuFilter.handleMessage(topic, msg) - - proc archiveHandler(topic: PubsubTopic, msg: WakuMessage) {.async, gcsafe.} = - if node.wakuArchive.isNil(): - return - - await node.wakuArchive.handleMessage(topic, msg) - - proc syncHandler(topic: PubsubTopic, msg: WakuMessage) {.async, gcsafe.} = - if node.wakuStoreReconciliation.isNil(): - return - - node.wakuStoreReconciliation.messageIngress(topic, msg) - - proc internalHandler(topic: PubsubTopic, msg: WakuMessage) {.async, gcsafe.} = - MessageSeenEvent.emit(node.brokerCtx, topic, msg) - - let uniqueTopicHandler = proc( - topic: PubsubTopic, msg: WakuMessage - ): Future[void] {.async, gcsafe.} = - await traceHandler(topic, msg) - await filterHandler(topic, msg) - await archiveHandler(topic, msg) - await syncHandler(topic, msg) - await internalHandler(topic, msg) - - # Call the legacy (kernel API) app handler if it exists. - # Normally, hasKey is false and the MessageSeenEvent bus (new API) is used instead. - # But we need to support legacy behavior (kernel API use), hence this. - # NOTE: We can delete `legacyAppHandlers` if instead we refactor WakuRelay to support multiple - # PubsubTopic handlers, since that's actually supported by libp2p PubSub (bigger refactor...) - if node.legacyAppHandlers.hasKey(topic) and not node.legacyAppHandlers[topic].isNil(): - await node.legacyAppHandlers[topic](topic, msg) - - node.wakuRelay.subscribe(topic, uniqueTopicHandler) - proc getTopicOfSubscriptionEvent( node: WakuNode, subscription: SubscriptionEvent ): Result[(PubsubTopic, Option[ContentTopic]), string] = @@ -143,21 +71,15 @@ proc subscribe*( error "Invalid API call to `subscribe`. WakuRelay not mounted." return err("Invalid API call to `subscribe`. WakuRelay not mounted.") - let (pubsubTopic, contentTopicOp) = getTopicOfSubscriptionEvent(node, subscription).valueOr: + let (pubsubTopic, _) = getTopicOfSubscriptionEvent(node, subscription).valueOr: error "Failed to decode subscription event", error = error return err("Failed to decode subscription event: " & error) - if node.registerRelayHandler(pubsubTopic, handler): - info "subscribe", pubsubTopic, contentTopicOp - node.topicSubscriptionQueue.emit((kind: PubsubSub, topic: pubsubTopic)) - else: - if isNil(handler): - warn "No-effect API call to subscribe. Already subscribed to topic", pubsubTopic - else: - info "subscribe (was already subscribed in the mesh; appHandler set)", - pubsubTopic = pubsubTopic - - return ok() + # strict version + #if contentTopicOp.isSome(): + # return + # node.subscriptionManager.subscribe(pubsubTopic, contentTopicOp.get(), handler) + return node.subscriptionManager.subscribeShard(pubsubTopic, handler) proc unsubscribe*( node: WakuNode, subscription: SubscriptionEvent @@ -170,26 +92,14 @@ proc unsubscribe*( error "Invalid API call to `unsubscribe`. WakuRelay not mounted." return err("Invalid API call to `unsubscribe`. WakuRelay not mounted.") - let (pubsubTopic, contentTopicOp) = getTopicOfSubscriptionEvent(node, subscription).valueOr: + let (pubsubTopic, _) = getTopicOfSubscriptionEvent(node, subscription).valueOr: error "Failed to decode unsubscribe event", error = error return err("Failed to decode unsubscribe event: " & error) - let hadHandler = node.legacyAppHandlers.hasKey(pubsubTopic) - if hadHandler: - node.legacyAppHandlers.del(pubsubTopic) - - if node.wakuRelay.isSubscribed(pubsubTopic): - info "unsubscribe", pubsubTopic, contentTopicOp - node.wakuRelay.unsubscribe(pubsubTopic) - node.topicSubscriptionQueue.emit((kind: PubsubUnsub, topic: pubsubTopic)) - else: - if not hadHandler: - warn "No-effect API call to `unsubscribe`. Was not subscribed", pubsubTopic - else: - info "unsubscribe (was not subscribed in the mesh; appHandler removed)", - pubsubTopic = pubsubTopic - - return ok() + # strict version + #if contentTopicOp.isSome(): + # return node.subscriptionManager.unsubscribe(pubsubTopic, contentTopicOp.get()) + return node.subscriptionManager.unsubscribeAll(pubsubTopic) proc isSubscribed*( node: WakuNode, subscription: SubscriptionEvent diff --git a/waku/node/kernel_api/store.nim b/waku/node/waku_node/store.nim similarity index 100% rename from waku/node/kernel_api/store.nim rename to waku/node/waku_node/store.nim diff --git a/waku/persistency/backend_comm.nim b/waku/persistency/backend_comm.nim index dd7e71297..193e52825 100644 --- a/waku/persistency/backend_comm.nim +++ b/waku/persistency/backend_comm.nim @@ -68,7 +68,7 @@ proc mtMarshalValue*( of txPut: if not mtMarshalValue(buf, cap, value.payload, pos): return false - of txDelete: + of txDelete, txDeletePrefix: discard return true @@ -93,6 +93,8 @@ proc mtUnmarshalValue*( value = TxOp(category: category, key: key, kind: txPut, payload: payload) of txDelete: value = TxOp(category: category, key: key, kind: txDelete) + of txDeletePrefix: + value = TxOp(category: category, key: key, kind: txDeletePrefix) return true EventBroker(mt): diff --git a/waku/persistency/backend_sqlite.nim b/waku/persistency/backend_sqlite.nim index 6851febc1..95757bc2c 100644 --- a/waku/persistency/backend_sqlite.nim +++ b/waku/persistency/backend_sqlite.nim @@ -7,7 +7,7 @@ import std/options import results, sqlite3_abi import ../common/databases/[common, db_sqlite] -import ./[types, schema] +import ./[types, keys, schema] type KvBackend* = ref object @@ -121,6 +121,37 @@ proc close*(b: KvBackend) = b.db.close() b.db = nil +proc deletePrefix( + b: KvBackend, category: string, prefix: Key +): Result[void, PersistencyError] = + let rng = prefixRange(prefix) + let openEnded = bytes(rng.stop).len == 0 + let sql = + if openEnded: + "DELETE FROM kv WHERE category = ? AND key >= ?;" + else: + "DELETE FROM kv WHERE category = ? AND key >= ? AND key < ?;" + var s: ptr sqlite3_stmt + let rc = sqlite3_prepare_v2(b.db.env, sql.cstring, sql.len.cint, addr s, nil) + if rc != SQLITE_OK: + return err(toErr("deletePrefix prepare: " & $sqlite3_errstr(rc))) + defer: + discard sqlite3_finalize(s) + var bc = bindBlob(s, 1.cint, catBytes(category)) + if bc != SQLITE_OK: + return err(toErr("deletePrefix bind cat: " & $sqlite3_errstr(bc))) + bc = bindBlob(s, 2.cint, keyBytes(rng.start)) + if bc != SQLITE_OK: + return err(toErr("deletePrefix bind start: " & $sqlite3_errstr(bc))) + if not openEnded: + bc = bindBlob(s, 3.cint, keyBytes(rng.stop)) + if bc != SQLITE_OK: + return err(toErr("deletePrefix bind stop: " & $sqlite3_errstr(bc))) + let v = sqlite3_step(s) + if v != SQLITE_DONE: + return err(toErr("deletePrefix step: " & $sqlite3_errstr(v))) + return ok() + proc applyOne(b: KvBackend, op: TxOp): Result[void, PersistencyError] = case op.kind of txPut: @@ -131,6 +162,8 @@ proc applyOne(b: KvBackend, op: TxOp): Result[void, PersistencyError] = let r = b.deleteStmt.exec((catBytes(op.category), keyBytes(op.key))) if r.isErr: return err(toErr("delete failed: " & r.error)) + of txDeletePrefix: + ?b.deletePrefix(op.category, op.key) return ok() proc execSql(b: KvBackend, sql: string): Result[void, PersistencyError] = diff --git a/waku/persistency/persistency.nim b/waku/persistency/persistency.nim index 916f3ac8b..1e070dbb5 100644 --- a/waku/persistency/persistency.nim +++ b/waku/persistency/persistency.nim @@ -284,6 +284,11 @@ proc persistPut*( proc persistDelete*(t: Job, category: string, key: Key): Future[void] {.async.} = await persist(t, TxOp(category: category, key: key, kind: txDelete)) +proc persistDeletePrefix*( + t: Job, category: string, prefix: Key +): Future[void] {.async.} = + await persist(t, TxOp(category: category, key: prefix, kind: txDeletePrefix)) + proc persistEncoded*[T]( t: Job, category: string, key: Key, value: T ): Future[void] {.async.} = @@ -335,6 +340,13 @@ proc persistDelete*( if not j.isNil(): await j.persistDelete(category, key) +proc persistDeletePrefix*( + p: Persistency, jobId: string, category: string, prefix: Key +): Future[void] {.async.} = + let j = p.jobOrWarn(jobId) + if not j.isNil(): + await j.persistDeletePrefix(category, prefix) + proc persistEncoded*[T]( p: Persistency, jobId: string, category: string, key: Key, value: T ): Future[void] {.async.} = diff --git a/waku/persistency/sds_persistency.nim b/waku/persistency/sds_persistency.nim new file mode 100644 index 000000000..3c44f7802 --- /dev/null +++ b/waku/persistency/sds_persistency.nim @@ -0,0 +1,176 @@ +## Adapter that materialises the SDS `Persistence` contract (nim-sds 0.3.0, +## snapshot model) on top of a waku-persistency `Job`. One `Job` (== one +## SQLite file, one worker thread) services all channels for a given SDS +## context; rows are namespaced by category and the channelId is the first +## key component so per-channel prefix scans stay cheap. +## +## ## Snapshot contract (nim-sds 0.3.0) +## +## The fine-grained per-row callbacks of 0.2.4 are gone. SDS now persists via +## five procs, all `Future[Result[void, string]]` (load returns +## `Result[ChannelData, string]`), `{.async: (raises: []), gcsafe.}`: +## +## * **`saveChannelMeta`** — the complete fast-changing per-channel state +## (lamport clock, outgoing/incoming buffers, both SDS-R repair buffers) +## as ONE blob. Idempotent; a missed write self-heals on the next save. +## * **`updateHistory`** — append newly-delivered messages / evict the +## oldest past the cap, applied as one transactional batch. +## * **`loadChannel`** — bootstrap: returns the prior `ChannelData` +## (meta + ordered message history) or an empty one. Surfaces errors. +## * **`dropChannel`** — wipe all state for a channel. Surfaces errors. +## +## Failure policy mirrors the interface docs: save/update/hint are non-fatal +## (we log and still return the error string); load/drop are durability-intent +## and propagate their error to the caller. +## +## ## Codec +## +## The blob transform is owned by nim-sds: `ChannelMeta` round-trips through +## `sds/snapshot_codec` (protobuf, schema-versioned — refuses unknown +## versions), and each persisted `SdsMessage` log row through the SDS wire +## codec in `sds/protobuf`. We do not maintain a second codec for these +## shapes (the previous `payload_codec`/`BlobCodec` path is retired). +## +## ## Retrieval hints +## +## `setRetrievalHint` is intentionally a no-op: persisted hints are never read +## back — `loadChannel` returns `ChannelData` (meta + messageHistory) with no +## hint field, and `ChannelMeta` carries none. Hints are supplied live via the +## `onRetrievalHint` provider, so persisting them would be write-only dead +## data. The closure still exists because the field is required by the +## `Persistence` object (SDS calls it from `getRecentHistoryEntries`). +## +## ## Storage layout +## +## | Category | Key | Value | +## |---------------|--------------------------|----------------------------------------| +## | `sds.meta` | `key(channelId)` | `ChannelMeta` (snapshot_codec protobuf)| +## | `sds.log` | `key(channelId, msgId)` | `SdsMessage` (sds wire protobuf) | +## +## `messageHistory` is reconstructed in memory by sorting on +## `(lamportTimestamp, messageId)` — the same total order SDS uses for +## delivery (see sds/sds_utils.nim). + +{.push raises: [].} + +import std/[algorithm, options] +import chronos, chronicles, results +import libp2p/protobuf/minprotobuf +import ./persistency +import ./keys +import types/persistence +import snapshot_codec +import protobuf + +export persistence, persistency + +logScope: + topics = "sds-persistency" + +const + CatMeta* = "sds.meta" + CatLog* = "sds.log" + +# ── Public factory ────────────────────────────────────────────────────── + +proc newSdsPersistence*(job: Job): Persistence {.gcsafe, raises: [].} = + ## Build an SDS `Persistence` value backed by ``job``. One Job services + ## all channels — channelId is part of every key. + ## + ## The closures capture ``job`` by ref. They must be invoked from a thread + ## that owns a running chronos loop (the SDS context's worker thread + ## satisfies this). + doAssert not job.isNil, "newSdsPersistence: job is nil" + + # Built field-by-field via assignment rather than an object literal: every + # field is an async closure whose body uses `await`/`return` statements, + # which cannot be followed by the `,` field separator a `Persistence(..)` + # literal would require. Assignments have no separator, so bodies stay plain. + var persistence = Persistence() + + persistence.saveChannelMeta = proc( + channelId: SdsChannelID, meta: ChannelMeta + ): Future[Result[void, string]] {.async: (raises: []), gcsafe.} = + try: + await job.persistPut(CatMeta, toKey(channelId), encode(meta).buffer) + return ok() + except CatchableError as e: + warn "sds-persistency: saveChannelMeta failed", channelId, err = e.msg + return err(e.msg) + + persistence.updateHistory = proc( + channelId: SdsChannelID, update: HistoryUpdate + ): Future[Result[void, string]] {.async: (raises: []), gcsafe.} = + if update.isEmpty: + return ok() + # One transactional batch: append rows (txPut) and evictions (txDelete). + var ops = newSeq[TxOp]() + for m in update.append: + ops.add TxOp( + category: CatLog, + key: key(channelId, m.messageId), + kind: txPut, + payload: encode(m).buffer, + ) + for id in update.evict: + ops.add TxOp(category: CatLog, key: key(channelId, id), kind: txDelete) + try: + await job.persist(ops) + return ok() + except CatchableError as e: + warn "sds-persistency: updateHistory failed", + channelId, appended = update.append.len, evicted = update.evict.len, err = e.msg + return err(e.msg) + + persistence.loadChannel = proc( + channelId: SdsChannelID + ): Future[Result[ChannelData, string]] {.async: (raises: []), gcsafe.} = + let chanKey = toKey(channelId) + var data = ChannelData.init() + try: + block meta: + let opt = (await job.get(CatMeta, chanKey)).valueOr: + return err("loadChannel: get meta: " & $error) + if opt.isSome: + # schema-versioned decode; refuses unknown versions loudly. + data.meta = ChannelMeta.decode(opt.get).valueOr: + return err("loadChannel: corrupt or unsupported ChannelMeta blob") + + block history: + let rows = (await job.scanPrefix(CatLog, chanKey)).valueOr: + return err("loadChannel: scan log: " & $error) + var msgs = newSeq[SdsMessage]() + for row in rows: + let m = SdsMessage.decode(row.payload).valueOr: + warn "sds-persistency: skipping undecodable log row", channelId + continue + msgs.add(m) + msgs.sort do(a, b: SdsMessage) -> int: + result = cmp(a.lamportTimestamp, b.lamportTimestamp) + if result == 0: + result = cmp(a.messageId, b.messageId) + data.messageHistory = msgs + + return ok(data) + except CatchableError as e: + return err("loadChannel: " & e.msg) + + persistence.dropChannel = proc( + channelId: SdsChannelID + ): Future[Result[void, string]] {.async: (raises: []), gcsafe.} = + let chanKey = toKey(channelId) + try: + await job.persist( + @[ + TxOp(category: CatLog, key: chanKey, kind: txDeletePrefix), + TxOp(category: CatMeta, key: chanKey, kind: txDelete), + ] + ) + return ok() + except CatchableError as e: + error "sds-persistency: dropChannel failed", channelId, err = e.msg + return err(e.msg) + + return persistence + +{.pop.} diff --git a/waku/persistency/types.nim b/waku/persistency/types.nim index 4c4c2de3f..0fdf12af0 100644 --- a/waku/persistency/types.nim +++ b/waku/persistency/types.nim @@ -19,6 +19,7 @@ type TxOpKind* = enum txPut txDelete + txDeletePrefix TxOp* = object category*: string @@ -28,6 +29,8 @@ type payload*: seq[byte] of txDelete: discard + of txDeletePrefix: + discard PersistencyErrorKind* = enum peBackend diff --git a/waku/requests/health_requests.nim b/waku/requests/health_requests.nim index d48b3278f..ccf08f83d 100644 --- a/waku/requests/health_requests.nim +++ b/waku/requests/health_requests.nim @@ -38,14 +38,14 @@ RequestBroker: proc signature(protocol: WakuProtocol): Future[Result[RequestProtocolHealth, string]] -# Get edge filter health for a single shard (set by DeliveryService when edge mode is active) +# Get edge filter health for a single shard (set when edge mode is active) RequestBroker(sync): type RequestEdgeShardHealth* = object health*: TopicHealth proc signature(shard: PubsubTopic): Result[RequestEdgeShardHealth, string] -# Get edge filter confirmed peer count (set by DeliveryService when edge mode is active) +# Get edge filter confirmed peer count (set when edge mode is active) RequestBroker(sync): type RequestEdgeFilterPeerCount* = object peerCount*: int diff --git a/waku/rest_api/endpoint/builder.nim b/waku/rest_api/endpoint/builder.nim index 9b4ecf662..16cdde988 100644 --- a/waku/rest_api/endpoint/builder.nim +++ b/waku/rest_api/endpoint/builder.nim @@ -4,6 +4,7 @@ import net, tables import presto import waku/waku_node, + waku/node/health_monitor, waku/discovery/waku_discv5, waku/rest_api/message_cache, waku/rest_api/handlers, diff --git a/waku/rest_api/endpoint/health/handlers.nim b/waku/rest_api/endpoint/health/handlers.nim index 865133245..dc7588d16 100644 --- a/waku/rest_api/endpoint/health/handlers.nim +++ b/waku/rest_api/endpoint/health/handlers.nim @@ -1,7 +1,8 @@ {.push raises: [].} import chronicles, json_serialization, presto/route -import ../../../waku_node, ../responses, ../serdes, ./types +import + ../../../waku_node, ../../../node/health_monitor, ../responses, ../serdes, ./types logScope: topics = "waku node rest health_api" diff --git a/waku/rest_api/endpoint/health/types.nim b/waku/rest_api/endpoint/health/types.nim index 88fa736a8..4f85ebde5 100644 --- a/waku/rest_api/endpoint/health/types.nim +++ b/waku/rest_api/endpoint/health/types.nim @@ -3,7 +3,7 @@ import results import chronicles, json_serialization, json_serialization/std/options import ../serdes -import waku/[waku_node, api/types] +import waku/[waku_node, api/types, node/health_monitor] #### Serialization and deserialization diff --git a/waku/waku_node.nim b/waku/waku_node.nim index c8b13d4ea..674ac96e6 100644 --- a/waku/waku_node.nim +++ b/waku/waku_node.nim @@ -1,8 +1,13 @@ import - ./net/net_config, ./node/waku_switch as switch, ./node/waku_node as node, - ./node/health_monitor as health_monitor, - ./node/kernel_api as kernel_api + ./node/waku_node/filter as filter_api, + ./node/waku_node/lightpush as lightpush_api, + ./node/waku_node/store as store_api, + ./node/waku_node/relay as relay_api, + ./node/waku_node/peer_exchange as peer_exchange_api, + ./node/waku_node/ping as ping_api -export net_config, switch, node, health_monitor, kernel_api +export + switch, node, filter_api, lightpush_api, store_api, relay_api, peer_exchange_api, + ping_api diff --git a/waku/waku_persistency.nim b/waku/waku_persistency.nim new file mode 100644 index 000000000..5eb94e3f0 --- /dev/null +++ b/waku/waku_persistency.nim @@ -0,0 +1,3 @@ +import waku/persistency/persistency + +export persistency