From a30d6a8aae84e80585784ccd2e621561514ba7cc Mon Sep 17 00:00:00 2001 From: Hanno Cornelius <68783915+jm-clius@users.noreply.github.com> Date: Tue, 23 Mar 2021 10:04:51 +0200 Subject: [PATCH] Improvement/bridge improvements (#429) * General test, import and log improvements * Bridge improvements --- tests/v2/test_jsonrpc_waku.nim | 2 ++ tests/v2/test_message_store.nim | 1 + tests/v2/test_peer_manager.nim | 2 ++ tests/v2/test_waku_bridge.nim | 17 ++++++++-- tests/v2/test_waku_rln_relay.nim | 2 ++ tests/v2/test_waku_swap.nim | 2 ++ tests/v2/test_web3.nim | 2 ++ waku/common/wakubridge.nim | 33 +++++++++++-------- waku/v2/node/jsonrpc/admin_api.nim | 3 ++ waku/v2/node/jsonrpc/debug_api.nim | 3 ++ waku/v2/node/jsonrpc/filter_api.nim | 3 ++ waku/v2/node/jsonrpc/jsonrpc_utils.nim | 2 +- waku/v2/node/jsonrpc/private_api.nim | 3 ++ waku/v2/node/jsonrpc/relay_api.nim | 3 ++ waku/v2/node/jsonrpc/store_api.nim | 3 ++ .../waku_rln_relay/waku_rln_relay_utils.nim | 2 +- 16 files changed, 65 insertions(+), 18 deletions(-) diff --git a/tests/v2/test_jsonrpc_waku.nim b/tests/v2/test_jsonrpc_waku.nim index bc753eb4c..43e497595 100644 --- a/tests/v2/test_jsonrpc_waku.nim +++ b/tests/v2/test_jsonrpc_waku.nim @@ -1,3 +1,5 @@ +{.used.} + import std/[unittest, options, sets, tables, os, strutils, sequtils], stew/shims/net as stewNet, diff --git a/tests/v2/test_message_store.nim b/tests/v2/test_message_store.nim index 1f3a33278..ce51147ca 100644 --- a/tests/v2/test_message_store.nim +++ b/tests/v2/test_message_store.nim @@ -1,3 +1,4 @@ +{.used.} import std/[unittest, options, tables, sets], diff --git a/tests/v2/test_peer_manager.nim b/tests/v2/test_peer_manager.nim index 65961372f..fe9d49fb6 100644 --- a/tests/v2/test_peer_manager.nim +++ b/tests/v2/test_peer_manager.nim @@ -1,3 +1,5 @@ +{.used.} + import std/[unittest, options, sets, tables, sequtils], stew/shims/net as stewNet, diff --git a/tests/v2/test_waku_bridge.nim b/tests/v2/test_waku_bridge.nim index e79c616e2..0426e61d0 100644 --- a/tests/v2/test_waku_bridge.nim +++ b/tests/v2/test_waku_bridge.nim @@ -1,7 +1,7 @@ {.used.} import - std/unittest, + std/[unittest, strutils], chronicles, chronos, stew/shims/net as stewNet, stew/byteutils, libp2p/crypto/crypto, libp2p/crypto/secp, @@ -14,10 +14,10 @@ import eth/keys, ../../waku/common/wakubridge, ../../waku/v1/protocol/waku_protocol, - ../../waku/v2/protocol/[waku_message, message_notifier], + ../../waku/v2/protocol/waku_message, ../../waku/v2/protocol/waku_store/waku_store, ../../waku/v2/protocol/waku_filter/waku_filter, - ../../waku/v2/node/wakunode2, + ../../waku/v2/node/[wakunode2, waku_payload], ../test_helpers procSuite "WakuBridge": @@ -61,6 +61,10 @@ procSuite "WakuBridge": proc relayHandler(topic: string, data: seq[byte]) {.async, gcsafe.} = let msg = WakuMessage.init(data) if msg.isOk() and msg.value().version == 1: + check: + # Message fields are as expected + msg.value().contentTopic == contentTopic # Topic translation worked + string.fromBytes(msg.value().payload).contains("from V1") completionFut.complete(true) v2Node.subscribe(defaultBridgeTopic, relayHandler) @@ -75,6 +79,13 @@ procSuite "WakuBridge": check: # v1Node received message published by v2Node v1Node.protocolState(Waku).queue.items.len == 1 + + let msg = v1Node.protocolState(Waku).queue.items[0] + + check: + # Message fields are as expected + msg.env.topic == topic # Topic translation worked + string.fromBytes(msg.env.data).contains("from V2") # Test bridging from V1 to V2 check: diff --git a/tests/v2/test_waku_rln_relay.nim b/tests/v2/test_waku_rln_relay.nim index 6dfadb43f..fe44a4ee9 100644 --- a/tests/v2/test_waku_rln_relay.nim +++ b/tests/v2/test_waku_rln_relay.nim @@ -1,3 +1,5 @@ +{.used.} + import chronos, chronicles, options, stint, unittest, web3, diff --git a/tests/v2/test_waku_swap.nim b/tests/v2/test_waku_swap.nim index 56d765466..427f380c2 100644 --- a/tests/v2/test_waku_swap.nim +++ b/tests/v2/test_waku_swap.nim @@ -1,3 +1,5 @@ +{.used.} + import std/[unittest, options, tables, sets], chronos, chronicles, stew/shims/net as stewNet, stew/byteutils, diff --git a/tests/v2/test_web3.nim b/tests/v2/test_web3.nim index 3f59af95d..420cc7054 100644 --- a/tests/v2/test_web3.nim +++ b/tests/v2/test_web3.nim @@ -1,3 +1,5 @@ +{.used.} + import web3 proc web3Test() = var web3: Web3 # an identifier from web3 package diff --git a/waku/common/wakubridge.nim b/waku/common/wakubridge.nim index 1807ef70d..8ecbf1d8a 100644 --- a/waku/common/wakubridge.nim +++ b/waku/common/wakubridge.nim @@ -1,28 +1,21 @@ import - std/[strutils, tables], + std/tables, chronos, confutils, chronicles, chronicles/topics_registry, metrics, stew/endians2, stew/shims/net as stewNet, json_rpc/rpcserver, # Waku v1 imports eth/[keys, p2p], eth/common/utils, - eth/p2p/[enode, whispernodes], + eth/p2p/enode, ../v1/protocol/waku_protocol, - ./utils/nat, - ../v1/node/rpc/wakusim, - ../v1/node/rpc/waku, - ../v1/node/rpc/key_storage, - ../v1/node/waku_helpers, # Waku v2 imports libp2p/crypto/crypto, ../v2/protocol/waku_filter/waku_filter_types, ../v2/node/wakunode2, - ../v2/node/jsonrpc/[debug_api, - filter_api, - relay_api, - store_api], # Common cli config ./config_bridge +declarePublicCounter waku_bridge_transfers, "Number of messages transferred between Waku v1 and v2 networks", ["type"] + logScope: topics = "wakubridge" @@ -51,15 +44,17 @@ type func toWakuMessage(env: Envelope): WakuMessage = # Translate a Waku v1 envelope to a Waku v2 message WakuMessage(payload: env.data, - contentTopic: ContentTopic(uint32.fromBytes(env.topic)), + contentTopic: ContentTopic(uint32.fromBytes(env.topic, Endianness.bigEndian)), version: 1) proc toWakuV2(bridge: WakuBridge, env: Envelope) {.async.} = + waku_bridge_transfers.inc(labelValues = ["v1_to_v2"]) await bridge.nodev2.publish(defaultBridgeTopic, env.toWakuMessage()) proc toWakuV1(bridge: WakuBridge, msg: WakuMessage) {.gcsafe.} = + waku_bridge_transfers.inc(labelValues = ["v2_to_v1"]) discard bridge.nodev1.postMessage(ttl = defaultTTL, - topic = msg.contentTopic.toBytes(), + topic = msg.contentTopic.toBytes(Endianness.bigEndian), payload = msg.payload) ############## @@ -144,6 +139,18 @@ proc stop*(bridge: WakuBridge) {.async.} = await bridge.nodev2.stop() when isMainModule: + import + eth/p2p/whispernodes, + ./utils/nat, + ../v1/node/rpc/wakusim, + ../v1/node/rpc/waku, + ../v1/node/rpc/key_storage, + ../v1/node/waku_helpers, + ../v2/node/jsonrpc/[debug_api, + filter_api, + relay_api, + store_api] + proc startV2Rpc(node: WakuNode, rpcServer: RpcHttpServer, conf: WakuNodeConf) = installDebugApiHandlers(node, rpcServer) diff --git a/waku/v2/node/jsonrpc/admin_api.nim b/waku/v2/node/jsonrpc/admin_api.nim index d228592e2..00ace480a 100644 --- a/waku/v2/node/jsonrpc/admin_api.nim +++ b/waku/v2/node/jsonrpc/admin_api.nim @@ -14,6 +14,9 @@ import export jsonrpc_types +logScope: + topics = "admin api" + const futTimeout* = 30.seconds # Max time to wait for futures proc constructMultiaddrStr*(wireaddr: MultiAddress, peerId: PeerId): string = diff --git a/waku/v2/node/jsonrpc/debug_api.nim b/waku/v2/node/jsonrpc/debug_api.nim index fcb8cd7f1..3eeb9d215 100644 --- a/waku/v2/node/jsonrpc/debug_api.nim +++ b/waku/v2/node/jsonrpc/debug_api.nim @@ -2,6 +2,9 @@ import json_rpc/rpcserver, ../wakunode2 +logScope: + topics = "debug api" + proc installDebugApiHandlers*(node: WakuNode, rpcsrv: RpcServer) = ## Debug API version 1 definitions diff --git a/waku/v2/node/jsonrpc/filter_api.nim b/waku/v2/node/jsonrpc/filter_api.nim index d2477ab26..666242c5f 100644 --- a/waku/v2/node/jsonrpc/filter_api.nim +++ b/waku/v2/node/jsonrpc/filter_api.nim @@ -10,6 +10,9 @@ import export jsonrpc_types +logScope: + topics = "filter api" + const futTimeout* = 5.seconds # Max time to wait for futures const maxCache* = 100 # Max number of messages cached per topic @TODO make this configurable diff --git a/waku/v2/node/jsonrpc/jsonrpc_utils.nim b/waku/v2/node/jsonrpc/jsonrpc_utils.nim index 735da3c53..92347feec 100644 --- a/waku/v2/node/jsonrpc/jsonrpc_utils.nim +++ b/waku/v2/node/jsonrpc/jsonrpc_utils.nim @@ -1,5 +1,5 @@ import - std/[options, json, sequtils], + std/[options, json], eth/keys, ../../../v1/node/rpc/hexstrings, ../../protocol/waku_store/waku_store_types, diff --git a/waku/v2/node/jsonrpc/private_api.nim b/waku/v2/node/jsonrpc/private_api.nim index dc76742e0..7a4e15e7a 100644 --- a/waku/v2/node/jsonrpc/private_api.nim +++ b/waku/v2/node/jsonrpc/private_api.nim @@ -10,6 +10,9 @@ import export waku_payload, jsonrpc_types +logScope: + topics = "private api" + const futTimeout* = 5.seconds # Max time to wait for futures proc installPrivateApiHandlers*(node: WakuNode, rpcsrv: RpcServer, rng: ref BrHmacDrbgContext, topicCache: TopicCache) = diff --git a/waku/v2/node/jsonrpc/relay_api.nim b/waku/v2/node/jsonrpc/relay_api.nim index aaa0cf80b..3cc4f1c33 100644 --- a/waku/v2/node/jsonrpc/relay_api.nim +++ b/waku/v2/node/jsonrpc/relay_api.nim @@ -11,6 +11,9 @@ import export jsonrpc_types +logScope: + topics = "relay api" + const futTimeout* = 5.seconds # Max time to wait for futures const maxCache* = 100 # Max number of messages cached per topic @TODO make this configurable diff --git a/waku/v2/node/jsonrpc/store_api.nim b/waku/v2/node/jsonrpc/store_api.nim index 39928818b..05806ff1e 100644 --- a/waku/v2/node/jsonrpc/store_api.nim +++ b/waku/v2/node/jsonrpc/store_api.nim @@ -9,6 +9,9 @@ import export jsonrpc_types +logScope: + topics = "store api" + proc installStoreApiHandlers*(node: WakuNode, rpcsrv: RpcServer) = const futTimeout = 5.seconds diff --git a/waku/v2/protocol/waku_rln_relay/waku_rln_relay_utils.nim b/waku/v2/protocol/waku_rln_relay/waku_rln_relay_utils.nim index 98e9c673a..25fd270a7 100644 --- a/waku/v2/protocol/waku_rln_relay/waku_rln_relay_utils.nim +++ b/waku/v2/protocol/waku_rln_relay/waku_rln_relay_utils.nim @@ -1,5 +1,5 @@ import - chronicles, options, chronos, stint, sequtils, + chronicles, options, chronos, stint, web3, stew/byteutils, eth/keys,