diff --git a/.appveyor.yml b/.appveyor.yml index bf0cbff08..7847e8174 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -45,7 +45,6 @@ test_script: # Wrapper builds could then also get activated instead but need some rework for Windows. # - mingw32-make -j2 ARCH_OVERRIDE=%PLATFORM% libnimbus.so # - mingw32-make -j2 ARCH_OVERRIDE=%PLATFORM% libnimbus.a - - mingw32-make -j2 ARCH_OVERRIDE=%PLATFORM% wakusim deploy: off diff --git a/.travis.yml b/.travis.yml index c1b40384a..426c8fb88 100644 --- a/.travis.yml +++ b/.travis.yml @@ -54,7 +54,7 @@ script: - build/nimbus --help # "-static" option will not work for osx unless static system libraries are provided - if [ "$TRAVIS_OS_NAME" = "osx" ]; then - make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC}" test test-reproducibility wrappers wakusim; + make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC}" test test-reproducibility wrappers; else - make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC}" test test-reproducibility wrappers wrappers-static wakusim; + make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC}" test test-reproducibility wrappers wrappers-static; fi diff --git a/Makefile b/Makefile index bf4e79c0d..b69dffa72 100644 --- a/Makefile +++ b/Makefile @@ -25,8 +25,7 @@ TOOLS := \ persistBlockTestGen TOOLS_DIRS := \ premix \ - tests \ - waku + tests # comma-separated values for the "clean" target TOOLS_CSV := $(subst $(SPACE),$(COMMA),$(TOOLS)) @@ -165,13 +164,5 @@ wrappers-static: | build deps libnimbus.a echo -e $(BUILD_MSG) "build/go_wrapper_whisper_example_static" && \ go build -ldflags "-linkmode external -extldflags '-static $(EXTRA_LIBS_STATIC)'" -o build/go_wrapper_whisper_example_static wrappers/wrapper_whisper_example.go wrappers/cfuncs.go -wakunode: | build deps - echo -e $(BUILD_MSG) "build/$@" && \ - $(ENV_SCRIPT) nim wakunode $(NIM_PARAMS) nimbus.nims - -wakusim: | build deps wakunode - echo -e $(BUILD_MSG) "build/$@" && \ - $(ENV_SCRIPT) nim wakusim $(NIM_PARAMS) nimbus.nims - endif # "variables.mk" was not included diff --git a/azure-pipelines.yml b/azure-pipelines.yml index a55e5b0dd..a4d545e0f 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -62,7 +62,7 @@ jobs: build/nimbus.exe --help # fail fast export NIMTEST_ABORT_ON_ERROR=1 - mingw32-make -j2 ARCH_OVERRIDE=${PLATFORM} test test-reproducibility wakusim + mingw32-make -j2 ARCH_OVERRIDE=${PLATFORM} test test-reproducibility # Disable libnimbus builds until https://github.com/nim-lang/Nim/issues/12759 is fixed. # Wrapper builds could then also get activated instead but need some rework for Windows. # mingw32-make -j2 ARCH_OVERRIDE=${PLATFORM} libnimbus.so diff --git a/nimbus.nimble b/nimbus.nimble index 5c13753a4..49ac9a37a 100644 --- a/nimbus.nimble +++ b/nimbus.nimble @@ -40,11 +40,3 @@ task test, "Run tests": task nimbus, "Build Nimbus": buildBinary "nimbus", "nimbus/", "-d:chronicles_log_level=TRACE" - -task wakunode, "Build Waku node": - buildBinary "wakunode", "waku/", "-d:chronicles_log_level=TRACE" - -task wakusim, "Build Waku simulation tools": - buildBinary "quicksim", "waku/", "-d:chronicles_log_level=INFO" - buildBinary "start_network", "waku/", "-d:chronicles_log_level=DEBUG" - diff --git a/nimbus/rpc/waku.nim b/nimbus/rpc/waku.nim deleted file mode 100644 index d20981010..000000000 --- a/nimbus/rpc/waku.nim +++ /dev/null @@ -1,363 +0,0 @@ -import - json_rpc/rpcserver, tables, options, sequtils, - eth/[common, rlp, keys, p2p], eth/p2p/rlpx_protocols/waku_protocol, - nimcrypto/[sysrand, hmac, sha2, pbkdf2], - rpc_types, hexstrings, key_storage - -from stew/byteutils import hexToSeqByte, hexToByteArray - -# Blatant copy of Whisper RPC but for the Waku protocol - -proc setupWakuRPC*(node: EthereumNode, keys: KeyStorage, rpcsrv: RpcServer) = - - rpcsrv.rpc("waku_version") do() -> string: - ## Returns string of the current whisper protocol version. - result = wakuVersionStr - - rpcsrv.rpc("waku_info") do() -> WhisperInfo: - ## Returns diagnostic information about the whisper node. - let config = node.protocolState(Waku).config - result = WhisperInfo(minPow: config.powRequirement, - maxMessageSize: config.maxMsgSize, - memory: 0, - messages: 0) - - # TODO: uint32 instead of uint64 is OK here, but needs to be added in json_rpc - rpcsrv.rpc("waku_setMaxMessageSize") do(size: uint64) -> bool: - ## Sets the maximal message size allowed by this node. - ## Incoming and outgoing messages with a larger size will be rejected. - ## Whisper message size can never exceed the limit imposed by the underlying - ## P2P protocol (10 Mb). - ## - ## size: Message size in bytes. - ## - ## Returns true on success and an error on failure. - result = node.setMaxMessageSize(size.uint32) - if not result: - raise newException(ValueError, "Invalid size") - - rpcsrv.rpc("waku_setMinPoW") do(pow: float) -> bool: - ## Sets the minimal PoW required by this node. - ## - ## pow: The new PoW requirement. - ## - ## Returns true on success and an error on failure. - # Note: `setPowRequirement` does not raise on failures of sending the update - # to the peers. Hence in theory this should not causes errors. - await node.setPowRequirement(pow) - result = true - - # TODO: change string in to ENodeStr with extra checks - rpcsrv.rpc("waku_markTrustedPeer") do(enode: string) -> bool: - ## Marks specific peer trusted, which will allow it to send historic - ## (expired) messages. - ## Note: This function is not adding new nodes, the node needs to exists as - ## a peer. - ## - ## enode: Enode of the trusted peer. - ## - ## Returns true on success and an error on failure. - # TODO: It will now require an enode://pubkey@ip:port uri - # could also accept only the pubkey (like geth)? - let peerNode = newNode(enode) - result = node.setPeerTrusted(peerNode.id) - if not result: - raise newException(ValueError, "Not a peer") - - rpcsrv.rpc("waku_newKeyPair") do() -> Identifier: - ## Generates a new public and private key pair for message decryption and - ## encryption. - ## - ## Returns key identifier on success and an error on failure. - result = generateRandomID().Identifier - keys.asymKeys.add(result.string, KeyPair.random().tryGet()) - - rpcsrv.rpc("waku_addPrivateKey") do(key: PrivateKey) -> Identifier: - ## Stores the key pair, and returns its ID. - ## - ## key: Private key as hex bytes. - ## - ## Returns key identifier on success and an error on failure. - result = generateRandomID().Identifier - - keys.asymKeys.add(result.string, key.toKeyPair().tryGet()) - - rpcsrv.rpc("waku_deleteKeyPair") do(id: Identifier) -> bool: - ## Deletes the specifies key if it exists. - ## - ## id: Identifier of key pair - ## - ## Returns true on success and an error on failure. - var unneeded: KeyPair - result = keys.asymKeys.take(id.string, unneeded) - if not result: - raise newException(ValueError, "Invalid key id") - - rpcsrv.rpc("waku_hasKeyPair") do(id: Identifier) -> bool: - ## Checks if the whisper node has a private key of a key pair matching the - ## given ID. - ## - ## id: Identifier of key pair - ## - ## Returns (true or false) on success and an error on failure. - result = keys.asymkeys.hasKey(id.string) - - rpcsrv.rpc("waku_getPublicKey") do(id: Identifier) -> PublicKey: - ## Returns the public key for identity ID. - ## - ## id: Identifier of key pair - ## - ## Returns public key on success and an error on failure. - # Note: key not found exception as error in case not existing - result = keys.asymkeys[id.string].pubkey - - rpcsrv.rpc("waku_getPrivateKey") do(id: Identifier) -> PrivateKey: - ## Returns the private key for identity ID. - ## - ## id: Identifier of key pair - ## - ## Returns private key on success and an error on failure. - # Note: key not found exception as error in case not existing - result = keys.asymkeys[id.string].seckey - - rpcsrv.rpc("waku_newSymKey") do() -> Identifier: - ## Generates a random symmetric key and stores it under an ID, which is then - ## returned. Can be used encrypting and decrypting messages where the key is - ## known to both parties. - ## - ## Returns key identifier on success and an error on failure. - result = generateRandomID().Identifier - var key: SymKey - if randomBytes(key) != key.len: - raise newException(KeyGenerationError, "Failed generating key") - - keys.symKeys.add(result.string, key) - - - rpcsrv.rpc("waku_addSymKey") do(key: SymKey) -> Identifier: - ## Stores the key, and returns its ID. - ## - ## key: The raw key for symmetric encryption as hex bytes. - ## - ## Returns key identifier on success and an error on failure. - result = generateRandomID().Identifier - - keys.symKeys.add(result.string, key) - - rpcsrv.rpc("waku_generateSymKeyFromPassword") do(password: string) -> Identifier: - ## Generates the key from password, stores it, and returns its ID. - ## - ## password: Password. - ## - ## Returns key identifier on success and an error on failure. - ## Warning: an empty string is used as salt because the shh RPC API does not - ## allow for passing a salt. A very good password is necessary (calculate - ## yourself what that means :)) - var ctx: HMAC[sha256] - var symKey: SymKey - if pbkdf2(ctx, password, "", 65356, symKey) != sizeof(SymKey): - raise newException(KeyGenerationError, "Failed generating key") - - result = generateRandomID().Identifier - keys.symKeys.add(result.string, symKey) - - rpcsrv.rpc("waku_hasSymKey") do(id: Identifier) -> bool: - ## Returns true if there is a key associated with the name string. - ## Otherwise, returns false. - ## - ## id: Identifier of key. - ## - ## Returns (true or false) on success and an error on failure. - result = keys.symkeys.hasKey(id.string) - - rpcsrv.rpc("waku_getSymKey") do(id: Identifier) -> SymKey: - ## Returns the symmetric key associated with the given ID. - ## - ## id: Identifier of key. - ## - ## Returns Raw key on success and an error on failure. - # Note: key not found exception as error in case not existing - result = keys.symkeys[id.string] - - rpcsrv.rpc("waku_deleteSymKey") do(id: Identifier) -> bool: - ## Deletes the key associated with the name string if it exists. - ## - ## id: Identifier of key. - ## - ## Returns (true or false) on success and an error on failure. - var unneeded: SymKey - result = keys.symKeys.take(id.string, unneeded) - if not result: - raise newException(ValueError, "Invalid key id") - - rpcsrv.rpc("waku_subscribe") do(id: string, - options: WhisperFilterOptions) -> Identifier: - ## Creates and registers a new subscription to receive notifications for - ## inbound whisper messages. Returns the ID of the newly created - ## subscription. - ## - ## id: identifier of function call. In case of Whisper must contain the - ## value "messages". - ## options: WhisperFilterOptions - ## - ## Returns the subscription ID on success, the error on failure. - - # TODO: implement subscriptions, only for WS & IPC? - discard - - rpcsrv.rpc("waku_unsubscribe") do(id: Identifier) -> bool: - ## Cancels and removes an existing subscription. - ## - ## id: Subscription identifier - ## - ## Returns true on success, the error on failure - result = node.unsubscribeFilter(id.string) - if not result: - raise newException(ValueError, "Invalid filter id") - - proc validateOptions[T,U,V](asym: Option[T], sym: Option[U], topic: Option[V]) = - if (asym.isSome() and sym.isSome()) or (asym.isNone() and sym.isNone()): - raise newException(ValueError, - "Either privateKeyID/pubKey or symKeyID must be present") - if asym.isNone() and topic.isNone(): - raise newException(ValueError, "Topic mandatory with symmetric key") - - rpcsrv.rpc("waku_newMessageFilter") do(options: WhisperFilterOptions) -> Identifier: - ## Create a new filter within the node. This filter can be used to poll for - ## new messages that match the set of criteria. - ## - ## options: WhisperFilterOptions - ## - ## Returns filter identifier on success, error on failure - - # Check if either symKeyID or privateKeyID is present, and not both - # Check if there are Topics when symmetric key is used - validateOptions(options.privateKeyID, options.symKeyID, options.topics) - - var - src: Option[PublicKey] - privateKey: Option[PrivateKey] - symKey: Option[SymKey] - topics: seq[waku_protocol.Topic] - powReq: float64 - allowP2P: bool - - src = options.sig - - if options.privateKeyID.isSome(): - privateKey = some(keys.asymKeys[options.privateKeyID.get().string].seckey) - - if options.symKeyID.isSome(): - symKey= some(keys.symKeys[options.symKeyID.get().string]) - - if options.minPow.isSome(): - powReq = options.minPow.get() - - if options.topics.isSome(): - topics = options.topics.get() - - if options.allowP2P.isSome(): - allowP2P = options.allowP2P.get() - - let filter = initFilter(src, privateKey, symKey, topics, powReq, allowP2P) - result = node.subscribeFilter(filter).Identifier - - # TODO: Should we do this here "automatically" or separate it in another - # RPC call? Is there a use case for that? - # Same could be said about bloomfilter, except that there is a use case - # there to have a full node no matter what message filters. - # Could also be moved to waku_protocol.nim - let config = node.protocolState(Waku).config - if config.topics.isSome(): - try: - # TODO: an addTopics call would probably be more useful - let result = await node.setTopicInterest(config.topics.get().concat(filter.topics)) - if not result: - raise newException(ValueError, "Too many topics") - except CatchableError: - trace "setTopics error occured" - elif config.isLightNode: - try: - await node.setBloomFilter(node.filtersToBloom()) - except CatchableError: - trace "setBloomFilter error occured" - - rpcsrv.rpc("waku_deleteMessageFilter") do(id: Identifier) -> bool: - ## Uninstall a message filter in the node. - ## - ## id: Filter identifier as returned when the filter was created. - ## - ## Returns true on success, error on failure. - result = node.unsubscribeFilter(id.string) - if not result: - raise newException(ValueError, "Invalid filter id") - - rpcsrv.rpc("waku_getFilterMessages") do(id: Identifier) -> seq[WhisperFilterMessage]: - ## Retrieve messages that match the filter criteria and are received between - ## the last time this function was called and now. - ## - ## id: ID of filter that was created with `waku_newMessageFilter`. - ## - ## Returns array of messages on success and an error on failure. - let messages = node.getFilterMessages(id.string) - for msg in messages: - result.add WhisperFilterMessage( - sig: msg.decoded.src, - recipientPublicKey: msg.dst, - ttl: msg.ttl, - topic: msg.topic, - timestamp: msg.timestamp, - payload: msg.decoded.payload, - # Note: whisper_protocol padding is an Option as there is the - # possibility of 0 padding in case of custom padding. - padding: msg.decoded.padding.get(@[]), - pow: msg.pow, - hash: msg.hash) - - rpcsrv.rpc("waku_post") do(message: WhisperPostMessage) -> bool: - ## Creates a whisper message and injects it into the network for - ## distribution. - ## - ## message: Whisper message to post. - ## - ## Returns true on success and an error on failure. - - # Check if either symKeyID or pubKey is present, and not both - # Check if there is a Topic when symmetric key is used - validateOptions(message.pubKey, message.symKeyID, message.topic) - - var - sigPrivKey: Option[PrivateKey] - symKey: Option[SymKey] - topic: waku_protocol.Topic - padding: Option[seq[byte]] - targetPeer: Option[NodeId] - - if message.sig.isSome(): - sigPrivKey = some(keys.asymKeys[message.sig.get().string].seckey) - - if message.symKeyID.isSome(): - symKey = some(keys.symKeys[message.symKeyID.get().string]) - - # Note: If no topic it will be defaulted to 0x00000000 - if message.topic.isSome(): - topic = message.topic.get() - - if message.padding.isSome(): - padding = some(hexToSeqByte(message.padding.get().string)) - - if message.targetPeer.isSome(): - targetPeer = some(newNode(message.targetPeer.get()).id) - - result = node.postMessage(message.pubKey, - symKey, - sigPrivKey, - ttl = message.ttl.uint32, - topic = topic, - payload = hexToSeqByte(message.payload.string), - padding = padding, - powTime = message.powTime, - powTarget = message.powTarget, - targetPeer = targetPeer) - if not result: - raise newException(ValueError, "Message could not be posted") diff --git a/nimbus/rpc/wakusim.nim b/nimbus/rpc/wakusim.nim deleted file mode 100644 index 07ee950b9..000000000 --- a/nimbus/rpc/wakusim.nim +++ /dev/null @@ -1,31 +0,0 @@ -import - json_rpc/rpcserver, stew/endians2, nimcrypto/sysrand, - eth/[p2p, async_utils], eth/p2p/rlpx_protocols/waku_protocol - -proc generateTraffic(node: EthereumNode, amount = 100) {.async.} = - var topicNumber = 0'u32 - let payload = @[byte 0] - for i in 0.. bool: - traceAsyncErrors node.generateTraffic(amount) - return true - - rpcsrv.rpc("wakusim_generateRandomTraffic") do(amount: int) -> bool: - traceAsyncErrors node.generateRandomTraffic(amount) - return true diff --git a/waku/README.md b/waku/README.md deleted file mode 100644 index c73e1b66f..000000000 --- a/waku/README.md +++ /dev/null @@ -1,108 +0,0 @@ -# Introduction -`wakunode` is a cli application that allows you to run a -[Waku](https://specs.vac.dev/waku/waku.html) enabled node. - -The Waku specification is still in draft and thus this implementation will -change accordingly. - -Additionally the original Whisper (EIP-627) protocol can also be enabled as can -an experimental Whisper - Waku bridging option. - -# How to Build & Run - -## Prerequisites - -* GNU Make, Bash and the usual POSIX utilities. Git 2.9.4 or newer. -* PCRE - -More information on the installation of these can be found [here](https://github.com/status-im/nimbus#prerequisites). - -## Build & Run - -```bash -# The first `make` invocation will update all Git submodules. -# You'll run `make update` after each `git pull`, in the future, to keep those submodules up to date. -make wakunode - -# See available command line options -./build/wakunode --help - -# Connect the client directly with the Status test fleet -./build/wakunode --log-level:debug --discovery:off --fleet:test --log-metrics -``` - -# Using Metrics - -Metrics are available for valid envelopes and dropped envelopes. - -To compile in an HTTP endpoint for accessing the metrics we need to provide the -`insecure` flag: -```bash -make NIMFLAGS="-d:insecure" wakunode -./build/wakunode --metrics-server -``` - -Ensure your Prometheus config `prometheus.yml` contains the targets you care about, e.g.: - -``` -scrape_configs: - - job_name: "waku" - static_configs: - - targets: ['localhost:8008', 'localhost:8009', 'localhost:8010'] -``` - -For visualisation, similar steps can be used as is written down for Nimbus -[here](https://github.com/status-im/nimbus#metric-visualisation). - -There is a similar example dashboard that includes visualisation of the -envelopes available at `waku/examples/waku-grafana-dashboard.json`. - -# Testing Waku Protocol -One can set up several nodes, get them connected and then instruct them via the -JSON-RPC interface. This can be done via e.g. web3.js, nim-web3 (needs to be -updated) or simply curl your way out. - -The JSON-RPC interface is currently the same as the one of Whisper. The only -difference is the addition of broadcasting the topics interest when a filter -with a certain set of topics is subcribed. - -Example of a quick simulation using this approach: -```bash -# Build wakunode + quicksim -make NIMFLAGS="-d:insecure" wakusim - -# Start the simulation nodes, this currently requires multitail to be installed -./build/start_network --topology:FullMesh --amount:6 --test-node-peers:2 -# In another shell run -./build/quicksim -``` - -The `start_network` tool will also provide a `prometheus.yml` with targets -set to all simulation nodes that are started. This way you can easily start -prometheus with this config, e.g.: - -```bash -cd waku/metrics/prometheus -prometheus -``` - -A Grafana dashboard containing the example dashboard for each simulation node -is also generated and can be imported in case you have Grafana running. -This dashboard can be found at `./waku/metrics/waku-sim-all-nodes-grafana-dashboard.json` - -# Spec support - -*This section last updated April 21, 2020* - -This client of Waku is spec compliant with [Waku spec v1.0](https://specs.vac.dev/waku/waku.html). - -It doesn't yet implement the following recommended features: -- No support for rate limiting -- No support for DNS discovery to find Waku nodes -- It doesn't disconnect a peer if it receives a message before a Status message -- No support for negotiation with peer supporting multiple versions via Devp2p capabilities in `Hello` packet - -Additionally it makes the following choices: -- It doesn't send message confirmations -- It has partial support for accounting: - - Accounting of total resource usage and total circulated envelopes is done through metrics But no accounting is done for individual peers. diff --git a/waku/config.nim b/waku/config.nim deleted file mode 100644 index 688c4ff31..000000000 --- a/waku/config.nim +++ /dev/null @@ -1,154 +0,0 @@ -import - confutils/defs, chronicles, chronos, - eth/keys, eth/p2p/rlpx_protocols/waku_protocol - -type - Fleet* = enum - none - prod - staging - test - - WakuNodeConf* = object - logLevel* {. - desc: "Sets the log level." - defaultValue: LogLevel.INFO - name: "log-level" }: LogLevel - - tcpPort* {. - desc: "TCP listening port." - defaultValue: 30303 - name: "tcp-port" }: uint16 - - udpPort* {. - desc: "UDP listening port." - defaultValue: 30303 - name: "udp-port" }: uint16 - - portsShift* {. - desc: "Add a shift to all port numbers." - defaultValue: 0 - name: "ports-shift" }: uint16 - - nat* {. - desc: "Specify method to use for determining public address. " & - "Must be one of: any, none, upnp, pmp, extip:." - defaultValue: "any" }: string - - discovery* {. - desc: "Enable/disable discovery v4." - defaultValue: true - name: "discovery" }: bool - - noListen* {. - desc: "Disable listening for incoming peers." - defaultValue: false - name: "no-listen" }: bool - - fleet* {. - desc: "Select the fleet to connect to." - defaultValue: Fleet.none - name: "fleet" }: Fleet - - bootnodes* {. - desc: "Enode URL to bootstrap P2P discovery with. Argument may be repeated." - name: "bootnode" }: seq[string] - - staticnodes* {. - desc: "Enode URL to directly connect with. Argument may be repeated." - name: "staticnode" }: seq[string] - - whisper* {. - desc: "Enable the Whisper protocol." - defaultValue: false - name: "whisper" }: bool - - whisperBridge* {. - desc: "Enable the Whisper protocol and bridge with Waku protocol." - defaultValue: false - name: "whisper-bridge" }: bool - - lightNode* {. - desc: "Run as light node (no message relay).", - defaultValue: false - name: "light-node" }: bool - - wakuTopicInterest* {. - desc: "Run as node with a topic-interest", - defaultValue: false - name: "waku-topic-interest" }: bool - - wakuPow* {. - desc: "PoW requirement of Waku node.", - defaultValue: 0.002 - name: "waku-pow" }: float64 - - nodekey* {. - desc: "P2P node private key as hex.", - defaultValue: KeyPair.random().tryGet() - name: "nodekey" }: KeyPair - # TODO: Add nodekey file option - - bootnodeOnly* {. - desc: "Run only as discovery bootnode." - defaultValue: false - name: "bootnode-only" }: bool - - rpc* {. - desc: "Enable Waku RPC server.", - defaultValue: false - name: "rpc" }: bool - - rpcAddress* {. - desc: "Listening address of the RPC server.", - defaultValue: parseIpAddress("127.0.0.1") - name: "rpc-address" }: IpAddress - - rpcPort* {. - desc: "Listening port of the RPC server.", - defaultValue: 8545 - name: "rpc-port" }: uint16 - - metricsServer* {. - desc: "Enable the metrics server." - defaultValue: false - name: "metrics-server" }: bool - - metricsServerAddress* {. - desc: "Listening address of the metrics server." - defaultValue: parseIpAddress("127.0.0.1") - name: "metrics-server-address" }: IpAddress - - metricsServerPort* {. - desc: "Listening HTTP port of the metrics server." - defaultValue: 8008 - name: "metrics-server-port" }: uint16 - - logMetrics* {. - desc: "Enable metrics logging." - defaultValue: false - name: "log-metrics" }: bool - - # TODO: - # - discv5 + topic register - # - mailserver functionality - -proc parseCmdArg*(T: type KeyPair, p: TaintedString): T = - try: - # TODO: add isValidPrivateKey check from Nimbus? - result.seckey = PrivateKey.fromHex(string(p)).tryGet() - result.pubkey = result.seckey.toPublicKey()[] - except CatchableError as e: - raise newException(ConfigurationError, "Invalid private key") - -proc completeCmdArg*(T: type KeyPair, val: TaintedString): seq[string] = - return @[] - -proc parseCmdArg*(T: type IpAddress, p: TaintedString): T = - try: - result = parseIpAddress(p) - except CatchableError as e: - raise newException(ConfigurationError, "Invalid IP address") - -proc completeCmdArg*(T: type IpAddress, val: TaintedString): seq[string] = - return @[] diff --git a/waku/docker/Dockerfile b/waku/docker/Dockerfile deleted file mode 100644 index b4f1589db..000000000 --- a/waku/docker/Dockerfile +++ /dev/null @@ -1,36 +0,0 @@ -FROM debian:bullseye-slim AS build - -SHELL ["/bin/bash", "-c"] - -RUN apt-get -qq update \ - && apt-get -qq -y install build-essential make wget libpcre3-dev git curl &>/dev/null \ - && apt-get -qq clean - -ARG GIT_REVISION - -RUN cd /root \ - && git clone https://github.com/status-im/nimbus.git \ - && cd nimbus \ - && git reset --hard ${GIT_REVISION} \ - && { make &>/dev/null || true; } \ - && make -j$(nproc) update \ - && make NIMFLAGS="-d:debug -d:insecure" wakunode - -# --------------------------------- # -# Starting new image to reduce size # -# --------------------------------- # -FROM debian:bullseye-slim - -SHELL ["/bin/bash", "-c"] - -RUN apt-get -qq update \ - && apt-get -qq -y install libpcre3 &>/dev/null \ - && apt-get -qq clean \ - && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* - -COPY --from=build /root/nimbus/build/wakunode /usr/bin/wakunode - -MAINTAINER Kim De Mey -LABEL description="Wakunode: Waku and Whisper client" - -ENTRYPOINT ["/usr/bin/wakunode"] diff --git a/waku/docker/Makefile b/waku/docker/Makefile deleted file mode 100644 index 7fc69ab09..000000000 --- a/waku/docker/Makefile +++ /dev/null @@ -1,16 +0,0 @@ -SHELL := bash # the shell used internally by "make" - -# These default settings can be overriden by exporting env variables - -GIT_REVISION ?= $(shell git rev-parse HEAD) - -IMAGE_TAG ?= latest -IMAGE_NAME ?= statusteam/nimbus_wakunode:$(IMAGE_TAG) - -build: - docker build \ - --build-arg="GIT_REVISION=$(GIT_REVISION)" \ - -t $(IMAGE_NAME) . - -push: build - docker push $(IMAGE_NAME) diff --git a/waku/examples/waku-grafana-dashboard.json b/waku/examples/waku-grafana-dashboard.json deleted file mode 100644 index e7df1dff4..000000000 --- a/waku/examples/waku-grafana-dashboard.json +++ /dev/null @@ -1,812 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "id": 8, - "links": [], - "panels": [ - { - "datasource": null, - "gridPos": { - "h": 4, - "w": 6, - "x": 0, - "y": 0 - }, - "id": 16, - "options": { - "fieldOptions": { - "calcs": [ - "last" - ], - "defaults": { - "mappings": [], - "max": 100, - "min": 0, - "thresholds": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "override": {}, - "values": false - }, - "orientation": "auto", - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "6.4.5", - "targets": [ - { - "expr": "connected_peers{node=\"0\"}", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Connected Peers #0", - "type": "gauge" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": null, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 6, - "y": 0 - }, - "id": 22, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "options": {}, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false, - "ymax": null, - "ymin": null - }, - "tableColumn": "", - "targets": [ - { - "expr": "valid_envelopes_total{node=\"0\"}", - "refId": "A" - } - ], - "thresholds": "", - "timeFrom": null, - "timeShift": null, - "title": "Valid Envelopes #0", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": null, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 10, - "y": 0 - }, - "id": 20, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "options": {}, - "pluginVersion": "6.4.5", - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false, - "ymax": null, - "ymin": null - }, - "tableColumn": "", - "targets": [ - { - "expr": "dropped_expired_envelopes_total{node=\"0\"} + dropped_from_future_envelopes_total{node=\"0\"} + dropped_low_pow_envelopes_total{node=\"0\"} + dropped_too_large_envelopes_total{node=\"0\"} + dropped_bloom_filter_mismatch_envelopes_total{node=\"0\"} + dropped_topic_mismatch_envelopes_total{node=\"0\"} +dropped_benign_duplicate_envelopes_total{node=\"0\"} + dropped_duplicate_envelopes_total{node=\"0\"}", - "legendFormat": "Invalid envelopes", - "refId": "A" - } - ], - "thresholds": "", - "timeFrom": null, - "timeShift": null, - "title": "Invalid Envelopes #0", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "datasource": null, - "gridPos": { - "h": 4, - "w": 5, - "x": 14, - "y": 0 - }, - "id": 14, - "options": { - "fieldOptions": { - "calcs": [ - "lastNotNull" - ], - "defaults": { - "mappings": [], - "max": 200, - "min": 0, - "thresholds": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 200 - } - ], - "unit": "percent" - }, - "override": {}, - "values": false - }, - "orientation": "auto", - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "6.4.5", - "targets": [ - { - "expr": "rate(process_cpu_seconds_total{node=\"0\"}[5s]) * 100", - "legendFormat": "CPU Usage", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "CPU Usage #0", - "type": "gauge" - }, - { - "datasource": null, - "gridPos": { - "h": 4, - "w": 5, - "x": 19, - "y": 0 - }, - "id": 18, - "options": { - "fieldOptions": { - "calcs": [ - "lastNotNull" - ], - "defaults": { - "mappings": [], - "max": 2147483648, - "min": 0, - "thresholds": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 2147483648 - } - ], - "unit": "bytes" - }, - "override": {}, - "values": false - }, - "orientation": "auto", - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "6.4.5", - "targets": [ - { - "expr": "process_resident_memory_bytes{node=\"0\"}", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "RSS Memory #0", - "type": "gauge" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 4 - }, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pluginVersion": "6.4.5", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "valid_envelopes_total{node=\"0\"}", - "hide": false, - "instant": false, - "legendFormat": "Valid", - "refId": "A" - }, - { - "expr": "dropped_benign_duplicate_envelopes_total{node=\"0\"}", - "hide": false, - "instant": false, - "legendFormat": "Benign duplicate", - "refId": "B" - }, - { - "expr": "dropped_duplicate_envelopes_total{node=\"0\"}", - "hide": false, - "legendFormat": "Duplicate", - "refId": "C" - }, - { - "expr": "dropped_expired_envelopes_total{node=\"0\"}", - "hide": false, - "legendFormat": "Expired", - "refId": "D" - }, - { - "expr": "dropped_from_future_envelopes_total{node=\"0\"}", - "hide": false, - "legendFormat": "Future timestamped", - "refId": "E" - }, - { - "expr": "dropped_low_pow_envelopes_total{node=\"0\"}", - "hide": false, - "legendFormat": "Too low PoW", - "refId": "F" - }, - { - "expr": "dropped_bloom_filter_mismatch_envelopes_total{node=\"0\"}", - "hide": false, - "legendFormat": "Bloom filter mismatch", - "refId": "G" - }, - { - "expr": "dropped_topic_mismatch_envelopes_total{node=\"0\"}", - "hide": false, - "legendFormat": "Topic mismatch", - "refId": "H" - }, - { - "expr": "dropped_too_large_envelopes_total{node=\"0\"}", - "hide": false, - "legendFormat": "Too Large", - "refId": "I" - }, - { - "expr": "dropped_full_queue_new_envelopes_total{node=\"0\"}", - "hide": false, - "legendFormat": "Full queue new", - "refId": "J" - }, - { - "expr": "dropped_full_queue_old_envelopes_total{node=\"0\"}", - "hide": false, - "legendFormat": "Full queue old", - "refId": "K" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Waku Envelopes #0", - "tooltip": { - "shared": true, - "sort": 1, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 4 - }, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "RSS Memory", - "yaxis": 2 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "connected_peers{node=\"0\"}", - "intervalFactor": 1, - "legendFormat": "Connected Peers", - "refId": "A" - }, - { - "expr": "process_resident_memory_bytes{node=\"0\"}", - "interval": "", - "intervalFactor": 1, - "legendFormat": "RSS Memory", - "refId": "B" - }, - { - "expr": "rate(process_cpu_seconds_total{node=\"0\"}[15s]) * 100", - "legendFormat": "CPU usage %", - "refId": "C" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Waku Node #0", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 13 - }, - "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "process_max_fds{node=\"0\"}", - "legendFormat": "Maximum file descriptors", - "refId": "A" - }, - { - "expr": "process_open_fds{node=\"0\"}", - "legendFormat": "Open file descriptors", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "File Descriptors #0", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 13 - }, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "nim_gc_mem_bytes{node=\"0\"}", - "legendFormat": "Nim GC total memory", - "refId": "A" - }, - { - "expr": "nim_gc_mem_occupied_bytes{node=\"0\"}", - "legendFormat": "Nim GC used memory", - "refId": "B" - }, - { - "expr": "process_resident_memory_bytes{node=\"0\"}", - "legendFormat": "RSS memory", - "refId": "C" - }, - { - "expr": "process_virtual_memory_bytes{node=\"0\"}", - "legendFormat": "Virtual memory", - "refId": "D" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Memory Usage #0", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - ], - "refresh": "5s", - "schemaVersion": 20, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-30m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ] - }, - "timezone": "", - "title": "Waku Node", - "uid": "K7Z6IoBZk", - "version": 2 -} \ No newline at end of file diff --git a/waku/nim.cfg b/waku/nim.cfg deleted file mode 100644 index 3d4181563..000000000 --- a/waku/nim.cfg +++ /dev/null @@ -1,4 +0,0 @@ --d:chronicles_line_numbers --d:"chronicles_runtime_filtering=on" --d:nimDebugDlOpen - diff --git a/waku/quicksim.nim b/waku/quicksim.nim deleted file mode 100644 index 9b6e2a1ad..000000000 --- a/waku/quicksim.nim +++ /dev/null @@ -1,76 +0,0 @@ -import - os, strformat, chronicles, json_rpc/[rpcclient, rpcserver], nimcrypto/sysrand, - eth/common as eth_common, eth/keys, eth/p2p/rlpx_protocols/waku_protocol, - ../nimbus/rpc/[hexstrings, rpc_types, waku], - options as what # TODO: Huh? Redefinition? - -from os import DirSep -from strutils import rsplit -template sourceDir: string = currentSourcePath.rsplit(DirSep, 1)[0] - -const sigWakuPath = &"{sourceDir}{DirSep}rpc{DirSep}wakucallsigs.nim" -createRpcSigs(RpcHttpClient, sigWakuPath) - -const topicAmount = 100 - -let - trafficNode = newRpcHttpClient() - lightNode = newRpcHttpClient() - lightNode2 = newRpcHttpClient() - -waitFor lightNode.connect("localhost", Port(8545)) -waitFor lightNode2.connect("localhost", Port(8546)) -waitFor trafficNode.connect("localhost", Port(8548)) - -proc generateTopics(amount = topicAmount): seq[waku_protocol.Topic] = - var topic: waku_protocol.Topic - for i in 0.. 0: - for bootNode in bootNodes: - result.cmd &= "--bootnode:" & bootNode & " " - else: - result.cmd &= "--discovery:off" & " " - if staticNodes.len > 0: - for staticNode in staticNodes: - result.cmd &= "--staticnode:" & staticNode & " " - - result.master = master - result.enode = $enode - result.shift = shift - result.label = label - - debug "Node command created.", cmd=result.cmd - -proc starNetwork(amount: int): seq[NodeInfo] = - let masterNode = initNodeCmd(FullNode, portOffset, master = true, - label = "master node") - result.add(masterNode) - for i in 1.. 0: setBootNodes(config.bootnodes) - elif config.fleet == prod: setBootNodes(StatusBootNodes) - elif config.fleet == staging: setBootNodes(StatusBootNodesStaging) - elif config.fleet == test : setBootNodes(StatusBootNodesTest) - else: @[] - - traceAsyncErrors node.connectToNetwork(bootnodes, not config.noListen, - config.discovery) - - if not config.bootnodeOnly: - # Optionally direct connect with a set of nodes - if config.staticnodes.len > 0: connectToNodes(node, config.staticnodes) - elif config.fleet == prod: connectToNodes(node, WhisperNodes) - elif config.fleet == staging: connectToNodes(node, WhisperNodesStaging) - elif config.fleet == test: connectToNodes(node, WhisperNodesTest) - - if config.rpc: - let ta = initTAddress(config.rpcAddress, - Port(config.rpcPort + config.portsShift)) - var rpcServer = newRpcHttpServer([ta]) - let keys = newKeyStorage() - setupWakuRPC(node, keys, rpcServer) - setupWakuSimRPC(node, rpcServer) - rpcServer.start() - - when defined(insecure): - if config.metricsServer: - let - address = config.metricsServerAddress - port = config.metricsServerPort + config.portsShift - info "Starting metrics HTTP server", address, port - metrics.startHttpServer($address, Port(port)) - - if config.logMetrics: - proc logMetrics(udata: pointer) {.closure, gcsafe.} = - {.gcsafe.}: - let - connectedPeers = connected_peers.value - validEnvelopes = waku_protocol.valid_envelopes.value - invalidEnvelopes = waku_protocol.dropped_expired_envelopes.value + - waku_protocol.dropped_from_future_envelopes.value + - waku_protocol.dropped_low_pow_envelopes.value + - waku_protocol.dropped_too_large_envelopes.value + - waku_protocol.dropped_bloom_filter_mismatch_envelopes.value + - waku_protocol.dropped_topic_mismatch_envelopes.value + - waku_protocol.dropped_benign_duplicate_envelopes.value + - waku_protocol.dropped_duplicate_envelopes.value - - info "Node metrics", connectedPeers, validEnvelopes, invalidEnvelopes - addTimer(Moment.fromNow(2.seconds), logMetrics) - addTimer(Moment.fromNow(2.seconds), logMetrics) - - runForever() - -when isMainModule: - let conf = WakuNodeConf.load() - run(conf)