Move Nimbus Waku RPC code + add converted whisper rpc test

- Move Waku RPC code from Nimbus + necessary code around it
- Add Waku RPC test which is a copy of the Whisper RPC test
- Some renaming
- Remove nimbus submodule

.gitmodules

@@ -1,8 +1,3 @@
-[submodule "vendor/nimbus"]
-	path = vendor/nimbus
-	url = https://github.com/status-im/nimbus.git
-	ignore = dirty
-	branch = master
 [submodule "vendor/nim-eth"]
 	path = vendor/nim-eth
 	url = https://github.com/status-im/nim-eth.git

Makefile

@@ -91,7 +91,7 @@ test: | build deps
 
 # usual cleaning
 clean: | clean-common
-	rm -rf build/{wakunode,quicksim,start_network}
+	rm -rf build/{wakunode,quicksim,start_network,all_tests}
 ifneq ($(USE_LIBBACKTRACE), 0)
 	+ $(MAKE) -C vendor/nim-libbacktrace clean $(HANDLE_OUTPUT)
 endif

tests/all_tests.nim

@@ -0,0 +1,2 @@
+import
+  ./test_rpc_waku

tests/test_rpc_waku.nim

@@ -0,0 +1,238 @@
+import
+  unittest, strformat, options, stew/byteutils, json_rpc/[rpcserver, rpcclient],
+  eth/common as eth_common, eth/[rlp, keys, p2p],
+  eth/p2p/rlpx_protocols/waku_protocol,
+  ../waku/node/v0/rpc/[hexstrings, rpc_types, waku, key_storage]
+
+from os import DirSep, ParDir
+from strutils import rsplit
+template sourceDir: string = currentSourcePath.rsplit(DirSep, 1)[0]
+
+## Generate client convenience marshalling wrappers from forward declarations
+## For testing, ethcallsigs needs to be kept in sync with ../waku/node/v0/rpc/waku
+const sigPath = &"{sourceDir}{DirSep}{ParDir}{DirSep}waku{DirSep}node{DirSep}v0{DirSep}rpc{DirSep}wakucallsigs.nim"
+createRpcSigs(RpcSocketClient, sigPath)
+
+proc setupNode(capabilities: varargs[ProtocolInfo, `protocolInfo`]): EthereumNode =
+  let
+    keypair = KeyPair.random()[]
+    srvAddress = Address(ip: parseIpAddress("0.0.0.0"), tcpPort: Port(30303),
+      udpPort: Port(30303))
+
+  result = newEthereumNode(keypair, srvAddress, 1, nil, "waku test rpc",
+    addAllCapabilities = false)
+  for capability in capabilities:
+    result.addCapability capability
+
+proc doTests {.async.} =
+  var ethNode = setupNode(Waku)
+
+  # Create Ethereum RPCs
+  let rpcPort = 8545
+  var
+    rpcServer = newRpcSocketServer(["localhost:" & $rpcPort])
+    client = newRpcSocketClient()
+  let keys = newKeyStorage()
+  setupWakuRPC(ethNode, keys, rpcServer)
+
+  # Begin tests
+  rpcServer.start()
+  await client.connect("localhost", Port(rpcPort))
+
+  suite "Waku Remote Procedure Calls":
+    test "waku_version":
+      check await(client.waku_version()) == wakuVersionStr
+    test "waku_info":
+      let info = await client.waku_info()
+      check info.maxMessageSize == defaultMaxMsgSize
+    test "waku_setMaxMessageSize":
+      let testValue = 1024'u64
+      check await(client.waku_setMaxMessageSize(testValue)) == true
+      var info = await client.waku_info()
+      check info.maxMessageSize == testValue
+      expect ValueError:
+        discard await(client.waku_setMaxMessageSize(defaultMaxMsgSize + 1))
+      info = await client.waku_info()
+      check info.maxMessageSize == testValue
+    test "waku_setMinPoW":
+      let testValue = 0.0001
+      check await(client.waku_setMinPoW(testValue)) == true
+      let info = await client.waku_info()
+      check info.minPow == testValue
+    # test "waku_markTrustedPeer":
+      # TODO: need to connect a peer to test
+    test "waku asymKey tests":
+      let keyID = await client.waku_newKeyPair()
+      check:
+        await(client.waku_hasKeyPair(keyID)) == true
+        await(client.waku_deleteKeyPair(keyID)) == true
+        await(client.waku_hasKeyPair(keyID)) == false
+      expect ValueError:
+        discard await(client.waku_deleteKeyPair(keyID))
+
+      let privkey = "0x5dc5381cae54ba3174dc0d46040fe11614d0cc94d41185922585198b4fcef9d3"
+      let pubkey = "0x04e5fd642a0f630bbb1e4cd7df629d7b8b019457a9a74f983c0484a045cebb176def86a54185b50bbba6bbf97779173695e92835d63109c23471e6da382f922fdb"
+      let keyID2 = await client.waku_addPrivateKey(privkey)
+      check:
+        await(client.waku_getPublicKey(keyID2)) == pubkey.toPublicKey
+        await(client.waku_getPrivateKey(keyID2)).toRaw() == privkey.toPrivateKey.toRaw()
+        await(client.waku_hasKeyPair(keyID2)) == true
+        await(client.waku_deleteKeyPair(keyID2)) == true
+        await(client.waku_hasKeyPair(keyID2)) == false
+      expect ValueError:
+        discard await(client.waku_deleteKeyPair(keyID2))
+    test "waku symKey tests":
+      let keyID = await client.waku_newSymKey()
+      check:
+        await(client.waku_hasSymKey(keyID)) == true
+        await(client.waku_deleteSymKey(keyID)) == true
+        await(client.waku_hasSymKey(keyID)) == false
+      expect ValueError:
+        discard await(client.waku_deleteSymKey(keyID))
+
+      let symKey = "0x0000000000000000000000000000000000000000000000000000000000000001"
+      let keyID2 = await client.waku_addSymKey(symKey)
+      check:
+        await(client.waku_getSymKey(keyID2)) == symKey.toSymKey
+        await(client.waku_hasSymKey(keyID2)) == true
+        await(client.waku_deleteSymKey(keyID2)) == true
+        await(client.waku_hasSymKey(keyID2)) == false
+      expect ValueError:
+        discard await(client.waku_deleteSymKey(keyID2))
+
+      let keyID3 = await client.waku_generateSymKeyFromPassword("password")
+      let keyID4 = await client.waku_generateSymKeyFromPassword("password")
+      let keyID5 = await client.waku_generateSymKeyFromPassword("nimbus!")
+      check:
+        await(client.waku_getSymKey(keyID3)) ==
+          await(client.waku_getSymKey(keyID4))
+        await(client.waku_getSymKey(keyID3)) !=
+          await(client.waku_getSymKey(keyID5))
+        await(client.waku_hasSymKey(keyID3)) == true
+        await(client.waku_deleteSymKey(keyID3)) == true
+        await(client.waku_hasSymKey(keyID3)) == false
+      expect ValueError:
+        discard await(client.waku_deleteSymKey(keyID3))
+
+    # Some defaults for the filter & post tests
+    let
+      ttl = 30'u64
+      topicStr = "0x12345678"
+      payload = "0x45879632"
+      # A very low target and long time so we are sure the test never fails
+      # because of this
+      powTarget = 0.001
+      powTime = 1.0
+
+    test "waku filter create and delete":
+      let
+        topic = topicStr.toTopic()
+        symKeyID = await client.waku_newSymKey()
+        options = WakuFilterOptions(symKeyID: some(symKeyID),
+                                       topics: some(@[topic]))
+        filterID = await client.waku_newMessageFilter(options)
+
+      check:
+        filterID.string.isValidIdentifier
+        await(client.waku_deleteMessageFilter(filterID)) == true
+      expect ValueError:
+        discard await(client.waku_deleteMessageFilter(filterID))
+
+    test "waku symKey post and filter loop":
+      let
+        topic = topicStr.toTopic()
+        symKeyID = await client.waku_newSymKey()
+        options = WakuFilterOptions(symKeyID: some(symKeyID),
+                                       topics: some(@[topic]))
+        filterID = await client.waku_newMessageFilter(options)
+        message = WakuPostMessage(symKeyID: some(symKeyID),
+                                     ttl: ttl,
+                                     topic: some(topic),
+                                     payload: payload.HexDataStr,
+                                     powTime: powTime,
+                                     powTarget: powTarget)
+      check:
+        await(client.waku_setMinPoW(powTarget)) == true
+        await(client.waku_post(message)) == true
+
+      let messages = await client.waku_getFilterMessages(filterID)
+      check:
+        messages.len == 1
+        messages[0].sig.isNone()
+        messages[0].recipientPublicKey.isNone()
+        messages[0].ttl == ttl
+        messages[0].topic == topic
+        messages[0].payload == hexToSeqByte(payload)
+        messages[0].padding.len > 0
+        messages[0].pow >= powTarget
+
+        await(client.waku_deleteMessageFilter(filterID)) == true
+
+    test "waku asymKey post and filter loop":
+      let
+        topic = topicStr.toTopic()
+        privateKeyID = await client.waku_newKeyPair()
+        options = WakuFilterOptions(privateKeyID: some(privateKeyID))
+        filterID = await client.waku_newMessageFilter(options)
+        pubKey = await client.waku_getPublicKey(privateKeyID)
+        message = WakuPostMessage(pubKey: some(pubKey),
+                                     ttl: ttl,
+                                     topic: some(topic),
+                                     payload: payload.HexDataStr,
+                                     powTime: powTime,
+                                     powTarget: powTarget)
+      check:
+        await(client.waku_setMinPoW(powTarget)) == true
+        await(client.waku_post(message)) == true
+
+      let messages = await client.waku_getFilterMessages(filterID)
+      check:
+        messages.len == 1
+        messages[0].sig.isNone()
+        messages[0].recipientPublicKey.get() == pubKey
+        messages[0].ttl == ttl
+        messages[0].topic == topic
+        messages[0].payload == hexToSeqByte(payload)
+        messages[0].padding.len > 0
+        messages[0].pow >= powTarget
+
+        await(client.waku_deleteMessageFilter(filterID)) == true
+
+    test "waku signature in post and filter loop":
+      let
+        topic = topicStr.toTopic()
+        symKeyID = await client.waku_newSymKey()
+        privateKeyID = await client.waku_newKeyPair()
+        pubKey = await client.waku_getPublicKey(privateKeyID)
+        options = WakuFilterOptions(symKeyID: some(symKeyID),
+                                       topics: some(@[topic]),
+                                       sig: some(pubKey))
+        filterID = await client.waku_newMessageFilter(options)
+        message = WakuPostMessage(symKeyID: some(symKeyID),
+                                     sig: some(privateKeyID),
+                                     ttl: ttl,
+                                     topic: some(topic),
+                                     payload: payload.HexDataStr,
+                                     powTime: powTime,
+                                     powTarget: powTarget)
+      check:
+        await(client.waku_setMinPoW(powTarget)) == true
+        await(client.waku_post(message)) == true
+
+      let messages = await client.waku_getFilterMessages(filterID)
+      check:
+        messages.len == 1
+        messages[0].sig.get() == pubKey
+        messages[0].recipientPublicKey.isNone()
+        messages[0].ttl == ttl
+        messages[0].topic == topic
+        messages[0].payload == hexToSeqByte(payload)
+        messages[0].padding.len > 0
+        messages[0].pow >= powTarget
+
+        await(client.waku_deleteMessageFilter(filterID)) == true
+
+  rpcServer.stop()
+  rpcServer.close()
+
+waitFor doTests()

vendor/nimbus

@@ -1 +0,0 @@
-Subproject commit ff028982d6fc414c17d98d4c5e6d6d3856c0c598

waku.nimble

@@ -38,8 +38,7 @@ proc test(name: string, lang = "c") =
 
 ### Tasks
 task test, "Run tests":
-  discard
+  test "all_tests"
-#   test "all_tests"
 
 task wakunode, "Build Waku cli":
   buildBinary "wakunode", "waku/node/v0/", "-d:chronicles_log_level=TRACE"

waku/node/v0/config.nim

@@ -1,6 +1,5 @@
 import
-  confutils/defs, chronicles, chronos,
+  confutils/defs, chronicles, chronos, eth/keys
-  eth/keys, eth/p2p/rlpx_protocols/waku_protocol
 
 type
   Fleet* =  enum

waku/node/v0/quicksim.nim

@@ -1,7 +1,7 @@
 import
   os, strformat, chronicles, json_rpc/[rpcclient, rpcserver], nimcrypto/sysrand,
   eth/common as eth_common, eth/keys, eth/p2p/rlpx_protocols/waku_protocol,
-  ../../vendor/nimbus/nimbus/rpc/[hexstrings, rpc_types, waku],
+  ./rpc/[hexstrings, rpc_types],
   options as what # TODO: Huh? Redefinition?
 
 from os import DirSep
@@ -33,18 +33,18 @@ let
   symKey = "0x0000000000000000000000000000000000000000000000000000000000000001"
   topics = generateTopics()
   symKeyID = waitFor lightNode.waku_addSymKey(symKey)
-  options = WhisperFilterOptions(symKeyID: some(symKeyID),
+  options = WakuFilterOptions(symKeyID: some(symKeyID),
                                  topics: some(topics))
   filterID = waitFor lightNode.waku_newMessageFilter(options)
 
   symKeyID2 = waitFor lightNode2.waku_addSymKey(symKey)
-  options2 = WhisperFilterOptions(symKeyID: some(symKeyID2),
+  options2 = WakuFilterOptions(symKeyID: some(symKeyID2),
                                   topics: some(topics))
   filterID2 = waitFor lightNode2.waku_newMessageFilter(options2)
 
   symkeyID3 = waitFor trafficNode.waku_addSymKey(symKey)
 
-var message = WhisperPostMessage(symKeyID: some(symkeyID3),
+var message = WakuPostMessage(symKeyID: some(symkeyID3),
                                 ttl: 30,
                                 topic: some(topics[0]),
                                 payload: "0x45879632".HexDataStr,

waku/node/v0/rpc/hexstrings.nim

@@ -0,0 +1,225 @@
+# Nimbus
+# Copyright (c) 2018 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+## This module implements the Ethereum hexadecimal string formats for JSON
+## See: https://github.com/ethereum/wiki/wiki/JSON-RPC#hex-value-encoding
+
+#[
+  Note:
+  The following types are converted to hex strings when marshalled to JSON:
+    * Hash256
+    * UInt256
+    * seq[byte]
+    * openArray[seq]
+    * PublicKey
+    * PrivateKey
+    * SymKey
+    * Topic
+    * Bytes
+]#
+
+import
+  stint, stew/byteutils, eth/[keys, rlp], eth/common/eth_types,
+  eth/p2p/rlpx_protocols/waku_protocol
+
+type
+  HexDataStr* = distinct string
+  Identifier* = distinct string        # 32 bytes, no 0x prefix!
+  HexStrings = HexDataStr | Identifier
+
+# Hex validation
+
+template hasHexHeader(value: string): bool =
+  if value.len >= 2 and value[0] == '0' and value[1] in {'x', 'X'}: true
+  else: false
+
+template isHexChar(c: char): bool =
+  if  c notin {'0'..'9'} and
+      c notin {'a'..'f'} and
+      c notin {'A'..'F'}: false
+  else: true
+
+func isValidHexQuantity*(value: string): bool =
+  if not value.hasHexHeader:
+    return false
+  # No leading zeros (but allow 0x0)
+  if value.len < 3 or (value.len > 3 and value[2] == '0'): return false
+  for i in 2 ..< value.len:
+    let c = value[i]
+    if not c.isHexChar:
+      return false
+  return true
+
+func isValidHexData*(value: string, header = true): bool =
+  if header and not value.hasHexHeader:
+    return false
+  # Must be even number of digits
+  if value.len mod 2 != 0: return false
+  # Leading zeros are allowed
+  for i in 2 ..< value.len:
+    let c = value[i]
+    if not c.isHexChar:
+      return false
+  return true
+
+template isValidHexData(value: string, hexLen: int, header = true): bool =
+  value.len == hexLen and value.isValidHexData(header)
+
+func isValidIdentifier*(value: string): bool =
+  # 32 bytes for Whisper ID, no 0x prefix
+  result = value.isValidHexData(64, false)
+
+func isValidPublicKey*(value: string): bool =
+  # 65 bytes for Public Key plus 1 byte for 0x prefix
+  result = value.isValidHexData(132)
+
+func isValidPrivateKey*(value: string): bool =
+  # 32 bytes for Private Key plus 1 byte for 0x prefix
+  result = value.isValidHexData(66)
+
+func isValidSymKey*(value: string): bool =
+  # 32 bytes for Private Key plus 1 byte for 0x prefix
+  result = value.isValidHexData(66)
+
+func isValidHash256*(value: string): bool =
+  # 32 bytes for Hash256 plus 1 byte for 0x prefix
+  result = value.isValidHexData(66)
+
+func isValidTopic*(value: string): bool =
+  # 4 bytes for Topic plus 1 byte for 0x prefix
+  result = value.isValidHexData(10)
+
+const
+  SInvalidData = "Invalid hex data format for Ethereum"
+
+proc validateHexData*(value: string) {.inline.} =
+  if unlikely(not value.isValidHexData):
+    raise newException(ValueError, SInvalidData & ": " & value)
+
+# Initialisation
+
+proc hexDataStr*(value: string): HexDataStr {.inline.} =
+  value.validateHexData
+  result = value.HexDataStr
+
+# Converters for use in RPC
+
+import json
+from json_rpc/rpcserver import expect
+
+proc `%`*(value: HexStrings): JsonNode =
+  result = %(value.string)
+
+# Overloads to support expected representation of hex data
+
+proc `%`*(value: Hash256): JsonNode =
+  #result = %("0x" & $value) # More clean but no lowercase :(
+  result = %("0x" & value.data.toHex)
+
+proc `%`*(value: UInt256): JsonNode =
+  result = %("0x" & value.toString(16))
+
+proc `%`*(value: PublicKey): JsonNode =
+  result = %("0x04" & $value)
+
+proc `%`*(value: PrivateKey): JsonNode =
+  result = %("0x" & $value)
+
+proc `%`*(value: SymKey): JsonNode =
+  result = %("0x" & value.toHex)
+
+proc `%`*(value: waku_protocol.Topic): JsonNode =
+  result = %("0x" & value.toHex)
+
+proc `%`*(value: seq[byte]): JsonNode =
+  result = %("0x" & value.toHex)
+
+# Helpers for the fromJson procs
+
+proc toPublicKey*(key: string): PublicKey {.inline.} =
+  result = PublicKey.fromHex(key[4 .. ^1]).tryGet()
+
+proc toPrivateKey*(key: string): PrivateKey {.inline.} =
+  result = PrivateKey.fromHex(key[2 .. ^1]).tryGet()
+
+proc toSymKey*(key: string): SymKey {.inline.} =
+  hexToByteArray(key[2 .. ^1], result)
+
+proc toTopic*(topic: string): waku_protocol.Topic {.inline.} =
+  hexToByteArray(topic[2 .. ^1], result)
+
+# Marshalling from JSON to Nim types that includes format checking
+
+func invalidMsg(name: string): string = "When marshalling from JSON, parameter \"" & name & "\" is not valid"
+
+proc fromJson*(n: JsonNode, argName: string, result: var HexDataStr) =
+  n.kind.expect(JString, argName)
+  let hexStr = n.getStr()
+  if not hexStr.isValidHexData:
+    raise newException(ValueError, invalidMsg(argName) & " as Ethereum data \"" & hexStr & "\"")
+  result = hexStr.hexDataStr
+
+proc fromJson*(n: JsonNode, argName: string, result: var Identifier) =
+  n.kind.expect(JString, argName)
+  let hexStr = n.getStr()
+  if not hexStr.isValidIdentifier:
+    raise newException(ValueError, invalidMsg(argName) & " as a identifier \"" & hexStr & "\"")
+  result = hexStr.Identifier
+
+proc fromJson*(n: JsonNode, argName: string, result: var UInt256) =
+  n.kind.expect(JString, argName)
+  let hexStr = n.getStr()
+  if not (hexStr.len <= 66 and hexStr.isValidHexQuantity):
+    raise newException(ValueError, invalidMsg(argName) & " as a UInt256 \"" & hexStr & "\"")
+  result = readUintBE[256](hexToPaddedByteArray[32](hexStr))
+
+proc fromJson*(n: JsonNode, argName: string, result: var PublicKey) =
+  n.kind.expect(JString, argName)
+  let hexStr = n.getStr()
+  if not hexStr.isValidPublicKey:
+    raise newException(ValueError, invalidMsg(argName) & " as a public key \"" & hexStr & "\"")
+  result = hexStr.toPublicKey
+
+proc fromJson*(n: JsonNode, argName: string, result: var PrivateKey) =
+  n.kind.expect(JString, argName)
+  let hexStr = n.getStr()
+  if not hexStr.isValidPrivateKey:
+    raise newException(ValueError, invalidMsg(argName) & " as a private key \"" & hexStr & "\"")
+  result = hexStr.toPrivateKey
+
+proc fromJson*(n: JsonNode, argName: string, result: var SymKey) =
+  n.kind.expect(JString, argName)
+  let hexStr = n.getStr()
+  if not hexStr.isValidSymKey:
+    raise newException(ValueError, invalidMsg(argName) & " as a symmetric key \"" & hexStr & "\"")
+  result = toSymKey(hexStr)
+
+proc fromJson*(n: JsonNode, argName: string, result: var waku_protocol.Topic) =
+  n.kind.expect(JString, argName)
+  let hexStr = n.getStr()
+  if not hexStr.isValidTopic:
+    raise newException(ValueError, invalidMsg(argName) & " as a topic \"" & hexStr & "\"")
+  result = toTopic(hexStr)
+
+# Following procs currently required only for testing, the `createRpcSigs` macro
+# requires it as it will convert the JSON results back to the original Nim
+# types, but it needs the `fromJson` calls for those specific Nim types to do so
+proc fromJson*(n: JsonNode, argName: string, result: var seq[byte]) =
+  n.kind.expect(JString, argName)
+  let hexStr = n.getStr()
+  if not hexStr.isValidHexData:
+    raise newException(ValueError, invalidMsg(argName) & " as a hex data \"" & hexStr & "\"")
+  result = hexToSeqByte(hexStr)
+
+proc fromJson*(n: JsonNode, argName: string, result: var Hash256) =
+  n.kind.expect(JString, argName)
+  let hexStr = n.getStr()
+  if not hexStr.isValidHash256:
+    raise newException(ValueError, invalidMsg(argName) & " as a Hash256 \"" & hexStr & "\"")
+  hexToByteArray(hexStr, result.data)

waku/node/v0/rpc/key_storage.nim

@@ -0,0 +1,22 @@
+#
+#                 Nimbus
+#              (c) Copyright 2019
+#       Status Research & Development GmbH
+#
+#            Licensed under either of
+#  Apache License, version 2.0, (LICENSE-APACHEv2)
+#            MIT license (LICENSE-MIT)
+
+import tables, eth/keys, eth/p2p/rlpx_protocols/whisper/whisper_types
+
+type
+  KeyStorage* = ref object
+    asymKeys*: Table[string, KeyPair]
+    symKeys*: Table[string, SymKey]
+
+  KeyGenerationError* = object of CatchableError
+
+proc newKeyStorage*(): KeyStorage =
+  new(result)
+  result.asymKeys = initTable[string, KeyPair]()
+  result.symKeys = initTable[string, SymKey]()

waku/node/v0/rpc/rpc_types.nim

@@ -0,0 +1,58 @@
+import
+  hexstrings, options, eth/[keys, rlp],
+  eth/p2p/rlpx_protocols/waku_protocol
+
+#[
+  Notes:
+    * Some of the types suppose 'null' when there is no appropriate value.
+      To allow for this, you can use Option[T] or use refs so the JSON transform can convert to `JNull`.
+    * Parameter objects from users must have their data verified so will use EthAddressStr instead of EthAddres, for example
+    * Objects returned to the user can use native Waku types, where hexstrings provides converters to hex strings.
+      This is because returned arrays in JSON is
+      a) not an efficient use of space
+      b) not the format the user expects (for example addresses are expected to be hex strings prefixed by "0x")
+]#
+
+type
+  WakuInfo* = object
+    # Returned to user
+    minPow*: float64        # Current minimum PoW requirement.
+    # TODO: may be uint32
+    maxMessageSize*: uint64 # Current message size limit in bytes.
+    memory*: int            # Memory size of the floating messages in bytes.
+    messages*: int          # Number of floating messages.
+
+  WakuFilterOptions* = object
+    # Parameter from user
+    symKeyID*: Option[Identifier]         # ID of symmetric key for message decryption.
+    privateKeyID*: Option[Identifier]     # ID of private (asymmetric) key for message decryption.
+    sig*: Option[PublicKey]               # (Optional) Public key of the signature.
+    minPow*: Option[float64]              # (Optional) Minimal PoW requirement for incoming messages.
+    topics*: Option[seq[waku_protocol.Topic]] # (Optional when asym key): Array of possible topics (or partial topics).
+    allowP2P*: Option[bool]               # (Optional) Indicates if this filter allows processing of direct peer-to-peer messages.
+
+  WakuFilterMessage* = object
+    # Returned to user
+    sig*: Option[PublicKey]                 # Public key who signed this message.
+    recipientPublicKey*: Option[PublicKey]  # The recipients public key.
+    ttl*: uint64                            # Time-to-live in seconds.
+    timestamp*: uint64                      # Unix timestamp of the message generation.
+    topic*: waku_protocol.Topic          # 4 Bytes: Message topic.
+    payload*: seq[byte]                     # Decrypted payload.
+    padding*: seq[byte]                     # (Optional) Padding (byte array of arbitrary length).
+    pow*: float64                           # Proof of work value.
+    hash*: Hash                             # Hash of the enveloped message.
+
+  WakuPostMessage* = object
+    # Parameter from user
+    symKeyID*: Option[Identifier]     # ID of symmetric key for message encryption.
+    pubKey*: Option[PublicKey]        # Public key for message encryption.
+    sig*: Option[Identifier]          # (Optional) ID of the signing key.
+    ttl*: uint64                      # Time-to-live in seconds.
+    topic*: Option[waku_protocol.Topic] # Message topic (mandatory when key is symmetric).
+    payload*: HexDataStr              # Payload to be encrypted.
+    padding*: Option[HexDataStr]      # (Optional) Padding (byte array of arbitrary length).
+    powTime*: float64                 # Maximal time in seconds to be spent on proof of work.
+    powTarget*: float64               # Minimal PoW target required for this message.
+    # TODO: EnodeStr
+    targetPeer*: Option[string]       # (Optional) Peer ID (for peer-to-peer message only).

waku/node/v0/rpc/waku.nim

@@ -0,0 +1,363 @@
+import
+  json_rpc/rpcserver, tables, options, sequtils,
+  eth/[common, rlp, keys, p2p], eth/p2p/rlpx_protocols/waku_protocol,
+  nimcrypto/[sysrand, hmac, sha2, pbkdf2],
+  rpc_types, hexstrings, key_storage
+
+from stew/byteutils import hexToSeqByte, hexToByteArray
+
+# Blatant copy of Whisper RPC but for the Waku protocol
+
+proc setupWakuRPC*(node: EthereumNode, keys: KeyStorage, rpcsrv: RpcServer) =
+
+  rpcsrv.rpc("waku_version") do() -> string:
+    ## Returns string of the current Waku protocol version.
+    result = wakuVersionStr
+
+  rpcsrv.rpc("waku_info") do() -> WakuInfo:
+    ## Returns diagnostic information about the Waku node.
+    let config = node.protocolState(Waku).config
+    result = WakuInfo(minPow: config.powRequirement,
+                         maxMessageSize: config.maxMsgSize,
+                         memory: 0,
+                         messages: 0)
+
+  # TODO: uint32 instead of uint64 is OK here, but needs to be added in json_rpc
+  rpcsrv.rpc("waku_setMaxMessageSize") do(size: uint64) -> bool:
+    ## Sets the maximal message size allowed by this node.
+    ## Incoming and outgoing messages with a larger size will be rejected.
+    ## Waku message size can never exceed the limit imposed by the underlying
+    ## P2P protocol (10 Mb).
+    ##
+    ## size: Message size in bytes.
+    ##
+    ## Returns true on success and an error on failure.
+    result = node.setMaxMessageSize(size.uint32)
+    if not result:
+      raise newException(ValueError, "Invalid size")
+
+  rpcsrv.rpc("waku_setMinPoW") do(pow: float) -> bool:
+    ## Sets the minimal PoW required by this node.
+    ##
+    ## pow: The new PoW requirement.
+    ##
+    ## Returns true on success and an error on failure.
+    # Note: `setPowRequirement` does not raise on failures of sending the update
+    # to the peers. Hence in theory this should not causes errors.
+    await node.setPowRequirement(pow)
+    result = true
+
+  # TODO: change string in to ENodeStr with extra checks
+  rpcsrv.rpc("waku_markTrustedPeer") do(enode: string) -> bool:
+    ## Marks specific peer trusted, which will allow it to send historic
+    ## (expired) messages.
+    ## Note: This function is not adding new nodes, the node needs to exists as
+    ## a peer.
+    ##
+    ## enode: Enode of the trusted peer.
+    ##
+    ## Returns true on success and an error on failure.
+    # TODO: It will now require an enode://pubkey@ip:port uri
+    # could also accept only the pubkey (like geth)?
+    let peerNode = newNode(enode)
+    result = node.setPeerTrusted(peerNode.id)
+    if not result:
+      raise newException(ValueError, "Not a peer")
+
+  rpcsrv.rpc("waku_newKeyPair") do() -> Identifier:
+    ## Generates a new public and private key pair for message decryption and
+    ## encryption.
+    ##
+    ## Returns key identifier on success and an error on failure.
+    result = generateRandomID().Identifier
+    keys.asymKeys.add(result.string, KeyPair.random().tryGet())
+
+  rpcsrv.rpc("waku_addPrivateKey") do(key: PrivateKey) -> Identifier:
+    ## Stores the key pair, and returns its ID.
+    ##
+    ## key: Private key as hex bytes.
+    ##
+    ## Returns key identifier on success and an error on failure.
+    result = generateRandomID().Identifier
+
+    keys.asymKeys.add(result.string, key.toKeyPair().tryGet())
+
+  rpcsrv.rpc("waku_deleteKeyPair") do(id: Identifier) -> bool:
+    ## Deletes the specifies key if it exists.
+    ##
+    ## id: Identifier of key pair
+    ##
+    ## Returns true on success and an error on failure.
+    var unneeded: KeyPair
+    result = keys.asymKeys.take(id.string, unneeded)
+    if not result:
+      raise newException(ValueError, "Invalid key id")
+
+  rpcsrv.rpc("waku_hasKeyPair") do(id: Identifier) -> bool:
+    ## Checks if the Waku node has a private key of a key pair matching the
+    ## given ID.
+    ##
+    ## id: Identifier of key pair
+    ##
+    ## Returns (true or false) on success and an error on failure.
+    result = keys.asymkeys.hasKey(id.string)
+
+  rpcsrv.rpc("waku_getPublicKey") do(id: Identifier) -> PublicKey:
+    ## Returns the public key for identity ID.
+    ##
+    ## id: Identifier of key pair
+    ##
+    ## Returns public key on success and an error on failure.
+    # Note: key not found exception as error in case not existing
+    result = keys.asymkeys[id.string].pubkey
+
+  rpcsrv.rpc("waku_getPrivateKey") do(id: Identifier) -> PrivateKey:
+    ## Returns the private key for identity ID.
+    ##
+    ## id: Identifier of key pair
+    ##
+    ## Returns private key on success and an error on failure.
+    # Note: key not found exception as error in case not existing
+    result = keys.asymkeys[id.string].seckey
+
+  rpcsrv.rpc("waku_newSymKey") do() -> Identifier:
+    ## Generates a random symmetric key and stores it under an ID, which is then
+    ## returned. Can be used encrypting and decrypting messages where the key is
+    ## known to both parties.
+    ##
+    ## Returns key identifier on success and an error on failure.
+    result = generateRandomID().Identifier
+    var key: SymKey
+    if randomBytes(key) != key.len:
+      raise newException(KeyGenerationError, "Failed generating key")
+
+    keys.symKeys.add(result.string, key)
+
+
+  rpcsrv.rpc("waku_addSymKey") do(key: SymKey) -> Identifier:
+    ## Stores the key, and returns its ID.
+    ##
+    ## key: The raw key for symmetric encryption as hex bytes.
+    ##
+    ## Returns key identifier on success and an error on failure.
+    result = generateRandomID().Identifier
+
+    keys.symKeys.add(result.string, key)
+
+  rpcsrv.rpc("waku_generateSymKeyFromPassword") do(password: string) -> Identifier:
+    ## Generates the key from password, stores it, and returns its ID.
+    ##
+    ## password: Password.
+    ##
+    ## Returns key identifier on success and an error on failure.
+    ## Warning: an empty string is used as salt because the shh RPC API does not
+    ## allow for passing a salt. A very good password is necessary (calculate
+    ## yourself what that means :))
+    var ctx: HMAC[sha256]
+    var symKey: SymKey
+    if pbkdf2(ctx, password, "", 65356, symKey) != sizeof(SymKey):
+      raise newException(KeyGenerationError, "Failed generating key")
+
+    result = generateRandomID().Identifier
+    keys.symKeys.add(result.string, symKey)
+
+  rpcsrv.rpc("waku_hasSymKey") do(id: Identifier) -> bool:
+    ## Returns true if there is a key associated with the name string.
+    ## Otherwise, returns false.
+    ##
+    ## id: Identifier of key.
+    ##
+    ## Returns (true or false) on success and an error on failure.
+    result = keys.symkeys.hasKey(id.string)
+
+  rpcsrv.rpc("waku_getSymKey") do(id: Identifier) -> SymKey:
+    ## Returns the symmetric key associated with the given ID.
+    ##
+    ## id: Identifier of key.
+    ##
+    ## Returns Raw key on success and an error on failure.
+    # Note: key not found exception as error in case not existing
+    result = keys.symkeys[id.string]
+
+  rpcsrv.rpc("waku_deleteSymKey") do(id: Identifier) -> bool:
+    ## Deletes the key associated with the name string if it exists.
+    ##
+    ## id: Identifier of key.
+    ##
+    ## Returns (true or false) on success and an error on failure.
+    var unneeded: SymKey
+    result = keys.symKeys.take(id.string, unneeded)
+    if not result:
+      raise newException(ValueError, "Invalid key id")
+
+  rpcsrv.rpc("waku_subscribe") do(id: string,
+                                 options: WakuFilterOptions) -> Identifier:
+    ## Creates and registers a new subscription to receive notifications for
+    ## inbound Waku messages. Returns the ID of the newly created
+    ## subscription.
+    ##
+    ## id: identifier of function call. In case of Waku must contain the
+    ## value "messages".
+    ## options: WakuFilterOptions
+    ##
+    ## Returns the subscription ID on success, the error on failure.
+
+    # TODO: implement subscriptions, only for WS & IPC?
+    discard
+
+  rpcsrv.rpc("waku_unsubscribe") do(id: Identifier) -> bool:
+    ## Cancels and removes an existing subscription.
+    ##
+    ## id: Subscription identifier
+    ##
+    ## Returns true on success, the error on failure
+    result  = node.unsubscribeFilter(id.string)
+    if not result:
+      raise newException(ValueError, "Invalid filter id")
+
+  proc validateOptions[T,U,V](asym: Option[T], sym: Option[U], topic: Option[V]) =
+    if (asym.isSome() and sym.isSome()) or (asym.isNone() and sym.isNone()):
+      raise newException(ValueError,
+                         "Either privateKeyID/pubKey or symKeyID must be present")
+    if asym.isNone() and topic.isNone():
+      raise newException(ValueError, "Topic mandatory with symmetric key")
+
+  rpcsrv.rpc("waku_newMessageFilter") do(options: WakuFilterOptions) -> Identifier:
+    ## Create a new filter within the node. This filter can be used to poll for
+    ## new messages that match the set of criteria.
+    ##
+    ## options: WakuFilterOptions
+    ##
+    ## Returns filter identifier on success, error on failure
+
+    # Check if either symKeyID or privateKeyID is present, and not both
+    # Check if there are Topics when symmetric key is used
+    validateOptions(options.privateKeyID, options.symKeyID, options.topics)
+
+    var
+      src: Option[PublicKey]
+      privateKey: Option[PrivateKey]
+      symKey: Option[SymKey]
+      topics: seq[waku_protocol.Topic]
+      powReq: float64
+      allowP2P: bool
+
+    src = options.sig
+
+    if options.privateKeyID.isSome():
+      privateKey = some(keys.asymKeys[options.privateKeyID.get().string].seckey)
+
+    if options.symKeyID.isSome():
+      symKey= some(keys.symKeys[options.symKeyID.get().string])
+
+    if options.minPow.isSome():
+      powReq = options.minPow.get()
+
+    if options.topics.isSome():
+      topics = options.topics.get()
+
+    if options.allowP2P.isSome():
+      allowP2P = options.allowP2P.get()
+
+    let filter = initFilter(src, privateKey, symKey, topics, powReq, allowP2P)
+    result = node.subscribeFilter(filter).Identifier
+
+    # TODO: Should we do this here "automatically" or separate it in another
+    # RPC call? Is there a use case for that?
+    # Same could be said about bloomfilter, except that there is a use case
+    # there to have a full node no matter what message filters.
+    # Could also be moved to waku_protocol.nim
+    let config = node.protocolState(Waku).config
+    if config.topics.isSome():
+      try:
+        # TODO: an addTopics call would probably be more useful
+        let result = await node.setTopicInterest(config.topics.get().concat(filter.topics))
+        if not result:
+          raise newException(ValueError, "Too many topics")
+      except CatchableError:
+        trace "setTopics error occured"
+    elif config.isLightNode:
+      try:
+        await node.setBloomFilter(node.filtersToBloom())
+      except CatchableError:
+        trace "setBloomFilter error occured"
+
+  rpcsrv.rpc("waku_deleteMessageFilter") do(id: Identifier) -> bool:
+    ## Uninstall a message filter in the node.
+    ##
+    ## id: Filter identifier as returned when the filter was created.
+    ##
+    ## Returns true on success, error on failure.
+    result = node.unsubscribeFilter(id.string)
+    if not result:
+      raise newException(ValueError, "Invalid filter id")
+
+  rpcsrv.rpc("waku_getFilterMessages") do(id: Identifier) -> seq[WakuFilterMessage]:
+    ## Retrieve messages that match the filter criteria and are received between
+    ## the last time this function was called and now.
+    ##
+    ## id: ID of filter that was created with `waku_newMessageFilter`.
+    ##
+    ## Returns array of messages on success and an error on failure.
+    let messages = node.getFilterMessages(id.string)
+    for msg in messages:
+      result.add WakuFilterMessage(
+        sig: msg.decoded.src,
+        recipientPublicKey: msg.dst,
+        ttl: msg.ttl,
+        topic: msg.topic,
+        timestamp: msg.timestamp,
+        payload: msg.decoded.payload,
+        # Note: waku_protocol padding is an Option as there is the
+        # possibility of 0 padding in case of custom padding.
+        padding: msg.decoded.padding.get(@[]),
+        pow: msg.pow,
+        hash: msg.hash)
+
+  rpcsrv.rpc("waku_post") do(message: WakuPostMessage) -> bool:
+    ## Creates a Waku message and injects it into the network for
+    ## distribution.
+    ##
+    ## message: Waku message to post.
+    ##
+    ## Returns true on success and an error on failure.
+
+    # Check if either symKeyID or pubKey is present, and not both
+    # Check if there is a Topic when symmetric key is used
+    validateOptions(message.pubKey, message.symKeyID, message.topic)
+
+    var
+      sigPrivKey: Option[PrivateKey]
+      symKey: Option[SymKey]
+      topic: waku_protocol.Topic
+      padding: Option[seq[byte]]
+      targetPeer: Option[NodeId]
+
+    if message.sig.isSome():
+      sigPrivKey = some(keys.asymKeys[message.sig.get().string].seckey)
+
+    if message.symKeyID.isSome():
+      symKey = some(keys.symKeys[message.symKeyID.get().string])
+
+    # Note: If no topic it will be defaulted to 0x00000000
+    if message.topic.isSome():
+      topic = message.topic.get()
+
+    if message.padding.isSome():
+      padding = some(hexToSeqByte(message.padding.get().string))
+
+    if message.targetPeer.isSome():
+      targetPeer = some(newNode(message.targetPeer.get()).id)
+
+    result = node.postMessage(message.pubKey,
+                              symKey,
+                              sigPrivKey,
+                              ttl = message.ttl.uint32,
+                              topic = topic,
+                              payload = hexToSeqByte(message.payload.string),
+                              padding = padding,
+                              powTime = message.powTime,
+                              powTarget = message.powTarget,
+                              targetPeer = targetPeer)
+    if not result:
+      raise newException(ValueError, "Message could not be posted")

waku/node/v0/rpc/wakucallsigs.nim

@@ -1,5 +1,5 @@
 proc waku_version(): string
-proc waku_info(): WhisperInfo
+proc waku_info(): WakuInfo
 proc waku_setMaxMessageSize(size: uint64): bool
 proc waku_setMinPoW(pow: float): bool
 proc waku_markTrustedPeer(enode: string): bool
@@ -18,10 +18,10 @@ proc waku_hasSymKey(id: Identifier): bool
 proc waku_getSymKey(id: Identifier): SymKey
 proc waku_deleteSymKey(id: Identifier): bool
 
-proc waku_newMessageFilter(options: WhisperFilterOptions): Identifier
+proc waku_newMessageFilter(options: WakuFilterOptions): Identifier
 proc waku_deleteMessageFilter(id: Identifier): bool
-proc waku_getFilterMessages(id: Identifier): seq[WhisperFilterMessage]
+proc waku_getFilterMessages(id: Identifier): seq[WakuFilterMessage]
-proc waku_post(message: WhisperPostMessage): bool
+proc waku_post(message: WakuPostMessage): bool
 
 proc wakusim_generateTraffic(amount: int): bool
 proc wakusim_generateRandomTraffic(amount: int): bool

waku/node/v0/rpc/wakusim.nim

@@ -0,0 +1,31 @@
+import
+  json_rpc/rpcserver, stew/endians2, nimcrypto/sysrand,
+  eth/[p2p, async_utils], eth/p2p/rlpx_protocols/waku_protocol
+
+proc generateTraffic(node: EthereumNode, amount = 100) {.async.} =
+  var topicNumber  = 0'u32
+  let payload = @[byte 0]
+  for i in 0..<amount:
+    discard waku_protocol.postMessage(node, ttl = 10,
+      topic = toBytesLE(i.uint32), payload = payload)
+    await sleepAsync(1.milliseconds)
+
+proc generateRandomTraffic(node: EthereumNode, amount = 100) {.async.} =
+  var topic: array[4, byte]
+  let payload = @[byte 0]
+  for i in 0..<amount:
+    while randomBytes(topic) != 4:
+      discard
+    discard waku_protocol.postMessage(node, ttl = 10, topic = topic,
+      payload = payload)
+    await sleepAsync(1.milliseconds)
+
+proc setupWakuSimRPC*(node: EthereumNode, rpcsrv: RpcServer) =
+
+  rpcsrv.rpc("wakusim_generateTraffic") do(amount: int) -> bool:
+    traceAsyncErrors node.generateTraffic(amount)
+    return true
+
+  rpcsrv.rpc("wakusim_generateRandomTraffic") do(amount: int) -> bool:
+    traceAsyncErrors node.generateRandomTraffic(amount)
+    return true

waku/node/v0/wakunode.nim

@@ -4,7 +4,7 @@ import
   eth/[keys, p2p, async_utils], eth/common/utils, eth/net/nat,
   eth/p2p/[discovery, enode, peer_pool, bootnodes, whispernodes],
   eth/p2p/rlpx_protocols/[whisper_protocol, waku_protocol, waku_bridge],
-  ../../vendor/nimbus/nimbus/rpc/[waku, wakusim, key_storage]
+  ./rpc/[waku, wakusim, key_storage]
 
 const clientId = "Nimbus waku node"