Refactor: Put waku_type content where it belongs (#333)

- Separate out types and methods into appropriate module
- Add util folder for small utils used in multiple independent places
- Update paths

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## Next version
+
+- Refactor: Split out `waku_types` types into right place; create utils folder.
+
 ## 2021-01-05 v0.2
 
 This release contains the following changes:

examples/v2/basic2.nim

@@ -10,11 +10,7 @@ import
   json_rpc/[rpcclient, rpcserver],
   ../../waku/v2/node/[config, wakunode2],
   ../../waku/common/utils/nat,
-  ../../waku/v2/waku_types
-
-
-type
-  Topic* = waku_types.Topic
+  ../../waku/v2/protocol/waku_message
 
 # Node operations happens asynchronously
 proc runBackground() {.async.} =

examples/v2/chat2.nim

@@ -19,11 +19,10 @@ import libp2p/[switch,                   # manage transports, a single entry poi
                muxers/muxer,             # define an interface for stream multiplexing, allowing peers to offer many protocols over a single connection
                muxers/mplex/mplex]       # define some contants and message types for stream multiplexing
 import   ../../waku/v2/node/[config, wakunode2, waku_payload],
-         ../../waku/v2/protocol/[waku_relay],
+         ../../waku/v2/protocol/[waku_relay, waku_message],
          ../../waku/v2/protocol/waku_store/waku_store,
          ../../waku/v2/protocol/waku_filter/waku_filter,
-         ../../waku/common/utils/nat,
-         ../../waku/v2/waku_types
+         ../../waku/common/utils/nat
 
 const Help = """
   Commands: /[?|help|connect|disconnect|exit]
@@ -52,7 +51,7 @@ type Chat = ref object
 
 type
   PrivateKey* = crypto.PrivateKey
-  Topic* = waku_types.Topic
+  Topic* = wakunode2.Topic
 
 
 # Similarly as Status public chats now.

tests/v2/test_jsonrpc_waku.nim

@@ -10,7 +10,6 @@ import
   libp2p/protocols/pubsub/pubsub,
   libp2p/protocols/pubsub/rpc/message,
   ../../waku/v1/node/rpc/hexstrings,
-  ../../waku/v2/waku_types,
   ../../waku/v2/node/wakunode2,
   ../../waku/v2/node/jsonrpc/[store_api,
                               relay_api,

tests/v2/test_message_store.nim

@@ -5,7 +5,6 @@ import
   ../../waku/v2/node/message_store,
   ../../waku/v2/protocol/waku_store/waku_store,
   ./utils,
-  ../../waku/v2/waku_types,
   ../../waku/v2/node/sqlite
 
 suite "Message Store":

tests/v2/test_rpc_waku.nim

@@ -8,7 +8,6 @@ import
   ../../waku/v2/node/wakunode2,
   ../../waku/v2/node/rpc/wakurpc,
   ../../waku/v2/protocol/waku_relay,
-  ../../waku/v2/waku_types,
   ../test_helpers
 
 

tests/v2/test_waku_filter.nim

@@ -10,7 +10,6 @@ import
   libp2p/multistream,
   ../../waku/v2/protocol/[message_notifier],
   ../../waku/v2/protocol/waku_filter/waku_filter,
-  ../../waku/v2/waku_types,
   ../test_helpers, ./utils
 
 procSuite "Waku Filter":

tests/v2/test_waku_pagination.nim

@@ -2,7 +2,6 @@
 import
   std/[unittest,algorithm,options],
   nimcrypto/sha2,
-  ../../waku/v2/waku_types,
   ../../waku/v2/protocol/waku_store/waku_store,
   ../test_helpers
 

tests/v2/test_waku_payload.nim

@@ -2,6 +2,7 @@
 
 import
   std/unittest,
+  ../../waku/v2/protocol/waku_message,
   ../../waku/v2/node/waku_payload,
   ../test_helpers
 

tests/v2/test_waku_store.nim

@@ -8,11 +8,10 @@ import
   libp2p/stream/[bufferstream, connection],
   libp2p/crypto/crypto,
   libp2p/protocols/pubsub/rpc/message,
-  ../../waku/v2/protocol/message_notifier,
+  ../../waku/v2/protocol/[waku_message, message_notifier],
   ../../waku/v2/protocol/waku_store/waku_store,
   ../../waku/v2/node/[message_store, sqlite],
-  ../test_helpers, ./utils,
-  ../../waku/v2/waku_types
+  ../test_helpers, ./utils
 
 procSuite "Waku Store":
   asyncTest "handle query":

tests/v2/test_waku_swap.nim

@@ -7,12 +7,11 @@ import
   libp2p/crypto/[crypto, secp],
   libp2p/switch,
   eth/keys,
-  ../../waku/v2/protocol/[message_notifier],
+  ../../waku/v2/protocol/[waku_message, message_notifier],
   ../../waku/v2/protocol/waku_store/waku_store,
   ../../waku/v2/protocol/waku_swap/waku_swap,
   ../../waku/v2/node/wakunode2,
-  ../test_helpers, ./utils,
-  ../../waku/v2/waku_types
+  ../test_helpers, ./utils
 
 procSuite "Waku SWAP Accounting":
   test "Handshake Encode/Decode":

tests/v2/test_wakunode.nim

@@ -7,12 +7,11 @@ import
   libp2p/crypto/secp,
   libp2p/switch,
   eth/keys,
-  ../../waku/v2/protocol/[waku_relay, message_notifier],
+  ../../waku/v2/protocol/[waku_relay, waku_message, message_notifier],
   ../../waku/v2/protocol/waku_store/waku_store,
   ../../waku/v2/protocol/waku_filter/waku_filter,
   ../../waku/v2/node/wakunode2,
-  ../test_helpers,
-  ../../waku/v2/waku_types
+  ../test_helpers
 
 procSuite "WakuNode":
   let rng = keys.newRng()

tests/v2/utils.nim

@@ -8,7 +8,7 @@ import chronos
 import libp2p/[standard_setup,
                protocols/pubsub/pubsub,
                protocols/secure/secure]
-import ../../waku/v2/waku_types
+import ../../waku/v2/protocol/waku_relay
 
 export standard_setup
 

waku/v2/node/jsonrpc/admin_api.nim

@@ -4,7 +4,6 @@ import
   std/[options,sequtils],
   json_rpc/rpcserver,
   libp2p/[peerinfo, switch],
-  ../../waku_types,
   ../../protocol/waku_store/[waku_store_types, waku_store],
   ../../protocol/waku_swap/[waku_swap_types, waku_swap],
   ../../protocol/waku_filter/[waku_filter_types, waku_filter],

waku/v2/node/jsonrpc/debug_api.nim

@@ -1,6 +1,5 @@
 import
   json_rpc/rpcserver,
-  ../../waku_types,
   ../wakunode2
 
 proc installDebugApiHandlers*(node: WakuNode, rpcsrv: RpcServer) =

waku/v2/node/jsonrpc/filter_api.nim

@@ -4,7 +4,6 @@ import
   std/[tables,sequtils],
   json_rpc/rpcserver,
   eth/[common, rlp, keys, p2p],
-  ../../waku_types,
   ../../protocol/waku_filter/waku_filter_types,
   ../wakunode2,
   ./jsonrpc_types

waku/v2/node/jsonrpc/jsonrpc_types.nim

@@ -1,7 +1,8 @@
 import
+  std/[options,tables],
   eth/keys,
-  ../../waku_types,
-  std/[options,tables]
+  ../../protocol/waku_message,
+  ../../utils/pagination
 
 type
   StoreResponse* = object

waku/v2/node/jsonrpc/jsonrpc_utils.nim

@@ -2,7 +2,6 @@ import
   std/options,
   eth/keys,
   ../../../v1/node/rpc/hexstrings,
-  ../../waku_types,
   ../../protocol/waku_store/waku_store_types,
   ../wakunode2, ../waku_payload,
   ./jsonrpc_types

waku/v2/node/jsonrpc/private_api.nim

@@ -5,7 +5,6 @@ import
   json_rpc/rpcserver,
   nimcrypto/sysrand,
   eth/[common, rlp, keys, p2p],
-  ../../waku_types,  
   ../wakunode2, ../waku_payload,
   ./jsonrpc_types, ./jsonrpc_utils
 

waku/v2/node/jsonrpc/relay_api.nim

@@ -5,9 +5,9 @@ import
   json_rpc/rpcserver,
   libp2p/protocols/pubsub/pubsub,
   eth/[common, rlp, keys, p2p],
-  ../../waku_types,  
   ../wakunode2,
-  ./jsonrpc_types, ./jsonrpc_utils
+  ./jsonrpc_types, ./jsonrpc_utils,
+  ../../protocol/waku_message
 
 export jsonrpc_types
 

waku/v2/node/jsonrpc/store_api.nim

@@ -3,7 +3,6 @@
 import
   std/options,
   json_rpc/rpcserver,
-  ../../waku_types,
   ../../protocol/waku_store/waku_store_types,
   ../wakunode2,
   ./jsonrpc_types, ./jsonrpc_utils

waku/v2/node/message_store.nim

@@ -6,8 +6,9 @@ import
   libp2p/protobuf/minprotobuf,
   libp2p/stream/connection,
   stew/results, metrics,
-  ../waku_types,
-  ./sqlite
+  ./sqlite,
+  ../protocol/waku_message,
+  ../utils/pagination
 
 # The code in this file is an adaptation of the Sqlite KV Store found in nim-eth.
 # https://github.com/status-im/nim-eth/blob/master/eth/db/kvstore_sqlite3.nim
@@ -17,6 +18,11 @@ import
 type
   DataProc* = proc(timestamp: uint64, msg: WakuMessage) {.closure.}
 
+  MessageStoreResult*[T] = Result[T, string]
+
+  MessageStore* = ref object of RootObj
+    database*: SqliteDatabase
+
 proc init*(T: type MessageStore, db: SqliteDatabase): MessageStoreResult[T] =
   ## Table is the SQL query for creating the messages Table.
   ## It contains:

waku/v2/node/rpc/rpc_publish.nim

@@ -4,8 +4,7 @@ import
   libp2p/protobuf/minprotobuf,
   eth/common as eth_common, eth/keys,
   system,
-  options,
-  ../../waku_types
+  options
 
 from strutils import rsplit
 template sourceDir: string = currentSourcePath.rsplit(DirSep, 1)[0]

waku/v2/node/rpc/wakurpc.nim

@@ -4,7 +4,6 @@ import
   nimcrypto/[sysrand, hmac, sha2],
   eth/[common, rlp, keys, p2p],
   ../../protocol/waku_relay,
-  ../../waku_types,
   ../../protocol/waku_store/waku_store,
   ../../protocol/waku_filter/waku_filter,
   ../wakunode2

waku/v2/node/waku_payload.nim

@@ -2,9 +2,9 @@ import
   std/options,
   eth/keys,
   eth/p2p/rlpx_protocols/whisper/whisper_types,
-  ../waku_types
+  ../protocol/waku_message
 
-export whisper_types, waku_types, keys, options
+export whisper_types, keys, options
 
 type
   KeyKind* = enum
@@ -21,6 +21,10 @@ type
     of None:
       discard
 
+  # NOTE: Currently only used here, if we start using it elsewhere pull it out.
+  WakuResult*[T] = Result[T, cstring]
+
+
 # TODO:
 # - This is using `DecodedPayload` from Waku v1 / Whisper and could be altered
 # by making that a case object also, e.g. useful for the version 0, but

waku/v2/node/wakunode2.nim

@@ -14,11 +14,9 @@ import
   ../protocol/waku_store/waku_store,
   ../protocol/waku_swap/waku_swap,
   ../protocol/waku_filter/waku_filter,
-  ../waku_types,
   ./message_store,
-  ./sqlite
-
-export waku_types
+  ./sqlite,
+  ../utils/requests
 
 logScope:
   topics = "wakunode"
@@ -32,10 +30,15 @@ type
   PublicKey* = crypto.PublicKey
   PrivateKey* = crypto.PrivateKey
 
-  # TODO Get rid of this and use waku_types one
-  Topic* = waku_types.Topic
+  # XXX: Weird type, should probably be using pubsub Topic object name?
+  Topic* = string
   Message* = seq[byte]
 
+  WakuInfo* = object
+    # NOTE One for simplicity, can extend later as needed
+    listenStr*: string
+    #multiaddrStrings*: seq[string]
+
   # NOTE based on Eth2Node in NBC eth2_network.nim
   WakuNode* = ref object of RootObj
     switch*: Switch

waku/v2/protocol/message_notifier.nim

@@ -1,7 +1,7 @@
 import
   std/tables,
   chronos,
-  ../waku_types
+  waku_message
 
 ## The Message Notification system is a method to notify various protocols
 ## running on a node when a new message was received.
@@ -23,6 +23,19 @@ import
 ##   subscriptions["identifier"] = subscription
 ## 
 ##   await subscriptions.notify(topic, WakuMessage(payload: @[byte 1, 2, 3], contentTopic: ContentTopic(1)))
+
+type
+  MessageNotificationHandler* = proc(topic: string, msg: WakuMessage): Future[
+    void] {.gcsafe, closure.}
+
+  MessageNotificationSubscriptionIdentifier* = string
+
+  MessageNotificationSubscription* = object
+    topics*: seq[string] # @TODO TOPIC
+    handler*: MessageNotificationHandler
+
+  MessageNotificationSubscriptions* = TableRef[MessageNotificationSubscriptionIdentifier, MessageNotificationSubscription]
+
 proc subscribe*(subscriptions: MessageNotificationSubscriptions, name: string, subscription: MessageNotificationSubscription) =
   subscriptions.add(name, subscription)
 

waku/v2/protocol/waku_filter/waku_filter.nim

@@ -11,8 +11,8 @@ import
   libp2p/crypto/crypto,
   libp2p/switch,
   ../message_notifier,
-  ../../waku_types,
-  waku_filter_types
+  waku_filter_types,
+  ../../utils/requests
 
 # NOTE This is just a start, the design of this protocol isn't done yet. It
 # should be direct payload exchange (a la req-resp), not be coupled with the

waku/v2/protocol/waku_filter/waku_filter_types.nim

@@ -3,7 +3,9 @@ import
   bearssl,
   libp2p/[switch, peerinfo],
   libp2p/protocols/protocol,
-  ../../waku_types
+  ../waku_message
+
+export waku_message
 
 type
   ContentFilter* = object

waku/v2/protocol/waku_message.nim

@@ -0,0 +1,36 @@
+## Waku Message module.
+##
+## See https://github.com/vacp2p/specs/blob/master/specs/waku/v2/waku-message.md
+## for spec.
+##
+## For payload content and encryption, see waku/v2/node/waku_payload.nim
+
+
+import
+  libp2p/protobuf/minprotobuf
+
+type
+  ContentTopic* = uint32
+
+  WakuMessage* = object
+    payload*: seq[byte]
+    contentTopic*: ContentTopic
+    version*: uint32
+
+# Encoding and decoding -------------------------------------------------------
+proc init*(T: type WakuMessage, buffer: seq[byte]): ProtoResult[T] =
+  var msg = WakuMessage()
+  let pb = initProtoBuffer(buffer)
+
+  discard ? pb.getField(1, msg.payload)
+  discard ? pb.getField(2, msg.contentTopic)
+  discard ? pb.getField(3, msg.version)
+
+  ok(msg)
+
+proc encode*(message: WakuMessage): ProtoBuffer =
+  result = initProtoBuffer()
+
+  result.write(1, message.payload)
+  result.write(2, message.contentTopic)
+  result.write(3, message.version)

waku/v2/protocol/waku_relay.nim

@@ -1,14 +1,13 @@
-## Waku on libp2p
+## Waku Relay module. Thin layer on top of GossipSub.
 ##
-## This file should eventually correspond to waku_protocol as RLPx subprotocol.
-## Instead, it should likely be on top of GossipSub with a similar interface.
+## See https://github.com/vacp2p/specs/blob/master/specs/waku/v2/waku-relay.md
+## for spec.
 
 import
   chronos, chronicles, metrics,
   libp2p/protocols/pubsub/[pubsub, gossipsub],
   libp2p/protocols/pubsub/rpc/messages,
-  libp2p/stream/connection,
-  ../waku_types
+  libp2p/stream/connection
 
 declarePublicGauge total_messages, "number of messages received"
 
@@ -17,6 +16,9 @@ logScope:
 
 const WakuRelayCodec* = "/vac/waku/relay/2.0.0-beta2"
 
+type
+  WakuRelay* = ref object of GossipSub
+
 method init*(w: WakuRelay) =
   debug "init"
   proc handler(conn: Connection, proto: string) {.async.} =

waku/v2/protocol/waku_store/waku_store.nim

@@ -14,8 +14,8 @@ import
   ../message_notifier,
   ../../node/message_store,
   ../waku_swap/waku_swap,
-  ../../waku_types,
-  ./waku_store_types
+  ./waku_store_types,
+  ../../utils/requests
 
 export waku_store_types
 

waku/v2/protocol/waku_store/waku_store_types.nim

@@ -4,10 +4,19 @@ import
   bearssl,
   libp2p/[switch, peerinfo],
   libp2p/protocols/protocol,
-  ../../waku_types,
-  ../waku_swap/waku_swap_types
+  ../waku_swap/waku_swap_types,
+  ../waku_message,
+  ../../node/message_store,
+  ../../utils/pagination
+
+export waku_message
+export pagination
+
+# Constants required for pagination -------------------------------------------
+const MaxPageSize* = 100 # Maximum number of waku messages in each page
 
 type
+
   QueryHandlerFunc* = proc(response: HistoryResponse) {.gcsafe, closure.}
 
   IndexedWakuMessage* = object

waku/v2/utils/README.md

@@ -0,0 +1,4 @@
+# Utils
+
+Collection of misc utilities that don't fit anywhere else, and are used by
+multiple independent modules.

waku/v2/utils/pagination.nim

@@ -0,0 +1,11 @@
+## Contains types and utilities for pagination.
+##
+## Used by both message store and store protocol.
+
+import nimcrypto/hash
+
+type
+  Index* = object
+    ## This type contains the  description of an Index used in the pagination of WakuMessages
+    digest*: MDigest[256]
+    receivedTime*: float64

waku/v2/utils/requests.nim

@@ -0,0 +1,8 @@
+# Request utils.
+
+import bearssl, stew/byteutils
+
+proc generateRequestId*(rng: ref BrHmacDrbgContext): string =
+  var bytes: array[10, byte]
+  brHmacDrbgGenerate(rng[], bytes)
+  toHex(bytes)

waku/v2/waku_types.nim

@@ -1,90 +0,0 @@
-## Core Waku data types are defined here to avoid recursive dependencies.
-##
-## TODO Move types here into their appropriate place
-
-import
-  std/tables,
-  chronos, bearssl, stew/byteutils,
-  libp2p/[switch, peerinfo, multiaddress, crypto/crypto],
-  libp2p/protobuf/minprotobuf,
-  libp2p/protocols/protocol,
-  libp2p/switch,
-  libp2p/stream/connection,
-  libp2p/protocols/pubsub/[pubsub, gossipsub],
-  nimcrypto/sha2,
-  ./node/sqlite
-
-# Constants required for pagination -------------------------------------------
-const MaxPageSize* = 100 # Maximum number of waku messages in each page
-
-# Common data types -----------------------------------------------------------
-type
-
-  # Message -------------------------------------------------------------------
-
-  Index* = object
-    ## This type contains the  description of an Index used in the pagination of WakuMessages
-    digest*: MDigest[256]
-    receivedTime*: float64
-
-  ContentTopic* = uint32
-
-  Topic* = string
-  Message* = seq[byte]
-
-  WakuMessage* = object
-    payload*: seq[byte]
-    contentTopic*: ContentTopic
-    version*: uint32
-
-  MessageNotificationHandler* = proc(topic: string, msg: WakuMessage): Future[
-      void] {.gcsafe, closure.}
-
-  MessageNotificationSubscriptionIdentifier* = string
-
-  MessageNotificationSubscription* = object
-    topics*: seq[string] # @TODO TOPIC
-    handler*: MessageNotificationHandler
- 
-  MessageNotificationSubscriptions* = TableRef[MessageNotificationSubscriptionIdentifier, MessageNotificationSubscription]
-
-  # Relay protocol types -------------------------------------------------------
-
-  WakuRelay* = ref object of GossipSub
-
-  WakuInfo* = object
-    # NOTE One for simplicity, can extend later as needed
-    listenStr*: string
-    #multiaddrStrings*: seq[string]
-
-  WakuResult*[T] = Result[T, cstring]
-
-  MessageStoreResult*[T] = Result[T, string]
-
-  MessageStore* = ref object of RootObj
-    database*: SqliteDatabase
-
-# Encoding and decoding -------------------------------------------------------
-# TODO Move out to to waku_message module
-# Possibly same with util functions
-proc init*(T: type WakuMessage, buffer: seq[byte]): ProtoResult[T] =
-  var msg = WakuMessage()
-  let pb = initProtoBuffer(buffer)
-
-  discard ? pb.getField(1, msg.payload)
-  discard ? pb.getField(2, msg.contentTopic)
-  discard ? pb.getField(3, msg.version)
-
-  ok(msg)
-
-proc encode*(message: WakuMessage): ProtoBuffer =
-  result = initProtoBuffer()
-
-  result.write(1, message.payload)
-  result.write(2, message.contentTopic)
-  result.write(3, message.version)
-
-proc generateRequestId*(rng: ref BrHmacDrbgContext): string =
-  var bytes: array[10, byte]
-  brHmacDrbgGenerate(rng[], bytes)
-  toHex(bytes)

waku/waku.nim

@@ -5,7 +5,6 @@
 # - APACHEv2 ([LICENSE-APACHEv2](../LICENSE-APACHEv2) or https://www.apache.org/licenses/LICENSE-2.0)
 
 ## An implementation of the [Waku v1](https://specs.vac.dev/specs/waku/waku.html) and [Waku v2](https://specs.vac.dev/specs/waku/v2/waku-v2.html) in nim.
-import v2/node/wakunode2, v1/node/wakunode1, waku_types
+import v2/node/wakunode2, v1/node/wakunode1
 export wakunode2
 export wakunode1
-export waku_types