Refactor types store (#291)

* Refactor: Move waku_store into its own folder

* Refactor: Move waku store types to new home (WIP)

* Refactor: Fix errors and recursive imports

* Fix rebase errors

* Refactor: More rebase import fixes

examples/v2/chat2.nim

@@ -20,7 +20,8 @@ import libp2p/[switch,                   # manage transports, a single entry poi
                muxers/muxer,             # define an interface for stream multiplexing, allowing peers to offer many protocols over a single connection
                muxers/mplex/mplex]       # define some contants and message types for stream multiplexing
 import   ../../waku/v2/node/[config, wakunode2, waku_payload],
-         ../../waku/v2/protocol/[waku_relay, waku_store, waku_filter],
+         ../../waku/v2/protocol/[waku_relay, waku_filter],
+         ../../waku/v2/protocol/waku_store/waku_store,
          ../../waku/common/utils/nat,
          ../../waku/v2/waku_types
 

tests/v2/test_jsonrpc_waku.nim

@@ -11,7 +11,8 @@ import
   ../../waku/v2/waku_types,
   ../../waku/v2/node/wakunode2,
   ../../waku/v2/node/jsonrpc/[jsonrpc_types,store_api],
-  ../../waku/v2/protocol/[waku_store, message_notifier],
+  ../../waku/v2/protocol/message_notifier,
+  ../../waku/v2/protocol/waku_store/waku_store,
   ../test_helpers
 
 template sourceDir*: string = currentSourcePath.rsplit(DirSep, 1)[0]

tests/v2/test_message_store.nim

@@ -3,6 +3,7 @@ import
   std/[unittest, options, tables, sets],
   chronos, chronicles,
   ../../waku/v2/node/message_store,
+  ../../waku/v2/protocol/waku_store/waku_store,
   ./utils,
   ../../waku/v2/waku_types,
   ../../waku/v2/node/sqlite

tests/v2/test_waku_pagination.nim

@@ -3,7 +3,7 @@ import
   std/[unittest,algorithm,options],
   nimcrypto/sha2,
   ../../waku/v2/waku_types,
-  ../../waku/v2/protocol/waku_store,
+  ../../waku/v2/protocol/waku_store/waku_store,
   ../test_helpers
 
 

tests/v2/test_waku_store.nim

@@ -8,7 +8,8 @@ import
   libp2p/stream/[bufferstream, connection],
   libp2p/crypto/crypto,
   libp2p/protocols/pubsub/rpc/message,
-  ../../waku/v2/protocol/[waku_store, message_notifier],
+  ../../waku/v2/protocol/message_notifier,
+  ../../waku/v2/protocol/waku_store/waku_store,
   ../../waku/v2/node/[message_store, sqlite],
   ../test_helpers, ./utils,
   ../../waku/v2/waku_types

tests/v2/test_waku_swap.nim

@@ -11,7 +11,8 @@ import
   libp2p/transports/transport,
   libp2p/transports/tcptransport,
   eth/keys,
-  ../../waku/v2/protocol/[waku_store, message_notifier],
+  ../../waku/v2/protocol/[message_notifier],
+  ../../waku/v2/protocol/waku_store/waku_store,
   ../../waku/v2/protocol/waku_swap/waku_swap,
   ../../waku/v2/node/message_store,
   ../../waku/v2/node/wakunode2,

tests/v2/test_wakunode.nim

@@ -7,7 +7,8 @@ import
   libp2p/crypto/secp,
   libp2p/switch,
   eth/keys,
-  ../../waku/v2/protocol/[waku_relay, waku_store, waku_filter, message_notifier],
+  ../../waku/v2/protocol/[waku_relay, waku_filter, message_notifier],
+  ../../waku/v2/protocol/waku_store/waku_store,
   ../../waku/v2/node/wakunode2,
   ../test_helpers,
   ../../waku/v2/waku_types

waku/v2/node/jsonrpc/jsonrpc_utils.nim

@@ -1,6 +1,7 @@
 import
   std/options,
   ../../waku_types,
+  ../../protocol/waku_store/waku_store_types,
   ../wakunode2,
   ./jsonrpc_types
 

waku/v2/node/jsonrpc/store_api.nim

@@ -2,6 +2,7 @@ import
   std/options,
   json_rpc/rpcserver,
   ../../waku_types,
+  ../../protocol/waku_store/waku_store_types,
   ../wakunode2,
   ./jsonrpc_types, ./jsonrpc_utils
 

waku/v2/node/rpc/wakurpc.nim

@@ -5,6 +5,7 @@ import
   eth/[common, rlp, keys, p2p],
   ../../protocol/waku_relay,
   ../../waku_types,
+  ../../protocol/waku_store/waku_store,
   ../wakunode2
 
 proc setupWakuRPC*(node: WakuNode, rpcsrv: RpcServer) =

waku/v2/node/wakunode2.nim

@@ -10,7 +10,8 @@ import
   libp2p/protocols/pubsub/pubsub,
   libp2p/peerinfo,
   libp2p/standard_setup,
-  ../protocol/[waku_relay, waku_store, waku_filter, message_notifier],
+  ../protocol/[waku_relay, waku_filter, message_notifier],
+  ../protocol/waku_store/waku_store,
   ../protocol/waku_swap/waku_swap,
   ../waku_types,
   ./message_store,
@@ -34,6 +35,21 @@ type
   Topic* = waku_types.Topic
   Message* = seq[byte]
 
+  # NOTE based on Eth2Node in NBC eth2_network.nim
+  WakuNode* = ref object of RootObj
+    switch*: Switch
+    wakuRelay*: WakuRelay
+    wakuStore*: WakuStore
+    wakuFilter*: WakuFilter
+    wakuSwap*: WakuSwap
+    peerInfo*: PeerInfo
+    libp2pTransportLoops*: seq[Future[void]]
+  # TODO Revist messages field indexing as well as if this should be Message or WakuMessage
+    messages*: seq[(Topic, WakuMessage)]
+    filters*: Filters
+    subscriptions*: MessageNotificationSubscriptions
+    rng*: ref BrHmacDrbgContext
+
 # NOTE Any difference here in Waku vs Eth2?
 # E.g. Devp2p/Libp2p support, etc.
 #func asLibp2pKey*(key: keys.PublicKey): PublicKey =

waku/v2/protocol/waku_store/README.md

@@ -0,0 +1,3 @@
+# Waku Store protocol
+
+The store protocol implements historical message support. See https://github.com/vacp2p/specs/blob/master/specs/waku/v2/waku-store.md for more information.

waku/v2/protocol/waku_store/waku_store.nim

@@ -1,16 +1,23 @@
+## Waku Store protocol for historical messaging support.
+## See spec for more details:
+## https://github.com/vacp2p/specs/blob/master/specs/waku/v2/waku-store.md
+
 import
-  std/[tables, sequtils, algorithm, options],
+  std/[tables, times, sequtils, algorithm, options],
   bearssl,
-  chronos, chronicles, metrics, stew/[results,byteutils],
+  chronos, chronicles, metrics, stew/[results, byteutils, endians2],
   libp2p/switch,
   libp2p/crypto/crypto,
   libp2p/protocols/protocol,
   libp2p/protobuf/minprotobuf,
   libp2p/stream/connection,
-  ./message_notifier,
-  ../node/message_store,
-  waku_swap/waku_swap,
-  ../waku_types
+  ../message_notifier,
+  ../../node/message_store,
+  ../waku_swap/waku_swap,
+  ../../waku_types,
+  ./waku_store_types
+
+export waku_store_types
 
 logScope:
   topics = "wakustore"
@@ -18,6 +25,20 @@ logScope:
 const
   WakuStoreCodec* = "/vac/waku/store/2.0.0-beta1"
 
+# TODO Move serialization function to separate file, too noisy
+# TODO Move pagination to separate file, self-contained logic
+
+proc computeIndex*(msg: WakuMessage): Index =
+  ## Takes a WakuMessage and returns its Index
+  var ctx: sha256
+  ctx.init()
+  ctx.update(msg.contentTopic.toBytes()) # converts the contentTopic to bytes
+  ctx.update(msg.payload)
+  let digest = ctx.finish() # computes the hash
+  ctx.clear()
+  result.digest = digest
+  result.receivedTime = epochTime() # gets the unix timestamp
+
 proc encode*(index: Index): ProtoBuffer =
   ## encodes an Index object into a ProtoBuffer
   ## returns the resultant ProtoBuffer

waku/v2/protocol/waku_store/waku_store_types.nim

@@ -0,0 +1,49 @@
+## Types for waku_store protocol.
+
+import
+  bearssl, stew/[byteutils, endians2],
+  libp2p/[switch, peerinfo],
+  libp2p/protocols/protocol,
+  ../../waku_types
+
+type
+  QueryHandlerFunc* = proc(response: HistoryResponse) {.gcsafe, closure.}
+
+  IndexedWakuMessage* = object
+    ## This type is used to encapsulate a WakuMessage and its Index
+    msg*: WakuMessage
+    index*: Index
+
+  PagingDirection* {.pure.} = enum
+    ## PagingDirection determines the direction of pagination
+    BACKWARD = uint32(0)
+    FORWARD = uint32(1)
+
+  PagingInfo* = object
+    ## This type holds the information needed for the pagination
+    pageSize*: uint64
+    cursor*: Index
+    direction*: PagingDirection
+
+  HistoryQuery* = object
+    topics*: seq[ContentTopic]
+    pagingInfo*: PagingInfo # used for pagination
+
+  HistoryResponse* = object
+    messages*: seq[WakuMessage]
+    pagingInfo*: PagingInfo # used for pagination
+
+  HistoryRPC* = object
+    requestId*: string
+    query*: HistoryQuery
+    response*: HistoryResponse
+
+  HistoryPeer* = object
+    peerInfo*: PeerInfo
+
+  WakuStore* = ref object of LPProtocol
+    switch*: Switch
+    rng*: ref BrHmacDrbgContext
+    peers*: seq[HistoryPeer]
+    messages*: seq[IndexedWakuMessage]
+    store*: MessageStore

waku/v2/protocol/waku_swap/waku_swap.nim

@@ -31,7 +31,7 @@ import
   libp2p/protobuf/minprotobuf,
   libp2p/stream/connection,
   ../message_notifier,
-   waku_swap_types
+  ./waku_swap_types
 
 export waku_swap_types
 

waku/v2/waku_types.nim

@@ -1,6 +1,6 @@
 ## Core Waku data types are defined here to avoid recursive dependencies.
 ##
-## TODO Move more common data types here
+## TODO Move types here into their appropriate place
 
 import
   std/[tables, times],
@@ -20,6 +20,12 @@ const MaxPageSize* = 100 # Maximum number of waku messages in each page
 
 # Common data types -----------------------------------------------------------
 type
+
+  Index* = object
+    ## This type contains the  description of an Index used in the pagination of WakuMessages
+    digest*: MDigest[256]
+    receivedTime*: float64
+
   ContentTopic* = uint32
 
   Topic* = string
@@ -39,57 +45,6 @@ type
     topics*: seq[string] # @TODO TOPIC
     handler*: MessageNotificationHandler
 
-  QueryHandlerFunc* = proc(response: HistoryResponse) {.gcsafe, closure.}
-
-  Index* = object
-    ## This type contains the  description of an Index used in the pagination of WakuMessages
-    digest*: MDigest[256]
-    receivedTime*: float64
-
-  IndexedWakuMessage* = object
-    ## This type is used to encapsulate a WakuMessage and its Index
-    msg*: WakuMessage
-    index*: Index
-
-  PagingDirection* {.pure.} = enum
-    ## PagingDirection determines the direction of pagination
-    BACKWARD = uint32(0)
-    FORWARD = uint32(1)
-
-  PagingInfo* = object
-    ## This type holds the information needed for the pagination
-    pageSize*: uint64
-    cursor*: Index
-    direction*: PagingDirection
-
-  HistoryQuery* = object
-    topics*: seq[ContentTopic]
-    pagingInfo*: PagingInfo # used for pagination
-
-  HistoryResponse* = object
-    messages*: seq[WakuMessage]
-    pagingInfo*: PagingInfo # used for pagination
-
-  HistoryRPC* = object
-    requestId*: string
-    query*: HistoryQuery
-    response*: HistoryResponse
-
-  HistoryPeer* = object
-    peerInfo*: PeerInfo
-
-  MessageStoreResult*[T] = Result[T, string]
-
-  MessageStore* = ref object of RootObj
-    database*: SqliteDatabase
-
-  WakuStore* = ref object of LPProtocol
-    switch*: Switch
-    rng*: ref BrHmacDrbgContext
-    peers*: seq[HistoryPeer]
-    messages*: seq[IndexedWakuMessage]
-    store*: MessageStore
-
   FilterRequest* = object
     contentFilters*: seq[ContentFilter]
     topic*: string
@@ -132,21 +87,6 @@ type
   # @TODO MAYBE MORE INFO?
   Filters* = Table[string, Filter]
 
-  # NOTE based on Eth2Node in NBC eth2_network.nim
-  WakuNode* = ref object of RootObj
-    switch*: Switch
-    wakuRelay*: WakuRelay
-    wakuStore*: WakuStore
-    wakuFilter*: WakuFilter
-    wakuSwap*: WakuSwap
-    peerInfo*: PeerInfo
-    libp2pTransportLoops*: seq[Future[void]]
-  # TODO Revist messages field indexing as well as if this should be Message or WakuMessage
-    messages*: seq[(Topic, WakuMessage)]
-    filters*: Filters
-    subscriptions*: MessageNotificationSubscriptions
-    rng*: ref BrHmacDrbgContext
-
   WakuRelay* = ref object of GossipSub
     gossipEnabled*: bool
 
@@ -157,6 +97,11 @@ type
 
   WakuResult*[T] = Result[T, cstring]
 
+  MessageStoreResult*[T] = Result[T, string]
+
+  MessageStore* = ref object of RootObj
+    database*: SqliteDatabase
+
 # Encoding and decoding -------------------------------------------------------
 # TODO Move out to to waku_message module
 # Possibly same with util functions
@@ -199,15 +144,3 @@ proc generateRequestId*(rng: ref BrHmacDrbgContext): string =
   var bytes: array[10, byte]
   brHmacDrbgGenerate(rng[], bytes)
   toHex(bytes)
-
-proc computeIndex*(msg: WakuMessage): Index =
-  ## Takes a WakuMessage and returns its Index 
-  var ctx: sha256
-  ctx.init()
-  ctx.update(msg.contentTopic.toBytes()) # converts the contentTopic to bytes
-  ctx.update(msg.payload)
-  let digest = ctx.finish() # computes the hash
-  ctx.clear()
-  result.digest = digest
-  result.receivedTime = epochTime() # gets the unix timestamp
-