From 6b2be1de838d58c1c6ef433bcb46f0b5810d3add Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oskar=20Thor=C3=A9n?= Date: Tue, 24 Nov 2020 12:34:32 +0800 Subject: [PATCH] Refactor types store (#291) * Refactor: Move waku_store into its own folder * Refactor: Move waku store types to new home (WIP) * Refactor: Fix errors and recursive imports * Fix rebase errors * Refactor: More rebase import fixes --- examples/v2/chat2.nim | 3 +- tests/v2/test_jsonrpc_waku.nim | 3 +- tests/v2/test_message_store.nim | 1 + tests/v2/test_waku_pagination.nim | 2 +- tests/v2/test_waku_store.nim | 3 +- tests/v2/test_waku_swap.nim | 3 +- tests/v2/test_wakunode.nim | 3 +- waku/v2/node/jsonrpc/jsonrpc_utils.nim | 1 + waku/v2/node/jsonrpc/store_api.nim | 1 + waku/v2/node/rpc/wakurpc.nim | 1 + waku/v2/node/wakunode2.nim | 18 +++- waku/v2/protocol/waku_store/README.md | 3 + .../protocol/{ => waku_store}/waku_store.nim | 33 +++++-- .../protocol/waku_store/waku_store_types.nim | 49 ++++++++++ waku/v2/protocol/waku_swap/waku_swap.nim | 2 +- waku/v2/waku_types.nim | 91 +++---------------- 16 files changed, 124 insertions(+), 93 deletions(-) create mode 100644 waku/v2/protocol/waku_store/README.md rename waku/v2/protocol/{ => waku_store}/waku_store.nim (93%) create mode 100644 waku/v2/protocol/waku_store/waku_store_types.nim diff --git a/examples/v2/chat2.nim b/examples/v2/chat2.nim index 20753d395..55607a252 100644 --- a/examples/v2/chat2.nim +++ b/examples/v2/chat2.nim @@ -20,7 +20,8 @@ import libp2p/[switch, # manage transports, a single entry poi muxers/muxer, # define an interface for stream multiplexing, allowing peers to offer many protocols over a single connection muxers/mplex/mplex] # define some contants and message types for stream multiplexing import ../../waku/v2/node/[config, wakunode2, waku_payload], - ../../waku/v2/protocol/[waku_relay, waku_store, waku_filter], + ../../waku/v2/protocol/[waku_relay, waku_filter], + ../../waku/v2/protocol/waku_store/waku_store, ../../waku/common/utils/nat, ../../waku/v2/waku_types diff --git a/tests/v2/test_jsonrpc_waku.nim b/tests/v2/test_jsonrpc_waku.nim index 413eb9d5b..d47f3b57b 100644 --- a/tests/v2/test_jsonrpc_waku.nim +++ b/tests/v2/test_jsonrpc_waku.nim @@ -11,7 +11,8 @@ import ../../waku/v2/waku_types, ../../waku/v2/node/wakunode2, ../../waku/v2/node/jsonrpc/[jsonrpc_types,store_api], - ../../waku/v2/protocol/[waku_store, message_notifier], + ../../waku/v2/protocol/message_notifier, + ../../waku/v2/protocol/waku_store/waku_store, ../test_helpers template sourceDir*: string = currentSourcePath.rsplit(DirSep, 1)[0] diff --git a/tests/v2/test_message_store.nim b/tests/v2/test_message_store.nim index 34f3da07e..0c4dc50c4 100644 --- a/tests/v2/test_message_store.nim +++ b/tests/v2/test_message_store.nim @@ -3,6 +3,7 @@ import std/[unittest, options, tables, sets], chronos, chronicles, ../../waku/v2/node/message_store, + ../../waku/v2/protocol/waku_store/waku_store, ./utils, ../../waku/v2/waku_types, ../../waku/v2/node/sqlite diff --git a/tests/v2/test_waku_pagination.nim b/tests/v2/test_waku_pagination.nim index e99d0cbda..4523ac919 100644 --- a/tests/v2/test_waku_pagination.nim +++ b/tests/v2/test_waku_pagination.nim @@ -3,7 +3,7 @@ import std/[unittest,algorithm,options], nimcrypto/sha2, ../../waku/v2/waku_types, - ../../waku/v2/protocol/waku_store, + ../../waku/v2/protocol/waku_store/waku_store, ../test_helpers diff --git a/tests/v2/test_waku_store.nim b/tests/v2/test_waku_store.nim index 4e22630aa..a101fe4f3 100644 --- a/tests/v2/test_waku_store.nim +++ b/tests/v2/test_waku_store.nim @@ -8,7 +8,8 @@ import libp2p/stream/[bufferstream, connection], libp2p/crypto/crypto, libp2p/protocols/pubsub/rpc/message, - ../../waku/v2/protocol/[waku_store, message_notifier], + ../../waku/v2/protocol/message_notifier, + ../../waku/v2/protocol/waku_store/waku_store, ../../waku/v2/node/[message_store, sqlite], ../test_helpers, ./utils, ../../waku/v2/waku_types diff --git a/tests/v2/test_waku_swap.nim b/tests/v2/test_waku_swap.nim index 4f0ff6e25..9ab9ccec9 100644 --- a/tests/v2/test_waku_swap.nim +++ b/tests/v2/test_waku_swap.nim @@ -11,7 +11,8 @@ import libp2p/transports/transport, libp2p/transports/tcptransport, eth/keys, - ../../waku/v2/protocol/[waku_store, message_notifier], + ../../waku/v2/protocol/[message_notifier], + ../../waku/v2/protocol/waku_store/waku_store, ../../waku/v2/protocol/waku_swap/waku_swap, ../../waku/v2/node/message_store, ../../waku/v2/node/wakunode2, diff --git a/tests/v2/test_wakunode.nim b/tests/v2/test_wakunode.nim index c412467bf..f8ec8242e 100644 --- a/tests/v2/test_wakunode.nim +++ b/tests/v2/test_wakunode.nim @@ -7,7 +7,8 @@ import libp2p/crypto/secp, libp2p/switch, eth/keys, - ../../waku/v2/protocol/[waku_relay, waku_store, waku_filter, message_notifier], + ../../waku/v2/protocol/[waku_relay, waku_filter, message_notifier], + ../../waku/v2/protocol/waku_store/waku_store, ../../waku/v2/node/wakunode2, ../test_helpers, ../../waku/v2/waku_types diff --git a/waku/v2/node/jsonrpc/jsonrpc_utils.nim b/waku/v2/node/jsonrpc/jsonrpc_utils.nim index e41288a93..d950c2442 100644 --- a/waku/v2/node/jsonrpc/jsonrpc_utils.nim +++ b/waku/v2/node/jsonrpc/jsonrpc_utils.nim @@ -1,6 +1,7 @@ import std/options, ../../waku_types, + ../../protocol/waku_store/waku_store_types, ../wakunode2, ./jsonrpc_types diff --git a/waku/v2/node/jsonrpc/store_api.nim b/waku/v2/node/jsonrpc/store_api.nim index edf3bccd1..3e9e3501a 100644 --- a/waku/v2/node/jsonrpc/store_api.nim +++ b/waku/v2/node/jsonrpc/store_api.nim @@ -2,6 +2,7 @@ import std/options, json_rpc/rpcserver, ../../waku_types, + ../../protocol/waku_store/waku_store_types, ../wakunode2, ./jsonrpc_types, ./jsonrpc_utils diff --git a/waku/v2/node/rpc/wakurpc.nim b/waku/v2/node/rpc/wakurpc.nim index ef34e7e67..540b79e4d 100644 --- a/waku/v2/node/rpc/wakurpc.nim +++ b/waku/v2/node/rpc/wakurpc.nim @@ -5,6 +5,7 @@ import eth/[common, rlp, keys, p2p], ../../protocol/waku_relay, ../../waku_types, + ../../protocol/waku_store/waku_store, ../wakunode2 proc setupWakuRPC*(node: WakuNode, rpcsrv: RpcServer) = diff --git a/waku/v2/node/wakunode2.nim b/waku/v2/node/wakunode2.nim index 90faba3d5..efb649e7d 100644 --- a/waku/v2/node/wakunode2.nim +++ b/waku/v2/node/wakunode2.nim @@ -10,7 +10,8 @@ import libp2p/protocols/pubsub/pubsub, libp2p/peerinfo, libp2p/standard_setup, - ../protocol/[waku_relay, waku_store, waku_filter, message_notifier], + ../protocol/[waku_relay, waku_filter, message_notifier], + ../protocol/waku_store/waku_store, ../protocol/waku_swap/waku_swap, ../waku_types, ./message_store, @@ -34,6 +35,21 @@ type Topic* = waku_types.Topic Message* = seq[byte] + # NOTE based on Eth2Node in NBC eth2_network.nim + WakuNode* = ref object of RootObj + switch*: Switch + wakuRelay*: WakuRelay + wakuStore*: WakuStore + wakuFilter*: WakuFilter + wakuSwap*: WakuSwap + peerInfo*: PeerInfo + libp2pTransportLoops*: seq[Future[void]] + # TODO Revist messages field indexing as well as if this should be Message or WakuMessage + messages*: seq[(Topic, WakuMessage)] + filters*: Filters + subscriptions*: MessageNotificationSubscriptions + rng*: ref BrHmacDrbgContext + # NOTE Any difference here in Waku vs Eth2? # E.g. Devp2p/Libp2p support, etc. #func asLibp2pKey*(key: keys.PublicKey): PublicKey = diff --git a/waku/v2/protocol/waku_store/README.md b/waku/v2/protocol/waku_store/README.md new file mode 100644 index 000000000..f5adaf484 --- /dev/null +++ b/waku/v2/protocol/waku_store/README.md @@ -0,0 +1,3 @@ +# Waku Store protocol + +The store protocol implements historical message support. See https://github.com/vacp2p/specs/blob/master/specs/waku/v2/waku-store.md for more information. diff --git a/waku/v2/protocol/waku_store.nim b/waku/v2/protocol/waku_store/waku_store.nim similarity index 93% rename from waku/v2/protocol/waku_store.nim rename to waku/v2/protocol/waku_store/waku_store.nim index dfb593e54..b6b5416ec 100644 --- a/waku/v2/protocol/waku_store.nim +++ b/waku/v2/protocol/waku_store/waku_store.nim @@ -1,16 +1,23 @@ +## Waku Store protocol for historical messaging support. +## See spec for more details: +## https://github.com/vacp2p/specs/blob/master/specs/waku/v2/waku-store.md + import - std/[tables, sequtils, algorithm, options], + std/[tables, times, sequtils, algorithm, options], bearssl, - chronos, chronicles, metrics, stew/[results,byteutils], + chronos, chronicles, metrics, stew/[results, byteutils, endians2], libp2p/switch, libp2p/crypto/crypto, libp2p/protocols/protocol, libp2p/protobuf/minprotobuf, libp2p/stream/connection, - ./message_notifier, - ../node/message_store, - waku_swap/waku_swap, - ../waku_types + ../message_notifier, + ../../node/message_store, + ../waku_swap/waku_swap, + ../../waku_types, + ./waku_store_types + +export waku_store_types logScope: topics = "wakustore" @@ -18,6 +25,20 @@ logScope: const WakuStoreCodec* = "/vac/waku/store/2.0.0-beta1" +# TODO Move serialization function to separate file, too noisy +# TODO Move pagination to separate file, self-contained logic + +proc computeIndex*(msg: WakuMessage): Index = + ## Takes a WakuMessage and returns its Index + var ctx: sha256 + ctx.init() + ctx.update(msg.contentTopic.toBytes()) # converts the contentTopic to bytes + ctx.update(msg.payload) + let digest = ctx.finish() # computes the hash + ctx.clear() + result.digest = digest + result.receivedTime = epochTime() # gets the unix timestamp + proc encode*(index: Index): ProtoBuffer = ## encodes an Index object into a ProtoBuffer ## returns the resultant ProtoBuffer diff --git a/waku/v2/protocol/waku_store/waku_store_types.nim b/waku/v2/protocol/waku_store/waku_store_types.nim new file mode 100644 index 000000000..691f486fa --- /dev/null +++ b/waku/v2/protocol/waku_store/waku_store_types.nim @@ -0,0 +1,49 @@ +## Types for waku_store protocol. + +import + bearssl, stew/[byteutils, endians2], + libp2p/[switch, peerinfo], + libp2p/protocols/protocol, + ../../waku_types + +type + QueryHandlerFunc* = proc(response: HistoryResponse) {.gcsafe, closure.} + + IndexedWakuMessage* = object + ## This type is used to encapsulate a WakuMessage and its Index + msg*: WakuMessage + index*: Index + + PagingDirection* {.pure.} = enum + ## PagingDirection determines the direction of pagination + BACKWARD = uint32(0) + FORWARD = uint32(1) + + PagingInfo* = object + ## This type holds the information needed for the pagination + pageSize*: uint64 + cursor*: Index + direction*: PagingDirection + + HistoryQuery* = object + topics*: seq[ContentTopic] + pagingInfo*: PagingInfo # used for pagination + + HistoryResponse* = object + messages*: seq[WakuMessage] + pagingInfo*: PagingInfo # used for pagination + + HistoryRPC* = object + requestId*: string + query*: HistoryQuery + response*: HistoryResponse + + HistoryPeer* = object + peerInfo*: PeerInfo + + WakuStore* = ref object of LPProtocol + switch*: Switch + rng*: ref BrHmacDrbgContext + peers*: seq[HistoryPeer] + messages*: seq[IndexedWakuMessage] + store*: MessageStore diff --git a/waku/v2/protocol/waku_swap/waku_swap.nim b/waku/v2/protocol/waku_swap/waku_swap.nim index d63cbeb68..f4eb67320 100644 --- a/waku/v2/protocol/waku_swap/waku_swap.nim +++ b/waku/v2/protocol/waku_swap/waku_swap.nim @@ -31,7 +31,7 @@ import libp2p/protobuf/minprotobuf, libp2p/stream/connection, ../message_notifier, - waku_swap_types + ./waku_swap_types export waku_swap_types diff --git a/waku/v2/waku_types.nim b/waku/v2/waku_types.nim index d2d908799..74f3aa2b1 100644 --- a/waku/v2/waku_types.nim +++ b/waku/v2/waku_types.nim @@ -1,6 +1,6 @@ ## Core Waku data types are defined here to avoid recursive dependencies. ## -## TODO Move more common data types here +## TODO Move types here into their appropriate place import std/[tables, times], @@ -20,6 +20,12 @@ const MaxPageSize* = 100 # Maximum number of waku messages in each page # Common data types ----------------------------------------------------------- type + + Index* = object + ## This type contains the description of an Index used in the pagination of WakuMessages + digest*: MDigest[256] + receivedTime*: float64 + ContentTopic* = uint32 Topic* = string @@ -39,57 +45,6 @@ type topics*: seq[string] # @TODO TOPIC handler*: MessageNotificationHandler - QueryHandlerFunc* = proc(response: HistoryResponse) {.gcsafe, closure.} - - Index* = object - ## This type contains the description of an Index used in the pagination of WakuMessages - digest*: MDigest[256] - receivedTime*: float64 - - IndexedWakuMessage* = object - ## This type is used to encapsulate a WakuMessage and its Index - msg*: WakuMessage - index*: Index - - PagingDirection* {.pure.} = enum - ## PagingDirection determines the direction of pagination - BACKWARD = uint32(0) - FORWARD = uint32(1) - - PagingInfo* = object - ## This type holds the information needed for the pagination - pageSize*: uint64 - cursor*: Index - direction*: PagingDirection - - HistoryQuery* = object - topics*: seq[ContentTopic] - pagingInfo*: PagingInfo # used for pagination - - HistoryResponse* = object - messages*: seq[WakuMessage] - pagingInfo*: PagingInfo # used for pagination - - HistoryRPC* = object - requestId*: string - query*: HistoryQuery - response*: HistoryResponse - - HistoryPeer* = object - peerInfo*: PeerInfo - - MessageStoreResult*[T] = Result[T, string] - - MessageStore* = ref object of RootObj - database*: SqliteDatabase - - WakuStore* = ref object of LPProtocol - switch*: Switch - rng*: ref BrHmacDrbgContext - peers*: seq[HistoryPeer] - messages*: seq[IndexedWakuMessage] - store*: MessageStore - FilterRequest* = object contentFilters*: seq[ContentFilter] topic*: string @@ -132,21 +87,6 @@ type # @TODO MAYBE MORE INFO? Filters* = Table[string, Filter] - # NOTE based on Eth2Node in NBC eth2_network.nim - WakuNode* = ref object of RootObj - switch*: Switch - wakuRelay*: WakuRelay - wakuStore*: WakuStore - wakuFilter*: WakuFilter - wakuSwap*: WakuSwap - peerInfo*: PeerInfo - libp2pTransportLoops*: seq[Future[void]] - # TODO Revist messages field indexing as well as if this should be Message or WakuMessage - messages*: seq[(Topic, WakuMessage)] - filters*: Filters - subscriptions*: MessageNotificationSubscriptions - rng*: ref BrHmacDrbgContext - WakuRelay* = ref object of GossipSub gossipEnabled*: bool @@ -157,6 +97,11 @@ type WakuResult*[T] = Result[T, cstring] + MessageStoreResult*[T] = Result[T, string] + + MessageStore* = ref object of RootObj + database*: SqliteDatabase + # Encoding and decoding ------------------------------------------------------- # TODO Move out to to waku_message module # Possibly same with util functions @@ -199,15 +144,3 @@ proc generateRequestId*(rng: ref BrHmacDrbgContext): string = var bytes: array[10, byte] brHmacDrbgGenerate(rng[], bytes) toHex(bytes) - -proc computeIndex*(msg: WakuMessage): Index = - ## Takes a WakuMessage and returns its Index - var ctx: sha256 - ctx.init() - ctx.update(msg.contentTopic.toBytes()) # converts the contentTopic to bytes - ctx.update(msg.payload) - let digest = ctx.finish() # computes the hash - ctx.clear() - result.digest = digest - result.receivedTime = epochTime() # gets the unix timestamp -