mirror of https://github.com/waku-org/nwaku.git
Refactor types store (#291)
* Refactor: Move waku_store into its own folder * Refactor: Move waku store types to new home (WIP) * Refactor: Fix errors and recursive imports * Fix rebase errors * Refactor: More rebase import fixes
This commit is contained in:
parent
135eaae9fb
commit
44e9d4d86b
|
@ -20,7 +20,8 @@ import libp2p/[switch, # manage transports, a single entry poi
|
|||
muxers/muxer, # define an interface for stream multiplexing, allowing peers to offer many protocols over a single connection
|
||||
muxers/mplex/mplex] # define some contants and message types for stream multiplexing
|
||||
import ../../waku/v2/node/[config, wakunode2, waku_payload],
|
||||
../../waku/v2/protocol/[waku_relay, waku_store, waku_filter],
|
||||
../../waku/v2/protocol/[waku_relay, waku_filter],
|
||||
../../waku/v2/protocol/waku_store/waku_store,
|
||||
../../waku/common/utils/nat,
|
||||
../../waku/v2/waku_types
|
||||
|
||||
|
|
|
@ -11,7 +11,8 @@ import
|
|||
../../waku/v2/waku_types,
|
||||
../../waku/v2/node/wakunode2,
|
||||
../../waku/v2/node/jsonrpc/[jsonrpc_types,store_api],
|
||||
../../waku/v2/protocol/[waku_store, message_notifier],
|
||||
../../waku/v2/protocol/message_notifier,
|
||||
../../waku/v2/protocol/waku_store/waku_store,
|
||||
../test_helpers
|
||||
|
||||
template sourceDir*: string = currentSourcePath.rsplit(DirSep, 1)[0]
|
||||
|
|
|
@ -3,6 +3,7 @@ import
|
|||
std/[unittest, options, tables, sets],
|
||||
chronos, chronicles,
|
||||
../../waku/v2/node/message_store,
|
||||
../../waku/v2/protocol/waku_store/waku_store,
|
||||
./utils,
|
||||
../../waku/v2/waku_types,
|
||||
../../waku/v2/node/sqlite
|
||||
|
|
|
@ -3,7 +3,7 @@ import
|
|||
std/[unittest,algorithm,options],
|
||||
nimcrypto/sha2,
|
||||
../../waku/v2/waku_types,
|
||||
../../waku/v2/protocol/waku_store,
|
||||
../../waku/v2/protocol/waku_store/waku_store,
|
||||
../test_helpers
|
||||
|
||||
|
||||
|
|
|
@ -8,7 +8,8 @@ import
|
|||
libp2p/stream/[bufferstream, connection],
|
||||
libp2p/crypto/crypto,
|
||||
libp2p/protocols/pubsub/rpc/message,
|
||||
../../waku/v2/protocol/[waku_store, message_notifier],
|
||||
../../waku/v2/protocol/message_notifier,
|
||||
../../waku/v2/protocol/waku_store/waku_store,
|
||||
../../waku/v2/node/[message_store, sqlite],
|
||||
../test_helpers, ./utils,
|
||||
../../waku/v2/waku_types
|
||||
|
|
|
@ -11,7 +11,8 @@ import
|
|||
libp2p/transports/transport,
|
||||
libp2p/transports/tcptransport,
|
||||
eth/keys,
|
||||
../../waku/v2/protocol/[waku_store, message_notifier],
|
||||
../../waku/v2/protocol/[message_notifier],
|
||||
../../waku/v2/protocol/waku_store/waku_store,
|
||||
../../waku/v2/protocol/waku_swap/waku_swap,
|
||||
../../waku/v2/node/message_store,
|
||||
../../waku/v2/node/wakunode2,
|
||||
|
|
|
@ -7,7 +7,8 @@ import
|
|||
libp2p/crypto/secp,
|
||||
libp2p/switch,
|
||||
eth/keys,
|
||||
../../waku/v2/protocol/[waku_relay, waku_store, waku_filter, message_notifier],
|
||||
../../waku/v2/protocol/[waku_relay, waku_filter, message_notifier],
|
||||
../../waku/v2/protocol/waku_store/waku_store,
|
||||
../../waku/v2/node/wakunode2,
|
||||
../test_helpers,
|
||||
../../waku/v2/waku_types
|
||||
|
|
|
@ -1,6 +1,7 @@
|
|||
import
|
||||
std/options,
|
||||
../../waku_types,
|
||||
../../protocol/waku_store/waku_store_types,
|
||||
../wakunode2,
|
||||
./jsonrpc_types
|
||||
|
||||
|
|
|
@ -2,6 +2,7 @@ import
|
|||
std/options,
|
||||
json_rpc/rpcserver,
|
||||
../../waku_types,
|
||||
../../protocol/waku_store/waku_store_types,
|
||||
../wakunode2,
|
||||
./jsonrpc_types, ./jsonrpc_utils
|
||||
|
||||
|
|
|
@ -5,6 +5,7 @@ import
|
|||
eth/[common, rlp, keys, p2p],
|
||||
../../protocol/waku_relay,
|
||||
../../waku_types,
|
||||
../../protocol/waku_store/waku_store,
|
||||
../wakunode2
|
||||
|
||||
proc setupWakuRPC*(node: WakuNode, rpcsrv: RpcServer) =
|
||||
|
|
|
@ -10,7 +10,8 @@ import
|
|||
libp2p/protocols/pubsub/pubsub,
|
||||
libp2p/peerinfo,
|
||||
libp2p/standard_setup,
|
||||
../protocol/[waku_relay, waku_store, waku_filter, message_notifier],
|
||||
../protocol/[waku_relay, waku_filter, message_notifier],
|
||||
../protocol/waku_store/waku_store,
|
||||
../protocol/waku_swap/waku_swap,
|
||||
../waku_types,
|
||||
./message_store,
|
||||
|
@ -34,6 +35,21 @@ type
|
|||
Topic* = waku_types.Topic
|
||||
Message* = seq[byte]
|
||||
|
||||
# NOTE based on Eth2Node in NBC eth2_network.nim
|
||||
WakuNode* = ref object of RootObj
|
||||
switch*: Switch
|
||||
wakuRelay*: WakuRelay
|
||||
wakuStore*: WakuStore
|
||||
wakuFilter*: WakuFilter
|
||||
wakuSwap*: WakuSwap
|
||||
peerInfo*: PeerInfo
|
||||
libp2pTransportLoops*: seq[Future[void]]
|
||||
# TODO Revist messages field indexing as well as if this should be Message or WakuMessage
|
||||
messages*: seq[(Topic, WakuMessage)]
|
||||
filters*: Filters
|
||||
subscriptions*: MessageNotificationSubscriptions
|
||||
rng*: ref BrHmacDrbgContext
|
||||
|
||||
# NOTE Any difference here in Waku vs Eth2?
|
||||
# E.g. Devp2p/Libp2p support, etc.
|
||||
#func asLibp2pKey*(key: keys.PublicKey): PublicKey =
|
||||
|
|
|
@ -0,0 +1,3 @@
|
|||
# Waku Store protocol
|
||||
|
||||
The store protocol implements historical message support. See https://github.com/vacp2p/specs/blob/master/specs/waku/v2/waku-store.md for more information.
|
|
@ -1,16 +1,23 @@
|
|||
## Waku Store protocol for historical messaging support.
|
||||
## See spec for more details:
|
||||
## https://github.com/vacp2p/specs/blob/master/specs/waku/v2/waku-store.md
|
||||
|
||||
import
|
||||
std/[tables, sequtils, algorithm, options],
|
||||
std/[tables, times, sequtils, algorithm, options],
|
||||
bearssl,
|
||||
chronos, chronicles, metrics, stew/[results,byteutils],
|
||||
chronos, chronicles, metrics, stew/[results, byteutils, endians2],
|
||||
libp2p/switch,
|
||||
libp2p/crypto/crypto,
|
||||
libp2p/protocols/protocol,
|
||||
libp2p/protobuf/minprotobuf,
|
||||
libp2p/stream/connection,
|
||||
./message_notifier,
|
||||
../node/message_store,
|
||||
waku_swap/waku_swap,
|
||||
../waku_types
|
||||
../message_notifier,
|
||||
../../node/message_store,
|
||||
../waku_swap/waku_swap,
|
||||
../../waku_types,
|
||||
./waku_store_types
|
||||
|
||||
export waku_store_types
|
||||
|
||||
logScope:
|
||||
topics = "wakustore"
|
||||
|
@ -18,6 +25,20 @@ logScope:
|
|||
const
|
||||
WakuStoreCodec* = "/vac/waku/store/2.0.0-beta1"
|
||||
|
||||
# TODO Move serialization function to separate file, too noisy
|
||||
# TODO Move pagination to separate file, self-contained logic
|
||||
|
||||
proc computeIndex*(msg: WakuMessage): Index =
|
||||
## Takes a WakuMessage and returns its Index
|
||||
var ctx: sha256
|
||||
ctx.init()
|
||||
ctx.update(msg.contentTopic.toBytes()) # converts the contentTopic to bytes
|
||||
ctx.update(msg.payload)
|
||||
let digest = ctx.finish() # computes the hash
|
||||
ctx.clear()
|
||||
result.digest = digest
|
||||
result.receivedTime = epochTime() # gets the unix timestamp
|
||||
|
||||
proc encode*(index: Index): ProtoBuffer =
|
||||
## encodes an Index object into a ProtoBuffer
|
||||
## returns the resultant ProtoBuffer
|
|
@ -0,0 +1,49 @@
|
|||
## Types for waku_store protocol.
|
||||
|
||||
import
|
||||
bearssl, stew/[byteutils, endians2],
|
||||
libp2p/[switch, peerinfo],
|
||||
libp2p/protocols/protocol,
|
||||
../../waku_types
|
||||
|
||||
type
|
||||
QueryHandlerFunc* = proc(response: HistoryResponse) {.gcsafe, closure.}
|
||||
|
||||
IndexedWakuMessage* = object
|
||||
## This type is used to encapsulate a WakuMessage and its Index
|
||||
msg*: WakuMessage
|
||||
index*: Index
|
||||
|
||||
PagingDirection* {.pure.} = enum
|
||||
## PagingDirection determines the direction of pagination
|
||||
BACKWARD = uint32(0)
|
||||
FORWARD = uint32(1)
|
||||
|
||||
PagingInfo* = object
|
||||
## This type holds the information needed for the pagination
|
||||
pageSize*: uint64
|
||||
cursor*: Index
|
||||
direction*: PagingDirection
|
||||
|
||||
HistoryQuery* = object
|
||||
topics*: seq[ContentTopic]
|
||||
pagingInfo*: PagingInfo # used for pagination
|
||||
|
||||
HistoryResponse* = object
|
||||
messages*: seq[WakuMessage]
|
||||
pagingInfo*: PagingInfo # used for pagination
|
||||
|
||||
HistoryRPC* = object
|
||||
requestId*: string
|
||||
query*: HistoryQuery
|
||||
response*: HistoryResponse
|
||||
|
||||
HistoryPeer* = object
|
||||
peerInfo*: PeerInfo
|
||||
|
||||
WakuStore* = ref object of LPProtocol
|
||||
switch*: Switch
|
||||
rng*: ref BrHmacDrbgContext
|
||||
peers*: seq[HistoryPeer]
|
||||
messages*: seq[IndexedWakuMessage]
|
||||
store*: MessageStore
|
|
@ -31,7 +31,7 @@ import
|
|||
libp2p/protobuf/minprotobuf,
|
||||
libp2p/stream/connection,
|
||||
../message_notifier,
|
||||
waku_swap_types
|
||||
./waku_swap_types
|
||||
|
||||
export waku_swap_types
|
||||
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
## Core Waku data types are defined here to avoid recursive dependencies.
|
||||
##
|
||||
## TODO Move more common data types here
|
||||
## TODO Move types here into their appropriate place
|
||||
|
||||
import
|
||||
std/[tables, times],
|
||||
|
@ -20,6 +20,12 @@ const MaxPageSize* = 100 # Maximum number of waku messages in each page
|
|||
|
||||
# Common data types -----------------------------------------------------------
|
||||
type
|
||||
|
||||
Index* = object
|
||||
## This type contains the description of an Index used in the pagination of WakuMessages
|
||||
digest*: MDigest[256]
|
||||
receivedTime*: float64
|
||||
|
||||
ContentTopic* = uint32
|
||||
|
||||
Topic* = string
|
||||
|
@ -39,57 +45,6 @@ type
|
|||
topics*: seq[string] # @TODO TOPIC
|
||||
handler*: MessageNotificationHandler
|
||||
|
||||
QueryHandlerFunc* = proc(response: HistoryResponse) {.gcsafe, closure.}
|
||||
|
||||
Index* = object
|
||||
## This type contains the description of an Index used in the pagination of WakuMessages
|
||||
digest*: MDigest[256]
|
||||
receivedTime*: float64
|
||||
|
||||
IndexedWakuMessage* = object
|
||||
## This type is used to encapsulate a WakuMessage and its Index
|
||||
msg*: WakuMessage
|
||||
index*: Index
|
||||
|
||||
PagingDirection* {.pure.} = enum
|
||||
## PagingDirection determines the direction of pagination
|
||||
BACKWARD = uint32(0)
|
||||
FORWARD = uint32(1)
|
||||
|
||||
PagingInfo* = object
|
||||
## This type holds the information needed for the pagination
|
||||
pageSize*: uint64
|
||||
cursor*: Index
|
||||
direction*: PagingDirection
|
||||
|
||||
HistoryQuery* = object
|
||||
topics*: seq[ContentTopic]
|
||||
pagingInfo*: PagingInfo # used for pagination
|
||||
|
||||
HistoryResponse* = object
|
||||
messages*: seq[WakuMessage]
|
||||
pagingInfo*: PagingInfo # used for pagination
|
||||
|
||||
HistoryRPC* = object
|
||||
requestId*: string
|
||||
query*: HistoryQuery
|
||||
response*: HistoryResponse
|
||||
|
||||
HistoryPeer* = object
|
||||
peerInfo*: PeerInfo
|
||||
|
||||
MessageStoreResult*[T] = Result[T, string]
|
||||
|
||||
MessageStore* = ref object of RootObj
|
||||
database*: SqliteDatabase
|
||||
|
||||
WakuStore* = ref object of LPProtocol
|
||||
switch*: Switch
|
||||
rng*: ref BrHmacDrbgContext
|
||||
peers*: seq[HistoryPeer]
|
||||
messages*: seq[IndexedWakuMessage]
|
||||
store*: MessageStore
|
||||
|
||||
FilterRequest* = object
|
||||
contentFilters*: seq[ContentFilter]
|
||||
topic*: string
|
||||
|
@ -132,21 +87,6 @@ type
|
|||
# @TODO MAYBE MORE INFO?
|
||||
Filters* = Table[string, Filter]
|
||||
|
||||
# NOTE based on Eth2Node in NBC eth2_network.nim
|
||||
WakuNode* = ref object of RootObj
|
||||
switch*: Switch
|
||||
wakuRelay*: WakuRelay
|
||||
wakuStore*: WakuStore
|
||||
wakuFilter*: WakuFilter
|
||||
wakuSwap*: WakuSwap
|
||||
peerInfo*: PeerInfo
|
||||
libp2pTransportLoops*: seq[Future[void]]
|
||||
# TODO Revist messages field indexing as well as if this should be Message or WakuMessage
|
||||
messages*: seq[(Topic, WakuMessage)]
|
||||
filters*: Filters
|
||||
subscriptions*: MessageNotificationSubscriptions
|
||||
rng*: ref BrHmacDrbgContext
|
||||
|
||||
WakuRelay* = ref object of GossipSub
|
||||
gossipEnabled*: bool
|
||||
|
||||
|
@ -157,6 +97,11 @@ type
|
|||
|
||||
WakuResult*[T] = Result[T, cstring]
|
||||
|
||||
MessageStoreResult*[T] = Result[T, string]
|
||||
|
||||
MessageStore* = ref object of RootObj
|
||||
database*: SqliteDatabase
|
||||
|
||||
# Encoding and decoding -------------------------------------------------------
|
||||
# TODO Move out to to waku_message module
|
||||
# Possibly same with util functions
|
||||
|
@ -199,15 +144,3 @@ proc generateRequestId*(rng: ref BrHmacDrbgContext): string =
|
|||
var bytes: array[10, byte]
|
||||
brHmacDrbgGenerate(rng[], bytes)
|
||||
toHex(bytes)
|
||||
|
||||
proc computeIndex*(msg: WakuMessage): Index =
|
||||
## Takes a WakuMessage and returns its Index
|
||||
var ctx: sha256
|
||||
ctx.init()
|
||||
ctx.update(msg.contentTopic.toBytes()) # converts the contentTopic to bytes
|
||||
ctx.update(msg.payload)
|
||||
let digest = ctx.finish() # computes the hash
|
||||
ctx.clear()
|
||||
result.digest = digest
|
||||
result.receivedTime = epochTime() # gets the unix timestamp
|
||||
|
||||
|
|
Loading…
Reference in New Issue