Add support for RFC31 ENR (#789)

This commit is contained in:
Hanno Cornelius 2021-12-06 20:51:37 +01:00 committed by GitHub
parent 1217e25245
commit dd256e3bd2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 362 additions and 101 deletions

View File

@ -18,7 +18,8 @@ import
./v2/test_migration_utils,
./v2/test_namespacing_utils,
./v2/test_waku_dnsdisc,
./v2/test_waku_discv5
./v2/test_waku_discv5,
./v2/test_enr_utils
when defined(rln):
import ./v2/test_waku_rln_relay

127
tests/v2/test_enr_utils.nim Normal file
View File

@ -0,0 +1,127 @@
{.used.}
import
testutils/unittests,
std/options,
stew/byteutils,
chronos,
../../waku/v2/utils/wakuenr,
../test_helpers
procSuite "ENR utils":
asyncTest "Parse multiaddr field":
let
reasonable = "0x000A047F0000010601BADD03".hexToSeqByte()
reasonableDns4 = ("0x002F36286E6F64652D30312E646F2D616D73332E77616B7576322E746" &
"573742E737461747573696D2E6E65740601BBDE03003837316E6F64652D" &
"30312E61632D636E2D686F6E676B6F6E672D632E77616B7576322E74657" &
"3742E737461747573696D2E6E65740601BBDE030029BD03ADADEC040BE0" &
"47F9658668B11A504F3155001F231A37F54C4476C07FB4CC139ED7E30304D2DE03").hexToSeqByte()
tooLong = "0x000B047F0000010601BADD03".hexToSeqByte()
tooShort = "0x000A047F0000010601BADD0301".hexToSeqByte()
gibberish = "0x3270ac4e5011123c".hexToSeqByte()
empty = newSeq[byte]()
## Note: we expect to fail optimistically, i.e. extract
## any addresses we can and ignore other errors.
## Worst case scenario is we return an empty `multiaddrs` seq.
check:
# Expected cases
reasonable.toMultiAddresses().contains(MultiAddress.init("/ip4/127.0.0.1/tcp/442/ws")[])
reasonableDns4.toMultiAddresses().contains(MultiAddress.init("/dns4/node-01.do-ams3.wakuv2.test.statusim.net/tcp/443/wss")[])
# Buffer exceeded
tooLong.toMultiAddresses().len() == 0
# Buffer remainder
tooShort.toMultiAddresses().contains(MultiAddress.init("/ip4/127.0.0.1/tcp/442/ws")[])
# Gibberish
gibberish.toMultiAddresses().len() == 0
# Empty
empty.toMultiAddresses().len() == 0
asyncTest "Init ENR for Waku Usage":
# Tests RFC31 encoding "happy path"
let
enrIp = ValidIpAddress.init("127.0.0.1")
enrTcpPort, enrUdpPort = Port(60000)
enrKey = PrivateKey.random(Secp256k1, rng[])[]
wakuFlags = initWakuFlags(false, true, false, true)
multiaddrs = @[MultiAddress.init("/ip4/127.0.0.1/tcp/442/ws")[],
MultiAddress.init("/ip4/127.0.0.1/tcp/443/wss")[]]
let
record = initEnr(enrKey, some(enrIp),
some(enrTcpPort), some(enrUdpPort),
some(wakuFlags),
multiaddrs)
typedRecord = record.toTypedRecord.get()
# Check EIP-778 ENR fields
check:
@(typedRecord.secp256k1.get()) == enrKey.getPublicKey()[].getRawBytes()[]
ipv4(typedRecord.ip.get()) == enrIp
Port(typedRecord.tcp.get()) == enrTcpPort
Port(typedRecord.udp.get()) == enrUdpPort
# Check Waku ENR fields
let
decodedFlags = record.get(WAKU_ENR_FIELD, seq[byte])
decodedAddrs = record.get(MULTIADDR_ENR_FIELD, seq[byte]).toMultiAddresses()
check:
decodedFlags == @[wakuFlags.byte]
decodedAddrs.contains(MultiAddress.init("/ip4/127.0.0.1/tcp/442/ws")[])
decodedAddrs.contains(MultiAddress.init("/ip4/127.0.0.1/tcp/443/wss")[])
asyncTest "Strip multiaddr peerId":
# Tests that peerId is stripped of multiaddrs as per RFC31
let
enrIp = ValidIpAddress.init("127.0.0.1")
enrTcpPort, enrUdpPort = Port(60000)
enrKey = PrivateKey.random(Secp256k1, rng[])[]
multiaddrs = @[MultiAddress.init("/ip4/127.0.0.1/tcp/443/wss/p2p/16Uiu2HAm4v86W3bmT1BiH6oSPzcsSr31iDQpSN5Qa882BCjjwgrD")[]]
let
record = initEnr(enrKey, some(enrIp),
some(enrTcpPort), some(enrUdpPort),
none(WakuEnrBitfield),
multiaddrs)
# Check Waku ENR fields
let
decodedAddrs = record.get(MULTIADDR_ENR_FIELD, seq[byte]).toMultiAddresses()
check decodedAddrs.contains(MultiAddress.init("/ip4/127.0.0.1/tcp/443/wss")[]) # Peer Id has been stripped
asyncTest "Decode ENR with multiaddrs field":
let
# Known values correspond to shared test vectors with other Waku implementations
knownIp = ValidIpAddress.init("18.223.219.100")
knownUdpPort = some(9000.int)
knownTcpPort = none(int)
knownMultiaddrs = @[MultiAddress.init("/dns4/node-01.do-ams3.wakuv2.test.statusim.net/tcp/443/wss")[],
MultiAddress.init("/dns6/node-01.ac-cn-hongkong-c.wakuv2.test.statusim.net/tcp/443/wss")[]]
knownEnr = "enr:-QEnuEBEAyErHEfhiQxAVQoWowGTCuEF9fKZtXSd7H_PymHFhGJA3rGAYDVSH" &
"KCyJDGRLBGsloNbS8AZF33IVuefjOO6BIJpZIJ2NIJpcIQS39tkim11bHRpYWRkcn" &
"O4lgAvNihub2RlLTAxLmRvLWFtczMud2FrdXYyLnRlc3Quc3RhdHVzaW0ubmV0BgG" &
"73gMAODcxbm9kZS0wMS5hYy1jbi1ob25na29uZy1jLndha3V2Mi50ZXN0LnN0YXR1" &
"c2ltLm5ldAYBu94DACm9A62t7AQL4Ef5ZYZosRpQTzFVAB8jGjf1TER2wH-0zBOe1" &
"-MDBNLeA4lzZWNwMjU2azGhAzfsxbxyCkgCqq8WwYsVWH7YkpMLnU2Bw5xJSimxKav-g3VkcIIjKA"
var enrRecord: Record
check:
enrRecord.fromURI(knownEnr)
let typedRecord = enrRecord.toTypedRecord.get()
# Check EIP-778 ENR fields
check:
ipv4(typedRecord.ip.get()) == knownIp
typedRecord.tcp == knownTcpPort
typedRecord.udp == knownUdpPort
# Check Waku ENR fields
let
decodedAddrs = enrRecord.get(MULTIADDR_ENR_FIELD, seq[byte]).toMultiAddresses()
for knownMultiaddr in knownMultiaddrs:
check decodedAddrs.contains(knownMultiaddr)

View File

@ -1266,7 +1266,8 @@ asyncTest "Messages relaying fails with non-overlapping transports (TCP or Webso
node2.mountRelay(@[pubSubTopic])
#delete websocket peer address
node2.peerInfo.addrs.delete(1)
# TODO: a better way to find the index - this is too brittle
node2.peerInfo.addrs.delete(0)
await node1.connectToNodes(@[node2.peerInfo.toRemotePeerInfo()])

View File

@ -1,16 +1,15 @@
{.push raises: [Defect].}
import
std/[bitops, sequtils, strutils, options],
std/[sequtils, strutils, options],
chronos, chronicles, metrics,
eth/keys,
eth/p2p/discoveryv5/[enr, node, protocol],
stew/shims/net,
stew/results,
../config,
../../utils/peers
../../utils/[peers, wakuenr]
export protocol
export protocol, wakuenr
declarePublicGauge waku_discv5_discovered, "number of nodes discovered"
declarePublicGauge waku_discv5_errors, "number of waku discv5 errors", ["type"]
@ -19,18 +18,10 @@ logScope:
topics = "wakudiscv5"
type
## 8-bit flag field to indicate Waku capabilities.
## Only the 4 LSBs are currently defined according
## to RFC31 (https://rfc.vac.dev/spec/31/).
WakuEnrBitfield* = uint8
WakuDiscoveryV5* = ref object
protocol*: protocol.Protocol
listening*: bool
const
WAKU_ENR_FIELD* = "waku2"
####################
# Helper functions #
####################
@ -67,16 +58,6 @@ proc addBootstrapNode(bootstrapAddr: string,
warn "Ignoring invalid bootstrap address",
bootstrapAddr, reason = enrRes.error
proc initWakuFlags*(lightpush, filter, store, relay: bool): WakuEnrBitfield =
## Creates an waku2 ENR flag bit field according to RFC 31 (https://rfc.vac.dev/spec/31/)
var v = 0b0000_0000'u8
if lightpush: v.setBit(3)
if filter: v.setBit(2)
if store: v.setBit(1)
if relay: v.setBit(0)
return v.WakuEnrBitfield
proc isWakuNode(node: Node): bool =
let wakuField = node.record.tryGet(WAKU_ENR_FIELD, uint8)
@ -123,7 +104,7 @@ proc new*(T: type WakuDiscoveryV5,
discv5UdpPort: Port,
bootstrapNodes: seq[string],
enrAutoUpdate = false,
privateKey: PrivateKey,
privateKey: keys.PrivateKey,
flags: WakuEnrBitfield,
enrFields: openArray[(string, seq[byte])],
rng: ref BrHmacDrbgContext): T =

View File

@ -33,23 +33,6 @@ type
client*: Client
resolver*: Resolver
##################
# Util functions #
##################
func createEnr*(privateKey: crypto.PrivateKey,
enrIp: Option[ValidIpAddress],
enrTcpPort, enrUdpPort: Option[Port]): enr.Record =
assert privateKey.scheme == PKScheme.Secp256k1
let
rawPk = privateKey.getRawBytes().expect("Private key is valid")
pk = keys.PrivateKey.fromRaw(rawPk).expect("Raw private key is of valid length")
enr = enr.Record.init(1, pk, enrIp, enrTcpPort, enrUdpPort).expect("Record within size limits")
return enr
#####################
# DNS Discovery API #
#####################

View File

@ -19,9 +19,7 @@ import
../protocol/waku_filter/waku_filter,
../protocol/waku_lightpush/waku_lightpush,
../protocol/waku_rln_relay/[waku_rln_relay_types],
../utils/peers,
../utils/requests,
../utils/wakuswitch,
../utils/[peers, requests, wakuswitch, wakuenr],
./storage/migration/migration_types,
./peer_manager/peer_manager,
./dnsdisc/waku_dnsdisc,
@ -144,13 +142,9 @@ proc updateSwitchPeerInfo(node: WakuNode) =
template tcpEndPoint(address, port): auto =
MultiAddress.init(address, tcpProtocol, port)
template addWsFlag() =
MultiAddress.init("/ws").tryGet()
template addWssFlag() =
MultiAddress.init("/wss").tryGet()
func wsFlag(wssEnabled: bool): MultiAddress {.raises: [Defect, LPError]} =
if wssEnabled: MultiAddress.init("/wss").tryGet()
else: MultiAddress.init("/ws").tryGet()
proc new*(T: type WakuNode, nodeKey: crypto.PrivateKey,
bindIp: ValidIpAddress, bindPort: Port,
@ -161,54 +155,72 @@ proc new*(T: type WakuNode, nodeKey: crypto.PrivateKey,
wsEnabled: bool = false,
wssEnabled: bool = false,
secureKey: string = "",
secureCert: string = ""): T
{.raises: [Defect, LPError, IOError,TLSStreamProtocolError].} =
secureCert: string = "",
wakuFlags = none(WakuEnrBitfield)
): T
{.raises: [Defect, LPError, IOError, TLSStreamProtocolError].} =
## Creates a Waku Node.
##
## Status: Implemented.
##
## Initialize addresses
let
# Bind addresses
hostAddress = tcpEndPoint(bindIp, bindPort)
wsHostAddress = if wsEnabled or wssEnabled: some(tcpEndPoint(bindIp, wsbindPort) & wsFlag(wssEnabled))
else: none(MultiAddress)
# External addresses
hostExtAddress = if extIp.isNone() or extPort.isNone(): none(MultiAddress)
else: some(tcpEndPoint(extIp.get(), extPort.get()))
wsExtAddress = if wsHostAddress.isNone(): none(MultiAddress)
elif hostExtAddress.isNone(): none(MultiAddress)
else: some(tcpEndPoint(extIp.get(), wsBindPort) & wsFlag(wssEnabled))
var announcedAddresses: seq[MultiAddress]
if hostExtAddress.isSome:
announcedAddresses.add(hostExtAddress.get())
else:
announcedAddresses.add(hostAddress) # We always have at least a bind address for the host
if wsExtAddress.isSome:
announcedAddresses.add(wsExtAddress.get())
elif wsHostAddress.isSome:
announcedAddresses.add(wsHostAddress.get())
## Initialize peer
let
rng = crypto.newRng()
hostAddress = tcpEndPoint(bindIp, bindPort)
wsHostAddress = if wssEnabled: tcpEndPoint(bindIp, wsbindPort) & addWssFlag
else: tcpEndPoint(bindIp, wsbindPort) & addWsFlag
announcedAddresses = if extIp.isNone() or extPort.isNone(): @[]
elif wsEnabled == false and wssEnabled == false:
@[tcpEndPoint(extIp.get(), extPort.get())]
elif wssEnabled:
@[tcpEndPoint(extIp.get(), extPort.get()),
tcpEndPoint(extIp.get(), wsBindPort) & addWssFlag]
else : @[tcpEndPoint(extIp.get(), extPort.get()),
tcpEndPoint(extIp.get(), wsBindPort) & addWsFlag]
peerInfo = PeerInfo.new(nodekey)
enrIp = if extIp.isSome(): extIp
else: some(bindIp)
enrTcpPort = if extPort.isSome(): extPort
else: some(bindPort)
enr = createEnr(nodeKey, enrIp, enrTcpPort, none(Port))
enrMultiaddrs = if wsExtAddress.isSome: @[wsExtAddress.get()] # Only add ws/wss to `multiaddrs` field
elif wsHostAddress.isSome: @[wsHostAddress.get()]
else: @[]
enr = initEnr(nodeKey,
enrIp,
enrTcpPort, none(Port),
wakuFlags,
enrMultiaddrs)
if wsEnabled or wssEnabled:
info "Initializing networking", hostAddress, wsHostAddress,
announcedAddresses
peerInfo.addrs.add(wsHostAddress)
else :
info "Initializing networking", hostAddress, announcedAddresses
peerInfo.addrs.add(hostAddress)
# TODO: local peerInfo should be removed
for multiaddr in announcedAddresses:
peerInfo.addrs.add(multiaddr)
info "Initializing networking", addrs=peerInfo.addrs
var switch = newWakuSwitch(some(nodekey),
hostAddress,
wsHostAddress,
transportFlags = {ServerFlags.ReuseAddr},
rng = rng,
maxConnections = maxConnections,
wsEnabled = wsEnabled,
wssEnabled = wssEnabled,
secureKeyPath = secureKey,
secureCertPath = secureCert)
hostAddress,
wsHostAddress,
transportFlags = {ServerFlags.ReuseAddr},
rng = rng,
maxConnections = maxConnections,
wssEnabled = wssEnabled,
secureKeyPath = secureKey,
secureCertPath = secureCert)
let wakuNode = WakuNode(
peerManager: PeerManager.new(switch, peerStorage),
@ -408,7 +420,9 @@ proc info*(node: WakuNode): WakuInfo =
## Status: Implemented.
##
let peerInfo = node.peerInfo
let
peerInfo = node.peerInfo
var listenStr : seq[string]
for address in node.announcedAddresses:
var fulladdr = $address & "/p2p/" & $peerInfo.peerId
@ -975,6 +989,11 @@ when isMainModule:
some(Port(uint16(conf.tcpPort) + conf.portsShift))
else:
extTcpPort
wakuFlags = initWakuFlags(conf.lightpush,
conf.filter,
conf.store,
conf.relay)
node = WakuNode.new(conf.nodekey,
conf.listenAddress, Port(uint16(conf.tcpPort) + conf.portsShift),
@ -985,7 +1004,8 @@ when isMainModule:
conf.websocketSupport,
conf.websocketSecureSupport,
conf.websocketSecureKeyPath,
conf.websocketSecureCertPath
conf.websocketSecureCertPath,
some(wakuFlags)
)
if conf.discv5Discovery:
@ -998,10 +1018,7 @@ when isMainModule:
conf.discv5BootstrapNodes,
conf.discv5EnrAutoUpdate,
keys.PrivateKey(conf.nodekey.skkey),
initWakuFlags(conf.lightpush,
conf.filter,
conf.store,
conf.relay),
wakuFlags,
[], # Empty enr fields, for now
node.rng
)

153
waku/v2/utils/wakuenr.nim Normal file
View File

@ -0,0 +1,153 @@
## Collection of utilities related to Waku's use of EIP-778 ENR
## Implemented according to the specified Waku v2 ENR usage
## More at https://rfc.vac.dev/spec/31/
{.push raises: [Defect]}
import
std/[bitops, sequtils],
eth/keys,
eth/p2p/discoveryv5/enr,
libp2p/[multiaddress, multicodec],
libp2p/crypto/crypto,
stew/[endians2, results],
stew/shims/net
export enr, crypto, multiaddress, net
const
MULTIADDR_ENR_FIELD* = "multiaddrs"
WAKU_ENR_FIELD* = "waku2"
type
## 8-bit flag field to indicate Waku capabilities.
## Only the 4 LSBs are currently defined according
## to RFC31 (https://rfc.vac.dev/spec/31/).
WakuEnrBitfield* = uint8
func toFieldPair(multiaddrs: seq[MultiAddress]): FieldPair =
## Converts a seq of multiaddrs to a `multiaddrs` ENR
## field pair according to https://rfc.vac.dev/spec/31/
var fieldRaw: seq[byte]
for multiaddr in multiaddrs:
let
maRaw = multiaddr.data.buffer # binary encoded multiaddr
maSize = maRaw.len.uint16.toBytes(Endianness.bigEndian) # size as Big Endian unsigned 16-bit integer
assert maSize.len == 2
fieldRaw.add(concat(@maSize, maRaw))
return toFieldPair(MULTIADDR_ENR_FIELD, fieldRaw)
func stripPeerId(multiaddr: MultiAddress): MultiAddress =
var cleanAddr = MultiAddress.init()
for item in multiaddr.items:
if item[].protoName()[] != "p2p":
# Add all parts except p2p peerId
discard cleanAddr.append(item[])
return cleanAddr
func stripPeerIds(multiaddrs: seq[MultiAddress]): seq[MultiAddress] =
var cleanAddrs: seq[MultiAddress]
for multiaddr in multiaddrs:
if multiaddr.contains(multiCodec("p2p"))[]:
cleanAddrs.add(multiaddr.stripPeerId())
else:
cleanAddrs.add(multiaddr)
return cleanAddrs
func readBytes(rawBytes: seq[byte], numBytes: int, pos: var int = 0): Result[seq[byte], cstring] =
## Attempts to read `numBytes` from a sequence, from
## position `pos`. Returns the requested slice or
## an error if `rawBytes` boundary is exceeded.
##
## If successful, `pos` is advanced by `numBytes`
if rawBytes[pos..^1].len() < numBytes:
return err("Exceeds maximum available bytes")
let slicedSeq = rawBytes[pos..<pos+numBytes]
pos += numBytes
return ok(slicedSeq)
################
# Public utils #
################
func initWakuFlags*(lightpush, filter, store, relay: bool): WakuEnrBitfield =
## Creates an waku2 ENR flag bit field according to RFC 31 (https://rfc.vac.dev/spec/31/)
var v = 0b0000_0000'u8
if lightpush: v.setBit(3)
if filter: v.setBit(2)
if store: v.setBit(1)
if relay: v.setBit(0)
return v.WakuEnrBitfield
func toMultiAddresses*(multiaddrsField: seq[byte]): seq[MultiAddress] =
## Parses a `multiaddrs` ENR field according to
## https://rfc.vac.dev/spec/31/
var multiaddrs: seq[MultiAddress]
let totalLen = multiaddrsField.len()
if totalLen < 2:
return multiaddrs
var pos = 0
while pos < totalLen:
let addrLenRes = multiaddrsField.readBytes(2, pos)
if addrLenRes.isErr():
return multiaddrs
let addrLen = uint16.fromBytesBE(addrLenRes.get())
if addrLen == 0.uint16:
# Ensure pos always advances and we don't get stuck in infinite loop
return multiaddrs
let addrRaw = multiaddrsField.readBytes(addrLen.int, pos)
if addrRaw.isErr():
return multiaddrs
let multiaddr = MultiAddress.init(addrRaw.get())
if multiaddr.isErr():
return multiaddrs
multiaddrs.add(multiaddr.get())
return multiaddrs
func initEnr*(privateKey: crypto.PrivateKey,
enrIp: Option[ValidIpAddress],
enrTcpPort, enrUdpPort: Option[Port],
wakuFlags = none(WakuEnrBitfield),
multiaddrs: seq[MultiAddress] = @[]): enr.Record =
assert privateKey.scheme == PKScheme.Secp256k1
## Waku-specific ENR fields (https://rfc.vac.dev/spec/31/)
var wakuEnrFields: seq[FieldPair]
# `waku2` field
if wakuFlags.isSome:
wakuEnrFields.add(toFieldPair(WAKU_ENR_FIELD, @[wakuFlags.get().byte]))
# `multiaddrs` field
if multiaddrs.len > 0:
wakuEnrFields.add(multiaddrs.stripPeerIds().toFieldPair)
let
rawPk = privateKey.getRawBytes().expect("Private key is valid")
pk = keys.PrivateKey.fromRaw(rawPk).expect("Raw private key is of valid length")
enr = enr.Record.init(1, pk,
enrIp, enrTcpPort, enrUdpPort,
wakuEnrFields).expect("Record within size limits")
return enr

View File

@ -50,7 +50,7 @@ proc withWssTransport*(b: SwitchBuilder,
proc newWakuSwitch*(
privKey = none(crypto.PrivateKey),
address = MultiAddress.init("/ip4/127.0.0.1/tcp/0").tryGet(),
wsAddress = MultiAddress.init("/ip4/127.0.0.1/tcp/1").tryGet(),
wsAddress = none(MultiAddress),
secureManagers: openarray[SecureProtocol] = [
SecureProtocol.Noise,
],
@ -63,15 +63,11 @@ proc newWakuSwitch*(
maxOut = -1,
maxConnsPerPeer = MaxConnectionsPerPeer,
nameResolver: NameResolver = nil,
wsEnabled: bool = false,
wssEnabled: bool = false,
secureKeyPath: string = "",
secureCertPath: string = ""): Switch
{.raises: [Defect,TLSStreamProtocolError,IOError, LPError].} =
if wsEnabled == true and wssEnabled == true:
debug "Websocket and secure websocket are enabled simultaneously."
var b = SwitchBuilder
.new()
.withRng(rng)
@ -85,12 +81,14 @@ proc newWakuSwitch*(
.withNameResolver(nameResolver)
if privKey.isSome():
b = b.withPrivateKey(privKey.get())
if wsEnabled == true:
b = b.withAddresses(@[wsAddress, address])
b = b.withWsTransport()
elif wssEnabled == true:
b = b.withAddresses(@[wsAddress, address])
b = b.withWssTransport(secureKeyPath, secureCertPath)
if wsAddress.isSome():
b = b.withAddresses(@[wsAddress.get(), address])
if wssEnabled:
b = b.withWssTransport(secureKeyPath, secureCertPath)
else:
b = b.withWsTransport()
else :
b = b.withAddress(address)