feat(service): add wildcard address resolver (#1099)

Co-authored-by: Ludovic Chenut <ludovic@status.im>
This commit is contained in:
diegomrsantos 2024-06-06 13:05:45 +02:00 committed by GitHub
parent f9a6ef06cf
commit bccb305cf5
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
13 changed files with 340 additions and 43 deletions

View File

@ -19,7 +19,8 @@ runnableExamples:
{.push raises: [].}
import
options, tables, chronos, chronicles, sequtils,
options, tables, chronos, chronicles, sequtils
import
switch, peerid, peerinfo, stream/connection, multiaddress,
crypto/crypto, transports/[transport, tcptransport],
muxers/[muxer, mplex/mplex, yamux/yamux],
@ -28,6 +29,7 @@ import
connmanager, upgrademngrs/muxedupgrade, observedaddrmanager,
nameresolving/nameresolver,
errors, utility
import services/wildcardresolverservice
export
switch, peerid, peerinfo, connection, multiaddress, crypto, errors
@ -59,6 +61,7 @@ type
rdv: RendezVous
services: seq[Service]
observedAddrManager: ObservedAddrManager
enableWildcardResolver: bool
proc new*(T: type[SwitchBuilder]): T {.public.} =
## Creates a SwitchBuilder
@ -76,7 +79,8 @@ proc new*(T: type[SwitchBuilder]): T {.public.} =
maxOut: -1,
maxConnsPerPeer: MaxConnectionsPerPeer,
protoVersion: ProtoVersion,
agentVersion: AgentVersion)
agentVersion: AgentVersion,
enableWildcardResolver: true)
proc withPrivateKey*(b: SwitchBuilder, privateKey: PrivateKey): SwitchBuilder {.public.} =
## Set the private key of the switch. Will be used to
@ -85,20 +89,18 @@ proc withPrivateKey*(b: SwitchBuilder, privateKey: PrivateKey): SwitchBuilder {.
b.privKey = some(privateKey)
b
proc withAddress*(b: SwitchBuilder, address: MultiAddress): SwitchBuilder {.public.} =
## | Set the listening address of the switch
## | Calling it multiple time will override the value
b.addresses = @[address]
b
proc withAddresses*(b: SwitchBuilder, addresses: seq[MultiAddress]): SwitchBuilder {.public.} =
proc withAddresses*(b: SwitchBuilder, addresses: seq[MultiAddress], enableWildcardResolver: bool = true): SwitchBuilder {.public.} =
## | Set the listening addresses of the switch
## | Calling it multiple time will override the value
b.addresses = addresses
b.enableWildcardResolver = enableWildcardResolver
b
proc withAddress*(b: SwitchBuilder, address: MultiAddress, enableWildcardResolver: bool = true): SwitchBuilder {.public.} =
## | Set the listening address of the switch
## | Calling it multiple time will override the value
b.withAddresses(@[address], enableWildcardResolver)
proc withSignedPeerRecord*(b: SwitchBuilder, sendIt = true): SwitchBuilder {.public.} =
b.sendSignedPeerRecord = sendIt
b
@ -261,6 +263,9 @@ proc build*(b: SwitchBuilder): Switch
else:
PeerStore.new(identify)
if b.enableWildcardResolver:
b.services.insert(WildcardAddressResolverService.new(), 0)
let switch = newSwitch(
peerInfo = peerInfo,
transports = transports,

View File

@ -137,7 +137,7 @@ proc dialAndUpgrade(
dir = Direction.Out):
Future[Muxer] {.async.} =
debug "Dialing peer", peerId = peerId.get(default(PeerId))
debug "Dialing peer", peerId = peerId.get(default(PeerId)), addrs
for rawAddress in addrs:
# resolve potential dnsaddr

View File

@ -12,7 +12,7 @@
{.push raises: [].}
import tables, hashes
import varint, vbuffer
import vbuffer
import stew/results
export results

View File

@ -24,12 +24,16 @@ type
AddressMapper* =
proc(listenAddrs: seq[MultiAddress]): Future[seq[MultiAddress]]
{.gcsafe, raises: [].}
## A proc that expected to resolve the listen addresses into dialable addresses
PeerInfo* {.public.} = ref object
peerId*: PeerId
listenAddrs*: seq[MultiAddress]
addrs: seq[MultiAddress]
## contains addresses the node listens on, which may include wildcard and private addresses (not directly reachable).
addrs*: seq[MultiAddress]
## contains resolved addresses that other peers can use to connect, including public-facing NAT and port-forwarded addresses.
addressMappers*: seq[AddressMapper]
## contains a list of procs that can be used to resolve the listen addresses into dialable addresses.
protocols*: seq[string]
protoVersion*: string
agentVersion*: string
@ -49,7 +53,11 @@ func shortLog*(p: PeerInfo): auto =
chronicles.formatIt(PeerInfo): shortLog(it)
proc update*(p: PeerInfo) {.async.} =
p.addrs = p.listenAddrs
# p.addrs.len == 0 overrides addrs only if it is the first time update is being executed or if the field is empty.
# p.addressMappers.len == 0 is for when all addressMappers have been removed,
# and we wish to have addrs in its initial state, i.e., a copy of listenAddrs.
if p.addrs.len == 0 or p.addressMappers.len == 0:
p.addrs = p.listenAddrs
for mapper in p.addressMappers:
p.addrs = await mapper(p.addrs)

View File

@ -38,7 +38,7 @@ proc isRunning*(self: AutoRelayService): bool =
proc addressMapper(
self: AutoRelayService,
listenAddrs: seq[MultiAddress]): Future[seq[MultiAddress]] {.async.} =
return concat(toSeq(self.relayAddresses.values))
return concat(toSeq(self.relayAddresses.values)) & listenAddrs
proc reserveAndUpdate(self: AutoRelayService, relayPid: PeerId, switch: Switch) {.async.} =
while self.running:

View File

@ -0,0 +1,205 @@
# Nim-LibP2P
# Copyright (c) 2024 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
{.push raises: [].}
import std/sequtils
import stew/[byteutils, results, endians2]
import chronos, chronos/transports/[osnet, ipnet], chronicles
import ../[multiaddress, multicodec]
import ../switch
logScope:
topics = "libp2p wildcardresolverservice"
type
WildcardAddressResolverService* = ref object of Service
## Service used to resolve wildcard addresses of the type "0.0.0.0" for IPv4 or "::" for IPv6.
## When used with a `Switch`, this service will be automatically set up and stopped
## when the switch starts and stops. This is facilitated by adding the service to the switch's
## list of services using the `.withServices(@[svc])` method in the `SwitchBuilder`.
networkInterfaceProvider: NetworkInterfaceProvider
## Provides a list of network addresses.
addressMapper: AddressMapper
## An implementation of an address mapper that takes a list of listen addresses and expands each wildcard address
## to the respective list of interface addresses. As an example, if the listen address is 0.0.0.0:4001
## and the machine has 2 interfaces with IPs 172.217.11.174 and 64.233.177.113, the address mapper will
## expand the wildcard address to 172.217.11.174:4001 and 64.233.177.113:4001.
NetworkInterfaceProvider* =
proc(addrFamily: AddressFamily): seq[InterfaceAddress] {.gcsafe, raises: [].}
proc isLoopbackOrUp(networkInterface: NetworkInterface): bool =
if (networkInterface.ifType == IfSoftwareLoopback) or
(networkInterface.state == StatusUp): true else: false
proc getAddresses(addrFamily: AddressFamily): seq[InterfaceAddress] =
## Retrieves the addresses of network interfaces based on the specified address family.
##
## It filters the available network interfaces to include only
## those that are either loopback or up. It then collects all the addresses from these
## interfaces and filters them to match the provided address family.
##
## Parameters:
## - `addrFamily`: The address family to filter the network addresses (e.g., `AddressFamily.IPv4` or `AddressFamily.IPv6`).
##
## Returns:
## - A sequence of `InterfaceAddress` objects that match the specified address family.
let
interfaces = getInterfaces().filterIt(it.isLoopbackOrUp())
flatInterfaceAddresses = concat(interfaces.mapIt(it.addresses))
filteredInterfaceAddresses =
flatInterfaceAddresses.filterIt(it.host.family == addrFamily)
return filteredInterfaceAddresses
proc new*(
T: typedesc[WildcardAddressResolverService],
networkInterfaceProvider: NetworkInterfaceProvider = getAddresses,
): T =
## This procedure initializes a new `WildcardAddressResolverService` with the provided network interface provider.
##
## Parameters:
## - `T`: The type descriptor for `WildcardAddressResolverService`.
## - `networkInterfaceProvider`: A provider that offers access to network interfaces. Defaults to a new instance of `NetworkInterfaceProvider`.
##
## Returns:
## - A new instance of `WildcardAddressResolverService`.
return T(networkInterfaceProvider: networkInterfaceProvider)
proc getProtocolArgument*(ma: MultiAddress, codec: MultiCodec): MaResult[seq[byte]] =
var buffer: seq[byte]
for item in ma:
let
ritem = ?item
code = ?ritem.protoCode()
if code == codec:
let arg = ?ritem.protoAddress()
return ok(arg)
err("Multiaddress codec has not been found")
proc getWildcardMultiAddresses(
interfaceAddresses: seq[InterfaceAddress], protocol: Protocol, port: Port
): seq[MultiAddress] =
var addresses: seq[MultiAddress]
for ifaddr in interfaceAddresses:
var address = ifaddr.host
address.port = port
MultiAddress.init(address, protocol).withValue(maddress):
addresses.add(maddress)
addresses
proc getWildcardAddress(
maddress: MultiAddress,
multiCodec: MultiCodec,
anyAddr: openArray[uint8],
addrFamily: AddressFamily,
port: Port,
networkInterfaceProvider: NetworkInterfaceProvider,
): seq[MultiAddress] =
var addresses: seq[MultiAddress]
maddress.getProtocolArgument(multiCodec).withValue(address):
if address == anyAddr:
let filteredInterfaceAddresses = networkInterfaceProvider(addrFamily)
addresses.add(
getWildcardMultiAddresses(filteredInterfaceAddresses, IPPROTO_TCP, port)
)
else:
addresses.add(maddress)
return addresses
proc expandWildcardAddresses(
networkInterfaceProvider: NetworkInterfaceProvider, listenAddrs: seq[MultiAddress]
): seq[MultiAddress] =
var addresses: seq[MultiAddress]
# In this loop we expand bound addresses like `0.0.0.0` and `::` to list of interface addresses.
for listenAddr in listenAddrs:
if TCP_IP.matchPartial(listenAddr):
listenAddr.getProtocolArgument(multiCodec("tcp")).withValue(portArg):
let port = Port(uint16.fromBytesBE(portArg))
if IP4.matchPartial(listenAddr):
let wildcardAddresses = getWildcardAddress(
listenAddr,
multiCodec("ip4"),
AnyAddress.address_v4,
AddressFamily.IPv4,
port,
networkInterfaceProvider,
)
addresses.add(wildcardAddresses)
elif IP6.matchPartial(listenAddr):
let wildcardAddresses = getWildcardAddress(
listenAddr,
multiCodec("ip6"),
AnyAddress6.address_v6,
AddressFamily.IPv6,
port,
networkInterfaceProvider,
)
addresses.add(wildcardAddresses)
else:
addresses.add(listenAddr)
else:
addresses.add(listenAddr)
addresses
method setup*(
self: WildcardAddressResolverService, switch: Switch
): Future[bool] {.async.} =
## Sets up the `WildcardAddressResolverService`.
##
## This method adds the address mapper to the peer's list of address mappers.
##
## Parameters:
## - `self`: The instance of `WildcardAddressResolverService` being set up.
## - `switch`: The switch context in which the service operates.
##
## Returns:
## - A `Future[bool]` that resolves to `true` if the setup was successful, otherwise `false`.
self.addressMapper = proc(
listenAddrs: seq[MultiAddress]
): Future[seq[MultiAddress]] {.async.} =
return expandWildcardAddresses(self.networkInterfaceProvider, listenAddrs)
debug "Setting up WildcardAddressResolverService"
let hasBeenSetup = await procCall Service(self).setup(switch)
if hasBeenSetup:
switch.peerInfo.addressMappers.add(self.addressMapper)
return hasBeenSetup
method run*(self: WildcardAddressResolverService, switch: Switch) {.async, public.} =
## Runs the WildcardAddressResolverService for a given switch.
##
## It updates the peer information for the provided switch by running the registered address mapper. Any other
## address mappers that are registered with the switch will also be run.
##
trace "Running WildcardAddressResolverService"
await switch.peerInfo.update()
method stop*(
self: WildcardAddressResolverService, switch: Switch
): Future[bool] {.async, public.} =
## Stops the WildcardAddressResolverService.
##
## Handles the shutdown process of the WildcardAddressResolverService for a given switch.
## It removes the address mapper from the switch's list of address mappers.
## It then updates the peer information for the provided switch. Any wildcard address wont be resolved anymore.
##
## Parameters:
## - `self`: The instance of the WildcardAddressResolverService.
## - `switch`: The Switch object associated with the service.
##
## Returns:
## - A future that resolves to `true` if the service was successfully stopped, otherwise `false`.
debug "Stopping WildcardAddressResolverService"
let hasBeenStopped = await procCall Service(self).stop(switch)
if hasBeenStopped:
switch.peerInfo.addressMappers.keepItIf(it != self.addressMapper)
await switch.peerInfo.update()
return hasBeenStopped

View File

@ -112,6 +112,9 @@ template withValue*[T](self: Opt[T] | Option[T], value, body: untyped): untyped
let value {.inject.} = temp.get()
body
template withValue*[T, E](self: Result[T, E], value, body: untyped): untyped =
self.toOpt().withValue(value, body)
macro withValue*[T](self: Opt[T] | Option[T], value, body, elseStmt: untyped): untyped =
let elseBody = elseStmt[0]
quote do:

View File

@ -215,9 +215,3 @@ proc default*(T: typedesc[MockResolver]): T =
resolver.ipResponses[("localhost", false)] = @["127.0.0.1"]
resolver.ipResponses[("localhost", true)] = @["::1"]
resolver
proc setDNSAddr*(switch: Switch) {.async.} =
proc addressMapper(listenAddrs: seq[MultiAddress]): Future[seq[MultiAddress]] {.async.} =
return @[MultiAddress.init("/dns4/localhost/").tryGet() & listenAddrs[0][1].tryGet()]
switch.peerInfo.addressMappers.add(addressMapper)
await switch.peerInfo.update()

View File

@ -110,12 +110,12 @@ suite "Autonat Service":
check autonatService.networkReachability == NetworkReachability.Reachable
check libp2p_autonat_reachability_confidence.value(["Reachable"]) == 0.3
check switch1.peerInfo.addrs == switch1.peerInfo.listenAddrs.mapIt(switch1.peerStore.guessDialableAddr(it))
check switch1.peerInfo.addrs == switch1.peerInfo.addrs.mapIt(switch1.peerStore.guessDialableAddr(it))
await allFuturesThrowing(
switch1.stop(), switch2.stop(), switch3.stop(), switch4.stop())
check switch1.peerInfo.addrs == switch1.peerInfo.listenAddrs
check switch1.peerInfo.addrs == switch1.peerInfo.addrs
asyncTest "Peer must be not reachable and then reachable":
@ -261,7 +261,6 @@ suite "Autonat Service":
let autonatService = AutonatService.new(AutonatClient.new(), newRng(), Opt.some(1.seconds), maxQueueSize = 1)
let switch1 = createSwitch(autonatService, maxConnsPerPeer = 0)
await switch1.setDNSAddr()
let switch2 = createSwitch(maxConnsPerPeer = 0, nameResolver = MockResolver.default())
@ -277,6 +276,8 @@ suite "Autonat Service":
autonatService.statusAndConfidenceHandler(statusAndConfidenceHandler)
await switch1.start()
switch1.peerInfo.addrs.add([ MultiAddress.init("/dns4/localhost/").tryGet() & switch1.peerInfo.addrs[0][1].tryGet() ])
await switch2.start()
await switch1.connect(switch2.peerInfo.peerId, switch2.peerInfo.addrs)

View File

@ -80,7 +80,7 @@ suite "Autorelay":
check:
addresses == @[buildRelayMA(switchRelay, switchClient)]
addresses.len() == 1
addresses == switchClient.peerInfo.addrs
addresses[0] in switchClient.peerInfo.addrs
await allFutures(switchClient.stop(), switchRelay.stop())
check addresses != switchClient.peerInfo.addrs

View File

@ -65,11 +65,6 @@ suite "Hole Punching":
let publicPeerSwitch = createSwitch(RelayClient.new())
proc addressMapper(listenAddrs: seq[MultiAddress]): Future[seq[MultiAddress]] {.async.} =
return @[MultiAddress.init("/dns4/localhost/").tryGet() & listenAddrs[0][1].tryGet()]
publicPeerSwitch.peerInfo.addressMappers.add(addressMapper)
await publicPeerSwitch.peerInfo.update()
proc checkMA(address: seq[MultiAddress]) =
if not privatePeerRelayAddr.completed():
privatePeerRelayAddr.complete(address)
@ -83,6 +78,7 @@ suite "Hole Punching":
let switchRelay = createSwitch(Relay.new())
await allFuturesThrowing(switchRelay.start(), privatePeerSwitch.start(), publicPeerSwitch.start(), peerSwitch.start())
publicPeerSwitch.peerInfo.addrs.add([ MultiAddress.init("/dns4/localhost/").tryGet() & publicPeerSwitch.peerInfo.addrs[0][1].tryGet() ])
await privatePeerSwitch.connect(switchRelay.peerInfo.peerId, switchRelay.peerInfo.addrs)
await privatePeerSwitch.connect(peerSwitch.peerInfo.peerId, peerSwitch.peerInfo.addrs) # for autonat
@ -106,7 +102,6 @@ suite "Hole Punching":
let privatePeerRelayAddr = newFuture[seq[MultiAddress]]()
let publicPeerSwitch = createSwitch(RelayClient.new())
await publicPeerSwitch.setDNSAddr()
proc checkMA(address: seq[MultiAddress]) =
if not privatePeerRelayAddr.completed():
@ -121,6 +116,7 @@ suite "Hole Punching":
let switchRelay = createSwitch(Relay.new())
await allFuturesThrowing(switchRelay.start(), privatePeerSwitch.start(), publicPeerSwitch.start(), peerSwitch.start())
publicPeerSwitch.peerInfo.addrs.add([ MultiAddress.init("/dns4/localhost/").tryGet() & publicPeerSwitch.peerInfo.addrs[0][1].tryGet() ])
await privatePeerSwitch.connect(switchRelay.peerInfo.peerId, switchRelay.peerInfo.addrs)
await privatePeerSwitch.connect(peerSwitch.peerInfo.peerId, peerSwitch.peerInfo.addrs) # for autonat
@ -162,7 +158,7 @@ suite "Hole Punching":
let privatePeerSwitch1 = SwitchStub.new(createSwitch(relayClient1, hpservice1, nameresolver = MockResolver.default()))
let privatePeerSwitch2 = SwitchStub.new(createSwitch(relayClient2, hpservice2))
await privatePeerSwitch2.setDNSAddr()
let switchRelay = createSwitch(Relay.new())
let switchAux = createSwitch()
let switchAux2 = createSwitch()

View File

@ -210,8 +210,7 @@ suite "Identify":
asyncTest "simple push identify":
switch2.peerInfo.protocols.add("/newprotocol/")
switch2.peerInfo.listenAddrs.add(MultiAddress.init("/ip4/127.0.0.1/tcp/5555").tryGet())
await switch2.peerInfo.update()
switch2.peerInfo.addrs.add(MultiAddress.init("/ip4/127.0.0.1/tcp/5555").tryGet())
check:
switch1.peerStore[AddressBook][switch2.peerInfo.peerId] != switch2.peerInfo.addrs
@ -224,16 +223,9 @@ suite "Identify":
await closeAll()
# Wait the very end to be sure that the push has been processed
check:
switch1.peerStore[ProtoBook][switch2.peerInfo.peerId] == switch2.peerInfo.protocols
switch1.peerStore[AddressBook][switch2.peerInfo.peerId] == switch2.peerInfo.addrs
asyncTest "wrong peer id push identify":
switch2.peerInfo.protocols.add("/newprotocol/")
switch2.peerInfo.listenAddrs.add(MultiAddress.init("/ip4/127.0.0.1/tcp/5555").tryGet())
await switch2.peerInfo.update()
switch2.peerInfo.addrs.add(MultiAddress.init("/ip4/127.0.0.1/tcp/5555").tryGet())
check:
switch1.peerStore[AddressBook][switch2.peerInfo.peerId] != switch2.peerInfo.addrs

View File

@ -0,0 +1,93 @@
{.used.}
# Nim-Libp2p
# Copyright (c) 2024 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
import std/[options, sequtils]
import stew/[byteutils]
import chronos, metrics
import unittest2
import ../libp2p/[builders, switch]
import ../libp2p/services/wildcardresolverservice
import ../libp2p/[multiaddress, multicodec]
import ./helpers
proc getAddressesMock(
addrFamily: AddressFamily
): seq[InterfaceAddress] {.gcsafe, raises: [].} =
try:
if addrFamily == AddressFamily.IPv4:
return
@[
InterfaceAddress.init(initTAddress("127.0.0.1:0"), 8),
InterfaceAddress.init(initTAddress("192.168.1.22:0"), 24),
]
else:
return
@[
InterfaceAddress.init(initTAddress("::1:0"), 8),
InterfaceAddress.init(initTAddress("fe80::1:0"), 64),
]
except TransportAddressError as e:
echo "Error: " & $e.msg
fail()
proc createSwitch(svc: Service): Switch =
SwitchBuilder
.new()
.withRng(newRng())
.withAddresses(
@[
MultiAddress.init("/ip4/127.0.0.1/tcp/0/").tryGet(),
MultiAddress.init("/ip4/0.0.0.0/tcp/0/").tryGet(),
MultiAddress.init("/ip6/::/tcp/0/").tryGet(),
],
false
)
.withTcpTransport()
.withMplex()
.withNoise()
.withServices(@[svc])
.build()
suite "WildcardAddressResolverService":
teardown:
checkTrackers()
asyncTest "WildcardAddressResolverService must resolve wildcard addresses and stop doing so when stopped":
let svc: Service =
WildcardAddressResolverService.new(networkInterfaceProvider = getAddressesMock)
let switch = createSwitch(svc)
await switch.start()
let tcpIp4Locahost = switch.peerInfo.addrs[0][multiCodec("tcp")].get
let tcpIp4Wildcard = switch.peerInfo.addrs[1][multiCodec("tcp")].get
let tcpIp6 = switch.peerInfo.addrs[2][multiCodec("tcp")].get # tcp port for ip6
check switch.peerInfo.addrs ==
@[
MultiAddress.init("/ip4/127.0.0.1" & $tcpIp4Locahost).get,
MultiAddress.init("/ip4/0.0.0.0" & $tcpIp4Wildcard).get,
MultiAddress.init("/ip6/::" & $tcpIp6).get,
]
await svc.run(switch)
check switch.peerInfo.addrs ==
@[
MultiAddress.init("/ip4/127.0.0.1" & $tcpIp4Locahost).get,
MultiAddress.init("/ip4/127.0.0.1" & $tcpIp4Wildcard).get,
MultiAddress.init("/ip4/192.168.1.22" & $tcpIp4Wildcard).get,
MultiAddress.init("/ip6/::1" & $tcpIp6).get,
MultiAddress.init("/ip6/fe80::1" & $tcpIp6).get,
]
await switch.stop()
check switch.peerInfo.addrs ==
@[
MultiAddress.init("/ip4/127.0.0.1" & $tcpIp4Locahost).get,
MultiAddress.init("/ip4/0.0.0.0" & $tcpIp4Wildcard).get,
MultiAddress.init("/ip6/::" & $tcpIp6).get,
]