diff --git a/.dockerignore b/.dockerignore index 6427da1d..e201c832 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,4 +3,3 @@ build docs metrics nimcache -tests diff --git a/.github/workflows/ci-reusable.yml b/.github/workflows/ci-reusable.yml index fe0c13af..dd4227ca 100644 --- a/.github/workflows/ci-reusable.yml +++ b/.github/workflows/ci-reusable.yml @@ -57,7 +57,7 @@ jobs: - name: Upload integration tests log files uses: actions/upload-artifact@v7 - if: (matrix.tests == 'integration' || matrix.tests == 'all') && always() + if: (matrix.tests == 'integration' || matrix.tests == 'nat-integration' || matrix.tests == 'all') && always() with: name: ${{ matrix.os }}-${{ matrix.cpu }}-${{ matrix.nim_version }}-${{ matrix.job_number }}-integration-tests-logs path: tests/integration/logs/ @@ -68,6 +68,16 @@ jobs: if: matrix.tests == 'libstorage' || matrix.tests == 'all' run: make -j${ncpu} testLibstorage + ## Part 4 Tests ## + - name: NAT integration tests + if: matrix.tests == 'nat-integration' + env: + STORAGE_INTEGRATION_TEST_INCLUDES: ${{ matrix.includes }} + run: | + sudo modprobe iptable_nat nf_conntrack + pipx install podman-compose + make testNatIntegration + status: if: always() needs: [build] diff --git a/.gitignore b/.gitignore index f689aee1..166a02ff 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ !LICENSE* !Makefile !Jenkinsfile +!Dockerfile nimcache/ diff --git a/.gitmodules b/.gitmodules index 8538670b..6c323ffb 100644 --- a/.gitmodules +++ b/.gitmodules @@ -42,7 +42,7 @@ path = vendor/asynctest url = https://github.com/status-im/asynctest.git ignore = untracked - branch = main + branch = main [submodule "vendor/nim-presto"] path = vendor/nim-presto url = https://github.com/status-im/nim-presto.git @@ -53,11 +53,6 @@ url = https://github.com/status-im/nim-confutils.git ignore = untracked branch = master -[submodule "vendor/nim-nat-traversal"] - path = vendor/nim-nat-traversal - url = https://github.com/status-im/nim-nat-traversal.git - ignore = untracked - branch = master [submodule "vendor/nim-libbacktrace"] path = vendor/nim-libbacktrace url = https://github.com/status-im/nim-libbacktrace.git @@ -196,3 +191,9 @@ url = https://github.com/vacp2p/nim-lsquic.git ignore = untracked branch = main +[submodule "vendor/nim-libplum"] + path = vendor/nim-libplum + url = https://github.com/logos-storage/nim-libplum.git +[submodule "vendor/nim-nat-traversal"] + path = vendor/nim-nat-traversal + url = https://github.com/status-im/nim-nat-traversal.git diff --git a/Makefile b/Makefile index f7945253..3ecc4c10 100644 --- a/Makefile +++ b/Makefile @@ -82,10 +82,13 @@ endif coverage \ deps \ libbacktrace \ + libplum \ test \ testAll \ testIntegration \ testLibstorage \ + buildNatImage \ + testNatIntegration \ update ifeq ($(NIM_PARAMS),) @@ -120,7 +123,7 @@ else NIM_PARAMS := $(NIM_PARAMS) -d:release endif -deps: | deps-common nat-libs +deps: | deps-common nat-libs libplum ifneq ($(USE_LIBBACKTRACE), 0) deps: | libbacktrace endif @@ -147,6 +150,17 @@ testIntegration: | build deps echo -e $(BUILD_MSG) "build/$@" && \ $(ENV_SCRIPT) nim testIntegration $(TEST_PARAMS) $(NIM_PARAMS) build.nims +DOCKER := $(or $(shell which podman 2>/dev/null), $(shell which docker 2>/dev/null)) + +# NAT real-topology scenarios (podman-compose), all sharing one image built here. +# Runs every scenario; limit it with STORAGE_INTEGRATION_TEST_INCLUDES (test file +# paths), as testIntegration does. +buildNatImage: + $(DOCKER) build -t localhost/storage-nat -f tests/integration/nat/Dockerfile . + +testNatIntegration: | deps buildNatImage + $(ENV_SCRIPT) nim testNatIntegration $(NIM_PARAMS) build.nims + # Builds a C example that uses the libstorage C library and runs it testLibstorage: | build deps $(MAKE) $(if $(ncpu),-j$(ncpu),) libstorage @@ -165,6 +179,23 @@ testAll: | build deps $(ENV_SCRIPT) nim testAll $(NIM_PARAMS) build.nims $(MAKE) $(if $(ncpu),-j$(ncpu),) testLibstorage +LIBPLUM_DIR := vendor/nim-libplum/vendor/libplum +LIBPLUM_BUILD_DIR := $(LIBPLUM_DIR)/build +LIBPLUM_CMAKE_FLAGS := -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF + +libplum: +ifeq ($(detected_OS), Windows) +ifneq ($(MSYSTEM),) + cmake -B $(LIBPLUM_BUILD_DIR) $(LIBPLUM_CMAKE_FLAGS) -G"MSYS Makefiles" $(LIBPLUM_DIR) $(HANDLE_OUTPUT) +else + cmake -B $(LIBPLUM_BUILD_DIR) $(LIBPLUM_CMAKE_FLAGS) $(LIBPLUM_DIR) $(HANDLE_OUTPUT) +endif +else + cmake -B $(LIBPLUM_BUILD_DIR) $(LIBPLUM_CMAKE_FLAGS) $(LIBPLUM_DIR) $(HANDLE_OUTPUT) +endif + + $(MAKE) -C $(LIBPLUM_BUILD_DIR) $(HANDLE_OUTPUT) + cp $(LIBPLUM_BUILD_DIR)/libplum.a $(LIBPLUM_DIR)/libplum.a + # nim-libbacktrace LIBBACKTRACE_MAKE_FLAGS := -C vendor/nim-libbacktrace --no-print-directory BUILD_CXX_LIB=0 libbacktrace: diff --git a/build.nims b/build.nims index b74a931f..25a31538 100644 --- a/build.nims +++ b/build.nims @@ -78,6 +78,10 @@ task testIntegration, "Run integration tests": # test "testIntegration", params = "-d:chronicles_sinks=textlines[notimestamps,stdout],textlines[dynamic] " & # "-d:chronicles_enabled_topics:integration:TRACE" +task testNatIntegration, + "Run NAT real-topology scenarios (needs the storage-nat image + podman-compose)": + test "testNatIntegration" + task build, "build Logos Storage binary": storageTask() diff --git a/config.nims b/config.nims index 5b1ecb00..f369ca80 100644 --- a/config.nims +++ b/config.nims @@ -140,7 +140,7 @@ switch("warning", "ObservableStores:off") # Too many false positives for "Warning: method has lock level , but another method has 0 [LockLevel]" switch("warning", "LockLevel:off") -switch("define", "libp2p_pki_schemes=secp256k1") +switch("define", "libp2p_pki_schemes=secp256k1,rsa") #TODO this infects everything in this folder, ideally it would only # apply to storage.nim, but since storage.nims is used for other purpose # we can't use it. And storage.cfg doesn't work diff --git a/library/storage_thread_requests/requests/node_debug_request.nim b/library/storage_thread_requests/requests/node_debug_request.nim index 8bf3106c..2c2808a2 100644 --- a/library/storage_thread_requests/requests/node_debug_request.nim +++ b/library/storage_thread_requests/requests/node_debug_request.nim @@ -9,12 +9,15 @@ import std/[options] import chronos import chronicles import codexdht/discv5/spr +import pkg/libp2p/services/autorelayservice import ../../alloc import ../../../storage/conf import ../../../storage/rest/json import ../../../storage/node -from ../../../storage/storage import StorageServer, node +from ../../../storage/storage import StorageServer, node, config +import ../../../storage/nat +import ../../../storage/discovery logScope: topics = "libstorage libstoragedebug" @@ -51,14 +54,25 @@ proc getDebug( ): Future[Result[string, string]] {.async: (raises: []).} = let node = storage[].node let table = RestRoutingTable.init(node.discovery.protocol.routingTable) + let nodeSpr = node.discovery.getSpr() let json = %*{ "id": $node.switch.peerInfo.peerId, "addrs": node.switch.peerInfo.addrs.mapIt($it), - "spr": - if node.discovery.dhtRecord.isSome: node.discovery.dhtRecord.get.toURI else: "", + "repo": storage[].config.dataDir.string, + "spr": nodeSpr.toURI, "announceAddresses": node.discovery.announceAddrs, + "dhtAddresses": node.discovery.dhtAddrs, "table": table, + "storage": {"version": $storageVersion, "revision": $storageRevision}, + "nat": { + "reachability": reachabilityStr(storage[].autonatService), + "clientMode": node.discovery.protocol.clientMode, + "relayRunning": + storage[].autoRelayService.isSome and storage[].autoRelayService.get.isRunning, + "portMapping": portMappingStr(storage[].natMapper), + }, + "connections": peerConnections(node.switch), } return ok($json) diff --git a/library/storage_thread_requests/requests/node_info_request.nim b/library/storage_thread_requests/requests/node_info_request.nim index 7e755a3a..931e8ca6 100644 --- a/library/storage_thread_requests/requests/node_info_request.nim +++ b/library/storage_thread_requests/requests/node_info_request.nim @@ -1,6 +1,5 @@ ## This file contains the lifecycle request type that will be handled. -import std/[options] import chronos import chronicles import confutils @@ -10,6 +9,7 @@ import ../../../storage/rest/json import ../../../storage/node from ../../../storage/storage import StorageServer, config, node +import ../../../storage/discovery logScope: topics = "libstorage libstorageinfo" @@ -38,11 +38,7 @@ proc getRepo( proc getSpr( storage: ptr StorageServer ): Future[Result[string, string]] {.async: (raises: []).} = - let spr = storage[].node.discovery.dhtRecord - if spr.isNone: - return err("Failed to get SPR: no SPR record found.") - - return ok(spr.get.toURI) + return ok(storage[].node.discovery.getSpr().toURI) proc getPeerId( storage: ptr StorageServer diff --git a/openapi.yaml b/openapi.yaml index 041127ee..5743de87 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -106,6 +106,7 @@ components: - repo - spr - announceAddresses + - dhtAddresses - table - storage properties: @@ -124,10 +125,55 @@ components: type: array items: $ref: "#/components/schemas/MultiAddress" + dhtAddresses: + type: array + items: + $ref: "#/components/schemas/MultiAddress" table: $ref: "#/components/schemas/PeersTable" storage: $ref: "#/components/schemas/StorageVersion" + nat: + $ref: "#/components/schemas/NatInfo" + connections: + type: array + items: + $ref: "#/components/schemas/Connection" + + NatInfo: + type: object + required: + - reachability + - clientMode + - relayRunning + - portMapping + properties: + reachability: + type: string + enum: [Unknown, Reachable, NotReachable] + description: AutoNAT reachability status + clientMode: + type: boolean + description: Whether the DHT is running in client mode (not added to remote routing tables) + relayRunning: + type: boolean + description: Whether the AutoRelay service is currently running + portMapping: + type: string + enum: [none, upnp, pmp, pcp, direct] + description: Active NAT port mapping type + + Connection: + type: object + required: + - peerId + - direct + properties: + peerId: + $ref: "#/components/schemas/PeerId" + direct: + type: boolean + description: Whether at least one connection to this peer is direct (not relayed) DataList: type: object diff --git a/storage/blockexchange/network/network.nim b/storage/blockexchange/network/network.nim index 1d7ebafb..b99d8af7 100644 --- a/storage/blockexchange/network/network.nim +++ b/storage/blockexchange/network/network.nim @@ -314,8 +314,15 @@ proc new*( ## Create a new BlockExcNetwork instance ## + # libp2p now requires a non-nil handler at construction; the real one is set + # by self.init() below. This placeholder only exists until then. + proc placeholder( + conn: Connection, proto: string + ): Future[void] {.async: (raises: [CancelledError]).} = + discard + let self = lp_protocol.new( - BlockExcNetwork, @[Codec], nil, maxIncomingStreamsTotal = maxInflight + BlockExcNetwork, @[Codec], placeholder, maxIncomingStreamsTotal = maxInflight ) self.switch = switch self.getConn = connProvider diff --git a/storage/conf.nim b/storage/conf.nim index 2937bad4..b965ea39 100644 --- a/storage/conf.nim +++ b/storage/conf.nim @@ -53,6 +53,8 @@ export DefaultQuotaBytes, DefaultBlockTtl, DefaultBlockInterval, DefaultNumBlocksPerInterval, DefaultBlockRetries +const DefaultNatScheduleInterval* = 2.minutes + type ThreadCount* = distinct Natural proc `==`*(a, b: ThreadCount): bool {.borrow.} @@ -156,10 +158,9 @@ type nat* {. desc: "Specify method to use for determining public address. " & - "Must be one of: any, none, upnp, pmp, extip:. " & - "If connecting to peers on a local network only, use 'none'.", + "Must be one of: auto, extip:.", defaultValue: defaultNatConfig(), - defaultValueDesc: "any", + defaultValueDesc: "auto", name: "nat" .}: NatConfig @@ -309,11 +310,129 @@ type desc: "Logs to file", defaultValue: string.none, name: "log-file", hidden .}: Option[string] + natScheduleInterval* {. + desc: "Interval between AutoNAT reachability checks", + defaultValue: DefaultNatScheduleInterval, + defaultValueDesc: $DefaultNatScheduleInterval, + name: "nat-schedule-interval" + .}: Duration + + natNumPeersToAsk* {. + desc: "Number of peers to ask per AutoNAT round", + defaultValue: 3, + name: "nat-num-peers-to-ask" + .}: int + + natMaxQueueSize* {. + desc: "Number of past AutoNAT results kept to calculate confidence", + defaultValue: 3, + name: "nat-max-queue-size" + .}: int + + natMinConfidence* {. + # With maxQueueSize=3, 0.6 confirms reachability on a 2/3 majority + # (2/3=0.667) instead of a 3/3 unanimous round, tolerating one inconsistent + # peer. + desc: "Minimum confidence threshold to confirm reachability", + defaultValue: 0.6, + name: "nat-min-confidence" + .}: float + + natObservedAddrMinCount* {. + desc: + "Number of identify observations of the same external address required " & + "before it is used as the node's dialable address", + defaultValue: 1, + name: "nat-observed-addr-min-count" + .}: int + + natMaxRelays* {. + desc: "Maximum number of relay servers to reserve slots on simultaneously", + defaultValue: 2, + name: "nat-max-relays" + .}: int + + natPortMappingDiscoverTimeout* {. + desc: "Timeout in milliseconds for UPnP/NAT-PMP/PCP device discovery", + defaultValue: 500, + name: "nat-port-mapping-discover-timeout" + .}: int + + natPortMappingTimeout* {. + desc: "Timeout in milliseconds for creating a port mapping on the router", + defaultValue: 500, + name: "nat-port-mapping-timeout" + .}: int + + natPortMappingRecheckPeriod* {. + desc: "Period in milliseconds between rechecks of existing port mappings", + defaultValue: 300000, + name: "nat-port-mapping-recheck-period" + .}: int + + autonatServer* {. + desc: "Enable AutoNAT server to help other nodes check their reachability", + defaultValue: false, + name: "autonat-server", + hidden + .}: bool + + isRelayServer* {. + desc: "Enable circuit relay server (hop) - use on publicly reachable nodes only", + defaultValue: false, + name: "relay-server" + .}: bool + func defaultAddress*(conf: StorageConf): IpAddress = result = static parseIpAddress("127.0.0.1") func defaultNatConfig*(): NatConfig = - result = NatConfig(hasExtIp: false, nat: NatStrategy.NatAny) + result = NatConfig(hasExtIp: false, nat: NatStrategy.NatAuto) + +func validateAutonatConfig*(config: StorageConf): ?!void = + # An autonat or relay server must be Reachable, assumed with extIp. + # In other words, a node cannot be autonat server AND autonat client. + # Currently, only bootstrap nodes should be autonat servers. + if config.autonatServer and not config.nat.hasExtIp: + return failure "--autonat-server requires --nat=extip:" + + if config.isRelayServer and not config.nat.hasExtIp: + return failure "--relay-server requires --nat=extip:" + + if config.noBootstrapNode and not config.nat.hasExtIp: + return failure( + "--no-bootstrap-node requires --nat=extip:: without bootstrap peers " & + "AutoNAT has no one to probe and the node can never become reachable" + ) + + if config.natMaxQueueSize < 1: + return failure "--nat-max-queue-size must be at least 1" + + if config.natNumPeersToAsk < 1: + return failure "--nat-num-peers-to-ask must be at least 1" + + if config.natObservedAddrMinCount < 1: + return failure "--nat-observed-addr-min-count must be at least 1" + + if config.natMinConfidence < 0.0 or config.natMinConfidence > 1.0: + return failure "--nat-min-confidence must be between 0 and 1" + + if config.natScheduleInterval <= 0.seconds: + return failure "--nat-schedule-interval must be greater than 0" + + if config.natMaxRelays < 1: + return failure "--nat-max-relays must be at least 1" + + if config.natPortMappingDiscoverTimeout < 1: + return failure "--nat-port-mapping-discover-timeout must be greater than 0" + + if config.natPortMappingTimeout < 1: + return failure "--nat-port-mapping-timeout must be greater than 0" + + if config.natPortMappingRecheckPeriod < 1: + return failure "--nat-port-mapping-recheck-period must be greater than 0" + + success() proc getStorageVersion(): string = let tag = strip(staticExec("git describe --tags --abbrev=0")) @@ -379,14 +498,8 @@ proc parseCmdArg*(T: type SignedPeerRecord, uri: string): T = func parse*(T: type NatConfig, p: string): Result[NatConfig, string] = case p.toLowerAscii - of "any": - return ok(NatConfig(hasExtIp: false, nat: NatStrategy.NatAny)) - of "none": - return ok(NatConfig(hasExtIp: false, nat: NatStrategy.NatNone)) - of "upnp": - return ok(NatConfig(hasExtIp: false, nat: NatStrategy.NatUpnp)) - of "pmp": - return ok(NatConfig(hasExtIp: false, nat: NatStrategy.NatPmp)) + of "auto": + return ok(NatConfig(hasExtIp: false, nat: NatStrategy.NatAuto)) else: if p.startsWith("extip:"): try: @@ -396,7 +509,7 @@ func parse*(T: type NatConfig, p: string): Result[NatConfig, string] = let error = "Not a valid IP address: " & p[6 ..^ 1] return err(error) else: - return err("Not a valid NAT option: " & p) + return err("Not a valid NAT option: " & p & ". Valid options: auto, extip:") proc parseCmdArg*(T: type NatConfig, p: string): T = let res = NatConfig.parse(p) diff --git a/storage/discovery.nim b/storage/discovery.nim index c5943d88..766a2fc0 100644 --- a/storage/discovery.nim +++ b/storage/discovery.nim @@ -22,6 +22,7 @@ import pkg/codexdht/discv5/[routing_table, protocol as discv5] from pkg/nimcrypto import keccak256 import ./rng as storage_rng +import ./utils/addrutils import ./errors import ./logutils @@ -42,7 +43,7 @@ type Discovery* = ref object of RootObj providerRecord*: ?SignedPeerRecord # record to advertice node connection information, this carry any # address that the node can be connected on - dhtRecord*: ?SignedPeerRecord # record to advertice DHT connection information + dhtAddrs*: seq[MultiAddress] # UDP discovery addresses, exposed for debugging isStarted: bool store: Datastore @@ -175,12 +176,36 @@ method removeProvider*( warn "Error removing provider", peerId = peerId, exc = exc.msg raiseAssert("Unexpected Exception in removeProvider") -proc updateAnnounceRecord*(d: Discovery, addrs: openArray[MultiAddress]) = - ## Update providers record - ## +proc getSpr*(d: Discovery): SignedPeerRecord = + ## Returns the node's current Signed Peer Record as registered in the DHT. + d.protocol.getRecord() +proc announceDirectAddrs*( + d: Discovery, announceAddrs: openArray[MultiAddress], udpPort: Port +) = + # UDP addresses are derived from TCP announce addresses by remapping protocol and port. + let tcpAddrs = @announceAddrs + let udpAddrs = + tcpAddrs.mapIt(it.remapAddr(protocol = some("udp"), port = some(udpPort))) + + info "Updating announce and DHT records", tcpAddrs, udpAddrs + + d.announceAddrs = tcpAddrs + d.dhtAddrs = udpAddrs + d.providerRecord = SignedPeerRecord + .init(d.key, PeerRecord.init(d.peerId, tcpAddrs)) + .expect("Should construct signed record").some + + if not d.protocol.isNil: + let spr = SignedPeerRecord + .init(d.key, PeerRecord.init(d.peerId, tcpAddrs & udpAddrs)) + .expect("Should construct signed record").some + d.protocol.updateRecord(spr).expect("Should update SPR") + +proc announceRelayAddrs*(d: Discovery, addrs: openArray[MultiAddress]) = + ## Updates the announce addresses and the SPR with the relay circuit addresses. + ## Unlike announceDirectAddrs, no UDP address is derived so dhtAddrs is left untouched. d.announceAddrs = @addrs - info "Updating announce record", addrs = d.announceAddrs d.providerRecord = SignedPeerRecord .init(d.key, PeerRecord.init(d.peerId, d.announceAddrs)) @@ -189,18 +214,6 @@ proc updateAnnounceRecord*(d: Discovery, addrs: openArray[MultiAddress]) = if not d.protocol.isNil: d.protocol.updateRecord(d.providerRecord).expect("Should update SPR") -proc updateDhtRecord*(d: Discovery, addrs: openArray[MultiAddress]) = - ## Update providers record - ## - - info "Updating Dht record", addrs = addrs - d.dhtRecord = SignedPeerRecord - .init(d.key, PeerRecord.init(d.peerId, @addrs)) - .expect("Should construct signed record").some - - if not d.protocol.isNil: - d.protocol.updateRecord(d.dhtRecord).expect("Should update SPR") - proc start*(d: Discovery) {.async: (raises: []).} = try: d.protocol.open() @@ -237,7 +250,8 @@ proc new*( key: PrivateKey, bindIp = IPv4_any(), bindPort = 0.Port, - announceAddrs: openArray[MultiAddress], + announceAddrs: openArray[MultiAddress] = [], + discoveryPort = 0.Port, bootstrapNodes: openArray[SignedPeerRecord] = [], store: Datastore = SQLiteDatastore.new(Memory).expect("Should not fail!"), tableIpLimits: TableIpLimits = DefaultTableIpLimits, @@ -249,7 +263,9 @@ proc new*( key: key, peerId: PeerId.init(key).expect("Should construct PeerId"), store: store ) - self.updateAnnounceRecord(announceAddrs) + # Called even when announceAddrs is empty: newProtocol below requires + # providerRecord to be set, and it will be updated with real addresses in start(). + self.announceDirectAddrs(announceAddrs, udpPort = discoveryPort) let discoveryConfig = DiscoveryConfig(tableIpLimits: tableIpLimits, bitsPerHop: DefaultBitsPerHop) diff --git a/storage/logutils.nim b/storage/logutils.nim index ebf41fd1..cbf41916 100644 --- a/storage/logutils.nim +++ b/storage/logutils.nim @@ -94,7 +94,7 @@ import std/typetraits import pkg/chronicles except toJson, `%` import json_serialization/writer as json_serialization_writer from pkg/chronos import TransportAddress -from pkg/libp2p import Cid, MultiAddress, `$` +from pkg/libp2p import Cid, MultiAddress, SignedPeerRecord, `$` import pkg/questionable import pkg/questionable/results import ./utils/json except formatIt # TODO: remove exception? @@ -248,6 +248,8 @@ formatIt(UInt256): $it formatIt(MultiAddress): $it +formatIt(SignedPeerRecord): + $it formatIt(LogFormat.textLines, array[32, byte]): it.short0xHexLog formatIt(LogFormat.json, array[32, byte]): diff --git a/storage/nat.nim b/storage/nat.nim index 60dedc49..53735cab 100644 --- a/storage/nat.nim +++ b/storage/nat.nim @@ -8,421 +8,345 @@ {.push raises: [].} -import - std/[options, os, times, net, atomics, exitprocs], - nat_traversal/[miniupnpc, natpmp], - json_serialization/std/net, - results +import std/[options, net, os, sequtils, json] +import results import pkg/chronos import pkg/chronicles import pkg/libp2p +import pkg/libp2p/services/autorelayservice +import pkg/libp2p/protocols/connectivity/autonatv2/service +import pkg/libp2p/protocols/connectivity/relay/relay as relayProtocol +import pkg/libp2p/protocols/connectivity/dcutr/client as dcutrClientModule +import pkg/libp2p/protocols/connectivity/dcutr/server as dcutrServerModule +import pkg/libp2p/wire import ./utils import ./utils/natutils import ./utils/addrutils +import ./discovery -const - UPNP_TIMEOUT = 200 # ms - PORT_MAPPING_INTERVAL = 20 * 60 # seconds - NATPMP_LIFETIME = 60 * 60 # in seconds, must be longer than PORT_MAPPING_INTERVAL - -type PortMappings* = object - internalTcpPort: Port - externalTcpPort: Port - internalUdpPort: Port - externalUdpPort: Port - description: string - -type PortMappingArgs = - tuple[strategy: NatStrategy, tcpPort, udpPort: Port, description: string] +logScope: + topics = "nat" type NatConfig* = object case hasExtIp*: bool of true: extIp*: IpAddress of false: nat*: NatStrategy -var - upnp {.threadvar.}: Miniupnp - npmp {.threadvar.}: NatPmp - strategy = NatStrategy.NatNone - natClosed: Atomic[bool] - extIp: Option[IpAddress] - activeMappings: seq[PortMappings] - natThreads: seq[Thread[PortMappingArgs]] = @[] +type PortMapping* = object + tcpMappingId: cint + udpMappingId: cint + activeMappingProtocol*: MappingProtocol + activeTcpPort*: Port + activeUdpPort*: Port -logScope: - topics = "nat" +type NatPortMapper* = ref object of RootObj + natConfig*: NatConfig + tcpPort*: Port + discoveryPort*: Port + discoverTimeout*: int + mappingTimeout*: int + recheckPeriod*: int + portMapping*: Option[PortMapping] + plumInitialized: bool + closed: bool -type PrefSrcStatus = enum - NoRoutingInfo - PrefSrcIsPublic - PrefSrcIsPrivate - BindAddressIsPublic - BindAddressIsPrivate +# libplum seams, extracted as methods so tests can override them without I/O. -## Also does threadvar initialisation. -## Must be called before redirectPorts() in each thread. -proc getExternalIP*(natStrategy: NatStrategy, quiet = false): Option[IpAddress] = - var externalIP: IpAddress +method initPlum*(m: NatPortMapper): Result[void, string] {.base, gcsafe.} = + let plumLogLevel = + if getEnv("DEBUG") == "1": PlumLogLevel.Verbose else: PlumLogLevel.None + init( + logLevel = plumLogLevel, + discoverTimeout = m.discoverTimeout.int32, + mappingTimeout = m.mappingTimeout.int32, + recheckPeriod = m.recheckPeriod.int32, + ) - if natStrategy == NatStrategy.NatAny or natStrategy == NatStrategy.NatUpnp: - if upnp == nil: - upnp = newMiniupnp() +method createMappingFor*( + m: NatPortMapper, protocol: PlumProtocol, port: uint16 +): Future[Result[MappingResult, string]] {. + base, async: (raises: [CancelledError]), gcsafe +.} = + await createMapping(protocol, port, port) - upnp.discoverDelay = UPNP_TIMEOUT - let dres = upnp.discover() - if dres.isErr: - debug "UPnP", msg = dres.error - else: - var - msg: cstring - canContinue = true - case upnp.selectIGD() - of IGDNotFound: - msg = "Internet Gateway Device not found. Giving up." - canContinue = false - of IGDFound: - msg = "Internet Gateway Device found." - of IGDNotConnected: - msg = "Internet Gateway Device found but it's not connected. Trying anyway." - of NotAnIGD: - msg = - "Some device found, but it's not recognised as an Internet Gateway Device. Trying anyway." - of IGDIpNotRoutable: - msg = - "Internet Gateway Device found and is connected, but with a reserved or non-routable IP. Trying anyway." - if not quiet: - debug "UPnP", msg - if canContinue: - let ires = upnp.externalIPAddress() - if ires.isErr: - debug "UPnP", msg = ires.error - else: - # if we got this far, UPnP is working and we don't need to try NAT-PMP - try: - externalIP = parseIpAddress(ires.value) - strategy = NatStrategy.NatUpnp - return some(externalIP) - except ValueError as e: - error "parseIpAddress() exception", err = e.msg - return +method destroyMappingFor*(m: NatPortMapper, id: cint) {.base, gcsafe.} = + destroyMapping(id) - if natStrategy == NatStrategy.NatAny or natStrategy == NatStrategy.NatPmp: - if npmp == nil: - npmp = newNatPmp() - let nres = npmp.init() - if nres.isErr: - debug "NAT-PMP", msg = nres.error - else: - let nires = npmp.externalIPAddress() - if nires.isErr: - debug "NAT-PMP", msg = nires.error - else: - try: - externalIP = parseIpAddress($(nires.value)) - strategy = NatStrategy.NatPmp - return some(externalIP) - except ValueError as e: - error "parseIpAddress() exception", err = e.msg - return +method hasLivePortMapping*(m: NatPortMapper): bool {.base, gcsafe.} = + ## True only when a mapping was created AND both the TCP and UDP mappings are + ## still live in the router. + if m.portMapping.isNone: + return false -# This queries the routing table to get the "preferred source" attribute and -# checks if it's a public IP. If so, then it's our public IP. -# -# Further more, we check if the bind address (user provided, or a "0.0.0.0" -# default) is a public IP. That's a long shot, because code paths involving a -# user-provided bind address are not supposed to get here. -proc getRoutePrefSrc(bindIp: IpAddress): (Option[IpAddress], PrefSrcStatus) = - let bindAddress = initTAddress(bindIp, Port(0)) + let pm = m.portMapping.get + hasMapping(pm.tcpMappingId) and hasMapping(pm.udpMappingId) - if bindAddress.isAnyLocal(): - let ip = getRouteIpv4() - if ip.isErr(): - # No route was found, log error and continue without IP. - error "No routable IP address found, check your network connection", - error = ip.error - return (none(IpAddress), NoRoutingInfo) - elif ip.get().isGlobalUnicast(): - return (some(ip.get()), PrefSrcIsPublic) - else: - return (none(IpAddress), PrefSrcIsPrivate) - elif bindAddress.isGlobalUnicast(): - return (some(bindIp), BindAddressIsPublic) - else: - return (none(IpAddress), BindAddressIsPrivate) +proc resetMappings(m: NatPortMapper) = + if m.portMapping.isSome: + let pm = m.portMapping.get + m.destroyMappingFor(pm.tcpMappingId) + m.destroyMappingFor(pm.udpMappingId) + m.portMapping = none(PortMapping) -# Try to detect a public IP assigned to this host, before trying NAT traversal. -proc getPublicRoutePrefSrcOrExternalIP*( - natStrategy: NatStrategy, bindIp: IpAddress, quiet = true -): Option[IpAddress] = - let (prefSrcIp, prefSrcStatus) = getRoutePrefSrc(bindIp) +method mapNatPorts*( + m: NatPortMapper +): Future[Option[(Port, Port, MappingProtocol)]] {. + async: (raises: [CancelledError]), base, gcsafe +.} = + if m.closed or m.natConfig.hasExtIp: + return none((Port, Port, MappingProtocol)) - case prefSrcStatus - of NoRoutingInfo, PrefSrcIsPublic, BindAddressIsPublic: - return prefSrcIp - of PrefSrcIsPrivate, BindAddressIsPrivate: - let extIp = getExternalIP(natStrategy, quiet) - if extIp.isSome: - return some(extIp.get) + # If both mappings are still live, return the stored ports without recreating. + if m.hasLivePortMapping(): + let pm = m.portMapping.get + return some((pm.activeTcpPort, pm.activeUdpPort, pm.activeMappingProtocol)) -proc doPortMapping( - strategy: NatStrategy, tcpPort, udpPort: Port, description: string -): Option[(Port, Port)] {.gcsafe.} = - var - extTcpPort: Port - extUdpPort: Port + if not m.plumInitialized: + let res = m.initPlum() + if res.isErr: + warn "Failed to initialize plum", msg = res.error + return none((Port, Port, MappingProtocol)) + m.plumInitialized = true - if strategy == NatStrategy.NatUpnp: - for t in [(tcpPort, UPNPProtocol.TCP), (udpPort, UPNPProtocol.UDP)]: - let - (port, protocol) = t - pmres = upnp.addPortMapping( - externalPort = $port, - protocol = protocol, - internalHost = upnp.lanAddr, - internalPort = $port, - desc = description, - leaseDuration = 0, - ) - if pmres.isErr: - error "UPnP port mapping", msg = pmres.error, port - return - else: - # let's check it - let cres = - upnp.getSpecificPortMapping(externalPort = $port, protocol = protocol) - if cres.isErr: - warn "UPnP port mapping check failed. Assuming the check itself is broken and the port mapping was done.", - msg = cres.error + # If there is only one mapping, something went wrong somewhere + # so we delete the mappings to recreate them. + m.resetMappings() - info "UPnP: added port mapping", - externalPort = port, internalPort = port, protocol = protocol - case protocol - of UPNPProtocol.TCP: - extTcpPort = port - of UPNPProtocol.UDP: - extUdpPort = port - elif strategy == NatStrategy.NatPmp: - for t in [(tcpPort, NatPmpProtocol.TCP), (udpPort, NatPmpProtocol.UDP)]: - let - (port, protocol) = t - pmres = npmp.addPortMapping( - eport = port.cushort, - iport = port.cushort, - protocol = protocol, - lifetime = NATPMP_LIFETIME, - ) - if pmres.isErr: - error "NAT-PMP port mapping", msg = pmres.error, port - return - else: - let extPort = Port(pmres.value) - info "NAT-PMP: added port mapping", - externalPort = extPort, internalPort = port, protocol = protocol - case protocol - of NatPmpProtocol.TCP: - extTcpPort = extPort - of NatPmpProtocol.UDP: - extUdpPort = extPort - return some((extTcpPort, extUdpPort)) + let tcpRes = await m.createMappingFor(TCP, m.tcpPort.uint16) -proc repeatPortMapping(args: PortMappingArgs) {.thread, raises: [ValueError].} = - ignoreSignalsInThread() - let - (strategy, tcpPort, udpPort, description) = args - interval = initDuration(seconds = PORT_MAPPING_INTERVAL) - sleepDuration = 1_000 # in ms, also the maximum delay after pressing Ctrl-C + if m.closed: + # Double check in case the node is stopping + return none((Port, Port, MappingProtocol)) - var lastUpdate = now() + if tcpRes.isErr: + warn "TCP port mapping failed", msg = tcpRes.error + return none((Port, Port, MappingProtocol)) - # We can't use copies of Miniupnp and NatPmp objects in this thread, because they share - # C pointers with other instances that have already been garbage collected, so - # we use threadvars instead and initialise them again with getExternalIP(), - # even though we don't need the external IP's value. - let ipres = getExternalIP(strategy, quiet = true) - if ipres.isSome: - while natClosed.load() == false: - let currTime = now() - if currTime >= (lastUpdate + interval): - discard doPortMapping(strategy, tcpPort, udpPort, description) - lastUpdate = currTime + let udpRes = await m.createMappingFor(UDP, m.discoveryPort.uint16) - sleep(sleepDuration) + if m.closed: + # Double check in case the node is stopping + return none((Port, Port, MappingProtocol)) -proc stopNatThreads() {.noconv.} = - # stop the thread - debug "Stopping NAT port mapping renewal threads" - try: - natClosed.store(true) - joinThreads(natThreads) - except Exception as exc: - warn "Failed to stop NAT port mapping renewal thread", exc = exc.msg + if udpRes.isErr: + warn "UDP port mapping failed", msg = udpRes.error + m.destroyMappingFor(tcpRes.value.id) + return none((Port, Port, MappingProtocol)) - # delete our port mappings - - # FIXME: if the initial port mapping failed because it already existed for the - # required external port, we should not delete it. It might have been set up - # by another program. - - # In Windows, a new thread is created for the signal handler, so we need to - # initialise our threadvars again. - - let ipres = getExternalIP(strategy, quiet = true) - if ipres.isSome: - if strategy == NatStrategy.NatUpnp: - for entry in activeMappings: - for t in [ - (entry.externalTcpPort, entry.internalTcpPort, UPNPProtocol.TCP), - (entry.externalUdpPort, entry.internalUdpPort, UPNPProtocol.UDP), - ]: - let - (eport, iport, protocol) = t - pmres = upnp.deletePortMapping(externalPort = $eport, protocol = protocol) - if pmres.isErr: - error "UPnP port mapping deletion", msg = pmres.error - else: - debug "UPnP: deleted port mapping", - externalPort = eport, internalPort = iport, protocol = protocol - elif strategy == NatStrategy.NatPmp: - for entry in activeMappings: - for t in [ - (entry.externalTcpPort, entry.internalTcpPort, NatPmpProtocol.TCP), - (entry.externalUdpPort, entry.internalUdpPort, NatPmpProtocol.UDP), - ]: - let - (eport, iport, protocol) = t - pmres = npmp.deletePortMapping( - eport = eport.cushort, iport = iport.cushort, protocol = protocol - ) - if pmres.isErr: - error "NAT-PMP port mapping deletion", msg = pmres.error - else: - debug "NAT-PMP: deleted port mapping", - externalPort = eport, internalPort = iport, protocol = protocol - -proc redirectPorts*( - strategy: NatStrategy, tcpPort, udpPort: Port, description: string -): Option[(Port, Port)] = - result = doPortMapping(strategy, tcpPort, udpPort, description) - if result.isSome: - let (externalTcpPort, externalUdpPort) = result.get() - # needed by NAT-PMP on port mapping deletion - # Port mapping works. Let's launch a thread that repeats it, in case the - # NAT-PMP lease expires or the router is rebooted and forgets all about - # these mappings. - activeMappings.add( - PortMappings( - internalTcpPort: tcpPort, - externalTcpPort: externalTcpPort, - internalUdpPort: udpPort, - externalUdpPort: externalUdpPort, - description: description, - ) + m.portMapping = some( + PortMapping( + tcpMappingId: tcpRes.value.id, + udpMappingId: udpRes.value.id, + activeMappingProtocol: tcpRes.value.mapping.mappingProtocol, + activeTcpPort: Port(tcpRes.value.mapping.externalPort), + activeUdpPort: Port(udpRes.value.mapping.externalPort), ) + ) + + let pm = m.portMapping.get + some((pm.activeTcpPort, pm.activeUdpPort, pm.activeMappingProtocol)) + +proc close*(m: NatPortMapper) = + m.resetMappings() + + if m.plumInitialized: + discard cleanup() + m.plumInitialized = false + +proc stop*(m: NatPortMapper) = + ## Ensure that any future AutoNAT callback does not re-initialize libplum. + m.closed = true + m.close() + +method handleNatStatus*( + m: NatPortMapper, + networkReachability: NetworkReachability, + dialBackAddr: Opt[MultiAddress], + discoveryPort: Port, + discovery: Discovery, + switch: Switch, + autoRelayService: AutoRelayService, +) {.async: (raises: [CancelledError]), base, gcsafe.} = + if m.closed: + return + + case networkReachability + of Unknown: + discard + of Reachable: + if dialBackAddr.isSome: + if autoRelayService.isRunning: + await autoRelayService.stop(switch) + debug "AutoRelayService stopped" + + discovery.protocol.clientMode = false + + # Here we don't rely on the port mapping because we consider + # that port mapped is the same as the discovery port. + # This can be wrong for PCP but it is an accepted limitation + discovery.announceDirectAddrs(@[dialBackAddr.get], udpPort = discoveryPort) + else: + warn "Empty dialback address in AutoNat when node is Reachable" + of NotReachable: + discovery.protocol.clientMode = true + + if not autoRelayService.isRunning and discovery.announceAddrs.len > 0: + # Remove any announced addresses, they will be replaced. + # If the relay is running, the addresses will be updated on reservation. + discovery.announceDirectAddrs(@[], udpPort = discoveryPort) + + if m.hasLivePortMapping(): + # The mapping is still live but the node is not reachable: keep it and let + # the relay take over. A dead mapping falls through to be recreated. + debug "Not Reachable with live port mapping, keeping it and starting relay if not started" + else: + debug "Node is not reachable trying port mapping now" + + let maybePorts = await m.mapNatPorts() + + if m.closed: + # Double check in case the node is stopping + return + + if maybePorts.isSome: + let (tcpPort, udpPort, protocol) = maybePorts.get() + + info "Port mapping created successfully", tcpPort, udpPort, protocol + + # The announce happens once AutoNAT confirms Reachable. + + return + else: + # In case of failure, close the port mapping in order to rerun discover + # on the next iteration + m.close() + + if not autoRelayService.isRunning: + debug "No port mapping found let's start autorelay" + + await autoRelayService.start(switch) + debug "AutoRelayService started" + +proc reachabilityStr*(autonat: Option[AutonatV2Service]): string = + if autonat.isSome: + $autonat.get.networkReachability + else: + "Unknown" + +proc portMappingStr*(natMapper: Option[NatPortMapper]): string = + if natMapper.isNone or natMapper.get.portMapping.isNone: + return "none" + case natMapper.get.portMapping.get.activeMappingProtocol + of MappingProtocol.UPnP: "upnp" + of MappingProtocol.NatPmp: "pmp" + of MappingProtocol.PCP: "pcp" + of MappingProtocol.Direct: "direct" + of MappingProtocol.Unknown: "none" + +proc peerConnections*(switch: Switch): JsonNode = + result = newJArray() + for peerId, muxers in switch.connManager.getConnections(): + let entry = newJObject() + entry["peerId"] = newJString($peerId) + entry["direct"] = newJBool(muxers.anyIt(not isRelayed(it.connection))) + result.add(entry) + +proc findReachableNodes*(bootstrapNodes: seq[SignedPeerRecord]): seq[SignedPeerRecord] = + ## Returns the list of nodes known to be directly reachable. + ## Currently returns bootstrap nodes. In the future, any network participant + ## confirmed reachable by AutoNAT could be included. + bootstrapNodes + +proc announceRelayReservation*( + discovery: Discovery, addresses: seq[MultiAddress] +) {.gcsafe.} = + ## Announce the publicly dialable circuit addresses from a relay reservation. + ## A reservation response can also carry loopback/private addresses, which a + ## remote peer can never dial, so they are dropped. If none are public, the + ## previous announce is kept untouched. + let publicAddrs = addresses.filterIt(it.hasPublicRelayTransport()) + if publicAddrs.len == 0: + warn "Relay reservation has no publicly dialable address, keeping previous announce", + addresses + return + info "Relay reservation updated", addresses = publicAddrs + # relay addresses are for download traffic only, not DHT routing + discovery.announceRelayAddrs(publicAddrs) + +# Hole punching logic below is adapted from libp2p's HPService +# (libp2p/services/hpservice.nim). HPService cannot be used directly because it +# depends on AutoNAT v1 and starts the relay immediately on NotReachable, +# bypassing the UPnP step. + +proc tryStartingDirectConn( + switch: Switch, peerId: PeerId +): Future[bool] {.async: (raises: [CancelledError]).} = + proc tryConnect( + address: MultiAddress + ): Future[bool] {.async: (raises: [DialFailedError, CancelledError]).} = + debug "Trying to create direct connection", peerId, address + await switch.connect(peerId, @[address], true, false) + debug "Direct connection created." + return true + + await sleepAsync(500.milliseconds) # wait for AddressBook to be populated + for address in switch.peerStore[AddressBook][peerId]: try: - natThreads.add(Thread[PortMappingArgs]()) - natThreads[^1].createThread( - repeatPortMapping, (strategy, externalTcpPort, externalUdpPort, description) - ) - # atexit() in disguise - if natThreads.len == 1: - # we should register the thread termination function only once - addExitProc(stopNatThreads) - except Exception as exc: - warn "Failed to create NAT port mapping renewal thread", exc = exc.msg + let isRelayedAddr = address.contains(multiCodec("p2p-circuit")) + if not isRelayedAddr.get(false) and address.isPublicMA(): + return await tryConnect(address) + except CancelledError as exc: + raise exc + except CatchableError as err: + debug "Failed to create direct connection.", description = err.msg + continue + return false -proc setupNat*( - natStrategy: NatStrategy, tcpPort, udpPort: Port, clientId: string -): tuple[ip: Option[IpAddress], tcpPort, udpPort: Option[Port]] = - ## Setup NAT port mapping and get external IP address. - ## If any of this fails, we don't return any IP address but do return the - ## original ports as best effort. - ## TODO: Allow for tcp or udp port mapping to be optional. - if extIp.isNone: - extIp = getExternalIP(natStrategy) - if extIp.isSome: - let ip = extIp.get - let extPorts = ( - {.gcsafe.}: - redirectPorts( - strategy, tcpPort = tcpPort, udpPort = udpPort, description = clientId - ) - ) - if extPorts.isSome: - let (extTcpPort, extUdpPort) = extPorts.get() - (ip: some(ip), tcpPort: some(extTcpPort), udpPort: some(extUdpPort)) - else: - warn "UPnP/NAT-PMP available but port forwarding failed" - (ip: none(IpAddress), tcpPort: some(tcpPort), udpPort: some(udpPort)) - else: - warn "UPnP/NAT-PMP not available" - (ip: none(IpAddress), tcpPort: some(tcpPort), udpPort: some(udpPort)) +proc closeRelayConn(relayedConn: Connection) {.async: (raises: [CancelledError]).} = + await sleepAsync(2000.milliseconds) # grace period before closing relayed connection + await relayedConn.close() -proc setupAddress*( - natConfig: NatConfig, bindIp: IpAddress, tcpPort, udpPort: Port, clientId: string -): tuple[ip: Option[IpAddress], tcpPort, udpPort: Option[Port]] {.gcsafe.} = - ## Set-up of the external address via any of the ways as configured in - ## `NatConfig`. In case all fails an error is logged and the bind ports are - ## selected also as external ports, as best effort and in hope that the - ## external IP can be figured out by other means at a later stage. - ## TODO: Allow for tcp or udp bind ports to be optional. +proc holePunchIfRelayed*( + switch: Switch, peerId: PeerId +) {.async: (raises: [CancelledError]).} = + ## Attempts to establish a direct connection when a peer connected via relay. + ## First tries a direct TCP connect (if the peer's address is known and public), + ## then falls back to dcutr simultaneous-open hole punching. + ## Closes the relay connection once a direct path is established. + let connections = + switch.connManager.getConnections().getOrDefault(peerId).mapIt(it.connection) + if connections.anyIt(not isRelayed(it)): + return + let incomingRelays = connections.filterIt(it.transportDir == Direction.In) + if incomingRelays.len == 0: + return - if natConfig.hasExtIp: - # any required port redirection must be done by hand - return (some(natConfig.extIp), some(tcpPort), some(udpPort)) + let relayedConn = incomingRelays[0] - case natConfig.nat - of NatStrategy.NatAny: - let (prefSrcIp, prefSrcStatus) = getRoutePrefSrc(bindIp) + if await tryStartingDirectConn(switch, peerId): + await closeRelayConn(relayedConn) + return - case prefSrcStatus - of NoRoutingInfo, PrefSrcIsPublic, BindAddressIsPublic: - return (prefSrcIp, some(tcpPort), some(udpPort)) - of PrefSrcIsPrivate, BindAddressIsPrivate: - return setupNat(natConfig.nat, tcpPort, udpPort, clientId) - of NatStrategy.NatNone: - let (prefSrcIp, prefSrcStatus) = getRoutePrefSrc(bindIp) + var natAddrs = switch.peerStore.getMostObservedProtosAndPorts() + if natAddrs.len == 0: + natAddrs = switch.peerInfo.listenAddrs.mapIt(switch.peerStore.guessDialableAddr(it)) + try: + await DcutrClient.new().startSync(switch, peerId, natAddrs) + await closeRelayConn(relayedConn) + except DcutrError as err: + debug "Hole punching failed during dcutr", description = err.msg - case prefSrcStatus - of NoRoutingInfo, PrefSrcIsPublic, BindAddressIsPublic: - return (prefSrcIp, some(tcpPort), some(udpPort)) - of PrefSrcIsPrivate: - error "No public IP address found. Should not use --nat:none option" - return (none(IpAddress), some(tcpPort), some(udpPort)) - of BindAddressIsPrivate: - error "Bind IP is not a public IP address. Should not use --nat:none option" - return (none(IpAddress), some(tcpPort), some(udpPort)) - of NatStrategy.NatUpnp, NatStrategy.NatPmp: - return setupNat(natConfig.nat, tcpPort, udpPort, clientId) +proc setupHolePunching*(switch: Switch): PeerEventHandler = + try: + switch.mount(Dcutr.new(switch)) + except LPError as err: + error "Failed to mount Dcutr protocol", description = err.msg -proc nattedAddress*( - natConfig: NatConfig, addrs: seq[MultiAddress], udpPort: Port -): tuple[libp2p, discovery: seq[MultiAddress]] = - ## Takes a NAT configuration, sequence of multiaddresses and UDP port and returns: - ## - Modified multiaddresses with NAT-mapped addresses for libp2p - ## - Discovery addresses with NAT-mapped UDP ports - - var discoveryAddrs = newSeq[MultiAddress](0) - let newAddrs = addrs.mapIt: - block: - # Extract IP address and port from the multiaddress - let (ipPart, port) = getAddressAndPort(it) - if ipPart.isSome and port.isSome: - # Try to setup NAT mapping for the address - let (newIP, tcp, udp) = - setupAddress(natConfig, ipPart.get, port.get, udpPort, "storage") - if newIP.isSome: - # NAT mapping successful - add discovery address with mapped UDP port - discoveryAddrs.add(getMultiAddrWithIPAndUDPPort(newIP.get, udp.get)) - # Remap original address with NAT IP and TCP port - it.remapAddr(ip = newIP, port = tcp) - else: - # NAT mapping failed - use original address - echo "Failed to get external IP, using original address", it - discoveryAddrs.add(getMultiAddrWithIPAndUDPPort(ipPart.get, udpPort)) - it - else: - # Invalid multiaddress format - return as is - it - (newAddrs, discoveryAddrs) + let handler = proc( + peerId: PeerId, event: PeerEvent + ) {.async: (raises: [CancelledError]).} = + await holePunchIfRelayed(switch, peerId) + switch.addPeerEventHandler(handler, PeerEventKind.Joined) + handler diff --git a/storage/rest/api.nim b/storage/rest/api.nim index 865591fc..ff9ac7f4 100644 --- a/storage/rest/api.nim +++ b/storage/rest/api.nim @@ -23,10 +23,13 @@ import pkg/confutils import pkg/libp2p import pkg/libp2p/routing_record -import pkg/codexdht/discv5/spr as spr +import pkg/libp2p/protocols/connectivity/autonatv2/service +import pkg/libp2p/services/autorelayservice +import pkg/codexdht/discv5/spr import ../logutils import ../node +import ../discovery import ../blocktype import ../storagetypes import ../conf @@ -37,6 +40,7 @@ import ../stores/repostore import ../blockexchange import ../units import ../utils/options +import ../nat import ./coders import ./json @@ -484,10 +488,7 @@ proc initNodeApi(node: StorageNodeRef, conf: StorageConf, router: var RestRouter var headers = buildCorsHeaders("GET", allowedOrigin) try: - without spr =? node.discovery.dhtRecord: - return RestApiResponse.response( - "", status = Http503, contentType = "application/json", headers = headers - ) + let spr = node.discovery.getSpr() if $preferredContentType().get() == "text/plain": return RestApiResponse.response( @@ -530,8 +531,8 @@ proc initNodeApi(node: StorageNodeRef, conf: StorageConf, router: var RestRouter ## to invoke peer discovery, if it succeeds ## the returned addresses will be used to dial ## - ## `addrs` the listening addresses of the peers to dial, which is - ## /ip4/0.0.0.0/tcp/, where port is specified with the + ## `addrs` the listening addresses of the peers to dial, which is + ## /ip4/0.0.0.0/tcp/, where port is specified with the ## `--listen-port` CLI flag. ## var headers = buildCorsHeaders("GET", allowedOrigin) @@ -557,7 +558,14 @@ proc initNodeApi(node: StorageNodeRef, conf: StorageConf, router: var RestRouter return RestApiResponse.error(Http500, "Unknown error dialling peer", headers = headers) -proc initDebugApi(node: StorageNodeRef, conf: StorageConf, router: var RestRouter) = +proc initDebugApi( + node: StorageNodeRef, + conf: StorageConf, + autonat: Option[AutonatV2Service], + autoRelay: Option[AutoRelayService], + natMapper: Option[NatPortMapper], + router: var RestRouter, +) = let allowedOrigin = router.allowedOrigin router.api(MethodGet, "/api/storage/v1/debug/info") do() -> RestApiResponse: @@ -567,16 +575,24 @@ proc initDebugApi(node: StorageNodeRef, conf: StorageConf, router: var RestRoute try: let table = RestRoutingTable.init(node.discovery.protocol.routingTable) + let nodeSpr = node.discovery.getSpr() let json = %*{ "id": $node.switch.peerInfo.peerId, "addrs": node.switch.peerInfo.addrs.mapIt($it), "repo": $conf.dataDir, - "spr": - if node.discovery.dhtRecord.isSome: node.discovery.dhtRecord.get.toURI else: "", + "spr": nodeSpr.toURI, "announceAddresses": node.discovery.announceAddrs, + "dhtAddresses": node.discovery.dhtAddrs, "table": table, "storage": {"version": $storageVersion, "revision": $storageRevision}, + "nat": { + "reachability": reachabilityStr(autonat), + "clientMode": node.discovery.protocol.clientMode, + "relayRunning": autoRelay.isSome and autoRelay.get.isRunning, + "portMapping": portMappingStr(natMapper), + }, + "connections": peerConnections(node.switch), } # return pretty json for human readability @@ -637,12 +653,15 @@ proc initRestApi*( node: StorageNodeRef, conf: StorageConf, repoStore: RepoStore, + autonat: Option[AutonatV2Service], + autoRelay: Option[AutoRelayService], + natMapper: Option[NatPortMapper], corsAllowedOrigin: ?string, ): RestRouter = var router = RestRouter.init(validate, corsAllowedOrigin) initDataApi(node, repoStore, router) initNodeApi(node, conf, router) - initDebugApi(node, conf, router) + initDebugApi(node, conf, autonat, autoRelay, natMapper, router) return router diff --git a/storage/storage.nim b/storage/storage.nim index 678f5f87..2a96d28a 100644 --- a/storage/storage.nim +++ b/storage/storage.nim @@ -17,6 +17,12 @@ import pkg/chronos import pkg/taskpools import pkg/presto import pkg/libp2p +import pkg/libp2p/connmanager +import pkg/libp2p/protocols/connectivity/autonatv2/[service, client] +import pkg/libp2p/protocols/connectivity/relay/client as relayClientModule +import pkg/libp2p/protocols/connectivity/relay/relay as relayModule +import pkg/libp2p/services/autorelayservice +import pkg/libp2p/transports/tcptransport import pkg/confutils import pkg/confutils/defs import pkg/stew/io2 @@ -33,11 +39,11 @@ import ./blockexchange import ./utils/fileutils import ./discovery import ./utils/addrutils -import ./utils/natutils import ./namespaces import ./storagetypes import ./logutils import ./nat +import ./utils/natutils logScope: topics = "storage node" @@ -51,6 +57,12 @@ type repoStore: RepoStore maintenance: BlockMaintainer taskpool: Taskpool + # Expose to make reachability accessible from rest api + autonatService*: Option[AutonatV2Service] + autoRelayService*: Option[AutoRelayService] + natMapper*: Option[NatPortMapper] + holePunchHandler: Option[connmanager.PeerEventHandler] + bootstrapNodes: seq[SignedPeerRecord] isStarted: bool StoragePrivateKey* = libp2p.PrivateKey # alias @@ -76,25 +88,73 @@ proc start*(s: StorageServer) {.async.} = await s.storageNode.switch.start() - let (announceAddrs, discoveryAddrs) = nattedAddress( - s.config.nat, s.storageNode.switch.peerInfo.addrs, s.config.discoveryPort - ) + # Activate SO_REUSEPORT for hole punching in tcptransport.nim. + # Without that, hole punching would use an ephemeral port assigned by the OS. + # NotReachable has nothing to do with AutoNAT Reachability + if s.holePunchHandler.isSome: + for t in s.storageNode.switch.transports: + t.networkReachability = NetworkReachability.NotReachable - var hasPublicAddr = false - for announceAddr in announceAddrs: - let (maybeIp, _) = getAddressAndPort(announceAddr) - if maybeIp.isSome and maybeIp.get.isGlobalUnicast(): - hasPublicAddr = true - break + # When listenPort is 0 the OS assigns a random port. For UDP, the port + # doesn't change so there is no need to update it. + if s.natMapper.isSome and s.config.listenPort == Port(0): + for listenAddr in s.storageNode.switch.peerInfo.listenAddrs: + let maybePort = getTcpPort(listenAddr) + if maybePort.isSome: + s.natMapper.get.tcpPort = maybePort.get + break - if not hasPublicAddr: - warn "Unable to determine a public IP address. This node will only be reachable on a private network." + # The addresses are announced during the start process + # only with extIp because they should be Reachable. + # For other nodes, wait for AutoNat to announce addresses and update SPR. + if s.config.nat.hasExtIp: + if s.storageNode.switch.peerInfo.addrs.len == 0: + raise + newException(StorageError, "extip is set but switch has no listen addresses") - s.storageNode.discovery.updateAnnounceRecord(announceAddrs) - s.storageNode.discovery.updateDhtRecord(discoveryAddrs) + # extip means that we assume the IP is reachable. + # So we just take the first peer addr and remap it with extip to keep the port only. + let announceAddresses = @[ + s.storageNode.switch.peerInfo.addrs[0].remapAddr( + ip = some(s.config.nat.extIp), port = none(Port) + ) + ] + s.storageNode.discovery.announceDirectAddrs( + announceAddresses, udpPort = s.config.discoveryPort + ) + else: + # Other nodes wait for AutoNAT to announce addresses and update SPR. + # They start in client mode to avoid polluting DHT with NotReachable records; + # it will be flipped off once AutoNAT confirms reachability. + s.storageNode.discovery.protocol.clientMode = true await s.storageNode.start() + # Connect to the bootstrap nodes in order to have connected peers + # for Autonat. The dials are run concurrently in case of + # a dead bootstrap node that could timeout. + proc connectBootstrapNode( + spr: SignedPeerRecord + ) {.async: (raises: [CancelledError]).} = + try: + let addrs = spr.data.addresses.mapIt(it.address) + await s.storageNode.switch.connect(spr.data.peerId, addrs) + except CancelledError as exc: + raise exc + except CatchableError as e: + warn "Cannot connect to bootstrap node", error = e.msg + + # noCancel: cancelling allFutures does not cancel the + # connectBootstrapNode futures. + await noCancel allFutures( + findReachableNodes(s.bootstrapNodes).mapIt(connectBootstrapNode(it)) + ) + + # AutoNAT is not in switch.services because we want to start it + # after the bootstrap connections to have connected peers for the first probe. + if s.autonatService.isSome: + await s.autonatService.get.start(s.storageNode.switch) + if s.restServer != nil: s.restServer.start() @@ -107,6 +167,14 @@ proc stop*(s: StorageServer) {.async.} = notice "Stopping Storage node" + if s.natMapper.isSome: + s.natMapper.get.stop() + + if s.holePunchHandler.isSome: + s.storageNode.switch.removePeerEventHandler( + s.holePunchHandler.get, PeerEventKind.Joined + ) + var futures = @[ s.storageNode.switch.stop(), s.storageNode.stop(), @@ -114,6 +182,15 @@ proc stop*(s: StorageServer) {.async.} = s.maintenance.stop(), ] + if s.autoRelayService.isSome and s.autoRelayService.get.isRunning: + proc stopAutoRelay(): Future[void] {.async: (raises: []).} = + await noCancel s.autoRelayService.get.stop(s.storageNode.switch) + + futures.add(stopAutoRelay()) + + if s.autonatService.isSome: + futures.add(s.autonatService.get.stop(s.storageNode.switch)) + if s.restServer != nil: futures.add(s.restServer.stop()) @@ -168,13 +245,26 @@ proc new*( privateKey: StoragePrivateKey, logFile: Option[IoHandle] = IoHandle.none, ): StorageServer = - ## create StorageServer including setting up datastore, repostore, etc + ## create StorageServer including setting up datastore, repostore, etc. + + if err =? config.validateAutonatConfig().errorOption: + raise newException(StorageError, err.msg) + + # Switch let listenMultiAddr = getMultiAddrWithIpAndTcpPort(config.listenIp, config.listenPort) - let switch = SwitchBuilder + let relayClient = RelayClient.new() + let relay: Relay = + if config.isRelayServer: + Relay.new() + else: + relayClient + + var switchBuilder = SwitchBuilder .new() .withPrivateKey(privateKey) - .withAddresses(@[listenMultiAddr], enableWildcardResolver = true) + .withAddresses(@[listenMultiAddr]) + .withWildcardResolver() .withIdentifyPusher(false) .withRng(random.Rng.instance().libp2pRng) .withNoise() @@ -182,11 +272,82 @@ proc new*( .withMaxConnections(config.maxPeers) .withAgentVersion(config.agentString) .withSignedPeerRecord(true) + .withCircuitRelay(relay) + + let bootstrapNodes = + if config.noBootstrapNode: + # Sanity checks that the user isn't doing anything funny. + if config.bootstrapNodes.len > 0: + error "Cannot specify bootstrap nodes when using no-bootstrap flag" + raise newException( + ValueError, "Cannot specify bootstrap nodes when using no-bootstrap flag" + ) + + warn "Node has been marked with --no-bootstrap-node and will NOT be bootstrapped" + seq[SignedPeerRecord](@[]) + elif config.bootstrapNodes.len > 0: + warn "Overriding network preset using custom bootstrap nodes", + nodes = config.bootstrapNodes + config.bootstrapNodes + else: + info "Bootstrapping node using a predefined network", network = $config.network + config.network.bootstrapNodes + + var autonatConfig = none(AutonatV2ServiceConfig) + if config.autonatServer: + info "AutoNAT server enabled" + switchBuilder = switchBuilder.withAutonatV2Server() + elif not config.nat.hasExtIp: + info "AutoNAT client enabled", + scheduleInterval = config.natScheduleInterval, + numPeersToAsk = config.natNumPeersToAsk, + maxQueueSize = config.natMaxQueueSize, + minConfidence = config.natMinConfidence + autonatConfig = some( + AutonatV2ServiceConfig.new( + scheduleInterval = Opt.some(config.natScheduleInterval), + askNewConnectedPeers = false, + numPeersToAsk = config.natNumPeersToAsk, + maxQueueSize = config.natMaxQueueSize, + minConfidence = config.natMinConfidence, + enableDialableCandidates = true, + ) + ) + + let observedAddrMinCount = min(config.natObservedAddrMinCount, bootstrapNodes.len) + switchBuilder = switchBuilder.withObservedAddrManager( + ObservedAddrManager.new(minCount = observedAddrMinCount) + ) + # libp2p keeps the private address in peerInfo.addrs. + # Since Autonat V2 uses the observed public address, + # we can filter the private addresses to keep only the dialable + # addresses. + switchBuilder = switchBuilder.withAddressPolicy(dialableAddressPolicy) + + let switch = switchBuilder .withTcpTransport({ServerFlags.ReuseAddr, ServerFlags.TcpNoDelay}) .build() var taskPool: Taskpool + # AutoNAT's first reachability probe fires immediately on start. + # Wired via withAutonatV2 it lands in switch.services and runs at switch.start, + # before bootstrap, on an empty peer set. + # We build and own it here so we can start it ourselves after bootstrap, + # with the bootstrap peers connected. + let autonatService: Option[AutonatV2Service] = + if autonatConfig.isSome: + let client = AutonatV2Client.new(switch.rng) + client.setup(switch) + switch.mount(client) + let service = AutonatV2Service.new(switch.rng, client, autonatConfig.get) + service.setup(switch) + some(service) + else: + none(AutonatV2Service) + + # Storage infrastructure + try: if config.numThreads == ThreadCount(0): taskPool = Taskpool.new(numThreads = min(countProcessors(), 16)) @@ -211,34 +372,16 @@ proc new*( error "Failed to initialize discovery datastore", path = providersPath, err = discoveryStoreRes.error.msg - let bootstrapNodes = - if config.noBootstrapNode: - # Sanity checks that the user isn't doing anything funny. - if config.bootstrapNodes.len > 0: - error "Cannot specify bootstrap nodes when using no-bootstrap flag" - raise newException( - ValueError, "Cannot specify bootstrap nodes when using no-bootstrap flag" - ) - - warn "Node has been marked with --no-bootstrap-node and will NOT be bootstrapped" - seq[SignedPeerRecord](@[]) - elif config.bootstrapNodes.len > 0: - warn "Overriding network preset using custom bootstrap nodes", - nodes = config.bootstrapNodes - config.bootstrapNodes - else: - info "Bootstrapping node using a predefined network", network = $config.network - config.network.bootstrapNodes - let discoveryStore = Datastore(discoveryStoreRes.expect("Should create discovery datastore!")) discovery = Discovery.new( switch.peerInfo.privateKey, - announceAddrs = @[listenMultiAddr], + announceAddrs = @[], bindPort = config.discoveryPort, bootstrapNodes = bootstrapNodes, + discoveryPort = config.discoveryPort, store = discoveryStore, ) @@ -299,21 +442,68 @@ proc new*( taskPool = taskPool, ) + switch.mount(network) + switch.mount(manifestProto) + + # NAT services + var natMapper: Option[NatPortMapper] + var autoRelayService: Option[AutoRelayService] + var holePunchHandler: Option[connmanager.PeerEventHandler] + + if autonatService.isSome: + let relayService = AutoRelayService.new( + maxNumRelays = config.natMaxRelays, + client = relayClient, + onReservation = proc(addresses: seq[MultiAddress]) {.gcsafe, raises: [].} = + discovery.announceRelayReservation(addresses), + rng = random.Rng.instance().libp2pRng, + ) + + relayService.setup(switch) + autoRelayService = some(relayService) + + natMapper = some( + NatPortMapper( + natConfig: config.nat, + tcpPort: config.listenPort, + discoveryPort: config.discoveryPort, + discoverTimeout: config.natPortMappingDiscoverTimeout, + mappingTimeout: config.natPortMappingTimeout, + recheckPeriod: config.natPortMappingRecheckPeriod, + ) + ) + + autonatService.get.setStatusAndConfidenceHandler( + proc( + networkReachability: NetworkReachability, + confidence: Opt[float], + addrs: Opt[MultiAddress], + ) {.async: (raises: [CancelledError]).} = + debug "AutoNAT status", reachability = networkReachability, confidence + await natMapper.get.handleNatStatus( + networkReachability, addrs, config.discoveryPort, discovery, switch, + relayService, + ) + ) + + holePunchHandler = some(setupHolePunching(switch)) + + # REST server var restServer: RestServerRef = nil if config.apiBindAddress.isSome: restServer = RestServerRef .new( - storageNode.initRestApi(config, repoStore, config.apiCorsAllowedOrigin), + storageNode.initRestApi( + config, repoStore, autonatService, autoRelayService, natMapper, + config.apiCorsAllowedOrigin, + ), initTAddress(config.apiBindAddress.get(), config.apiPort), bufferSize = (1024 * 64), maxRequestBodySize = int.high, ) .expect("Should create rest server!") - switch.mount(network) - switch.mount(manifestProto) - StorageServer( config: config, storageNode: storageNode, @@ -322,4 +512,9 @@ proc new*( maintenance: maintenance, taskPool: taskPool, logFile: logFile, + autonatService: autonatService, + autoRelayService: autoRelayService, + natMapper: natMapper, + holePunchHandler: holePunchHandler, + bootstrapNodes: bootstrapNodes, ) diff --git a/storage/utils/addrutils.nim b/storage/utils/addrutils.nim index 31570e06..add58d40 100644 --- a/storage/utils/addrutils.nim +++ b/storage/utils/addrutils.nim @@ -14,15 +14,18 @@ import std/strutils import std/options import pkg/libp2p +import pkg/libp2p/wire import pkg/stew/endians2 func remapAddr*( address: MultiAddress, ip: Option[IpAddress] = IpAddress.none, port: Option[Port] = Port.none, + protocol: Option[string] = string.none, ): MultiAddress = - ## Remap addresses to new IP and/or Port + ## Remap addresses to new IP, port, and/or transport protocol (e.g. "tcp" → "udp") ## + ## Assumes a /ip4|ip6//tcp|udp/ address: anything else crashes (Defect). var parts = ($address).split("/") @@ -32,6 +35,12 @@ func remapAddr*( else: parts[2] + parts[3] = + if protocol.isSome: + protocol.get + else: + parts[3] + parts[4] = if port.isSome: $port.get @@ -40,22 +49,35 @@ func remapAddr*( MultiAddress.init(parts.join("/")).expect("Should construct multiaddress") -proc getMultiAddrWithIPAndUDPPort*(ip: IpAddress, port: Port): MultiAddress = - ## Creates a MultiAddress with the specified IP address and UDP port - ## - ## Parameters: - ## - ip: A valid IP address (IPv4 or IPv6) - ## - port: The UDP port number - ## - ## Returns: - ## A MultiAddress in the format "/ip4/
/udp/" or "/ip6/
/udp/" +func getTcpPort*(ma: MultiAddress): Option[Port] = + ## Extracts the TCP port from a multiaddress; none when there is no TCP part. + let tcpPart = ma[multiCodec("tcp")] + if tcpPart.isErr: + return Port.none + let portBytes = tcpPart.get().protoArgument() + if portBytes.isErr or portBytes.get().len != 2: + return Port.none + some(Port(fromBytesBE(uint16, portBytes.get()))) - let ipFamily = if ip.family == IpAddressFamily.IPv4: "/ip4/" else: "/ip6/" - return MultiAddress.init(ipFamily & $ip & "/udp/" & $port).expect("valid multiaddr") +proc hasPublicRelayTransport*(ma: MultiAddress): bool = + ## True when ``ma`` is a circuit address whose relay is publicly dialable. + let relayWireStr = ($ma).split("/p2p/")[0] + let relayWireAddr = MultiAddress.init(relayWireStr).valueOr: + return false + relayWireAddr.isPublicMA() + +proc dialableAddressPolicy*(ma: MultiAddress): bool {.gcsafe, raises: [].} = + # Use with switchBuilder.withAddressPolicy. + # Filter the peerInfo.addrs updated by libp2p without + # declaring another address mapper. + if ma.isCircuitRelayMA(): + ma.hasPublicRelayTransport() + else: + ma.isPublicMA() proc getMultiAddrWithIpAndTcpPort*(ip: IpAddress, port: Port): MultiAddress = ## Creates a MultiAddress with the specified IP address and TCP port - ## + ## ## Parameters: ## - ip: A valid IP address (IPv4 or IPv6) ## - port: The TCP port number @@ -67,38 +89,3 @@ proc getMultiAddrWithIpAndTcpPort*(ip: IpAddress, port: Port): MultiAddress = return MultiAddress.init(ipFamily & $ip & "/tcp/" & $port).expect( "Failed to construct multiaddress with IP and TCP port" ) - -proc getAddressAndPort*( - ma: MultiAddress -): tuple[ip: Option[IpAddress], port: Option[Port]] = - try: - # Try IPv4 first - let ipv4Result = ma[multiCodec("ip4")] - let ip = - if ipv4Result.isOk: - let ipBytes = ipv4Result.get().protoArgument().expect("Invalid IPv4 format") - let ipArray = [ipBytes[0], ipBytes[1], ipBytes[2], ipBytes[3]] - some(IpAddress(family: IPv4, address_v4: ipArray)) - else: - # Try IPv6 if IPv4 not found - let ipv6Result = ma[multiCodec("ip6")] - if ipv6Result.isOk: - let ipBytes = ipv6Result.get().protoArgument().expect("Invalid IPv6 format") - var ipArray: array[16, byte] - for i in 0 .. 15: - ipArray[i] = ipBytes[i] - some(IpAddress(family: IPv6, address_v6: ipArray)) - else: - none(IpAddress) - - # Get TCP Port - let portResult = ma[multiCodec("tcp")] - let port = - if portResult.isOk: - let portBytes = portResult.get().protoArgument().expect("Invalid port format") - some(Port(fromBytesBE(uint16, portBytes))) - else: - none(Port) - (ip: ip, port: port) - except Exception: - (ip: none(IpAddress), port: none(Port)) diff --git a/storage/utils/natutils.nim b/storage/utils/natutils.nim index 45ad7589..82d0edc8 100644 --- a/storage/utils/natutils.nim +++ b/storage/utils/natutils.nim @@ -1,66 +1,11 @@ {.push raises: [].} -import std/[net, tables, hashes], pkg/results, chronos, chronicles +import std/[options, net] +import results +import libplum/plum +import libplum/libplum -import pkg/libp2p +export plum, libplum, results, options, net type NatStrategy* = enum - NatAny - NatUpnp - NatPmp - NatNone - -type IpLimits* = object - limit*: uint - ips: Table[IpAddress, uint] - -func hash*(ip: IpAddress): Hash = - case ip.family - of IpAddressFamily.IPv6: - hash(ip.address_v6) - of IpAddressFamily.IPv4: - hash(ip.address_v4) - -func inc*(ipLimits: var IpLimits, ip: IpAddress): bool = - let val = ipLimits.ips.getOrDefault(ip, 0) - if val < ipLimits.limit: - ipLimits.ips[ip] = val + 1 - true - else: - false - -func dec*(ipLimits: var IpLimits, ip: IpAddress) = - let val = ipLimits.ips.getOrDefault(ip, 0) - if val == 1: - ipLimits.ips.del(ip) - elif val > 1: - ipLimits.ips[ip] = val - 1 - -func isGlobalUnicast*(address: TransportAddress): bool = - if address.isGlobal() and address.isUnicast(): true else: false - -func isGlobalUnicast*(address: IpAddress): bool = - let a = initTAddress(address, Port(0)) - a.isGlobalUnicast() - -proc getRouteIpv4*(): Result[IpAddress, cstring] = - # Avoiding Exception with initTAddress and can't make it work with static. - # Note: `publicAddress` is only used an "example" IP to find the best route, - # no data is send over the network to this IP! - let - publicAddress = TransportAddress( - family: AddressFamily.IPv4, address_v4: [1'u8, 1, 1, 1], port: Port(0) - ) - route = getBestRoute(publicAddress) - - if route.source.isUnspecified(): - err("No best ipv4 route found") - else: - let ip = - try: - route.source.address() - except ValueError as e: - # This should not occur really. - error "Address conversion error", exception = e.name, msg = e.msg - return err("Invalid IP address") - ok(ip) + NatAuto diff --git a/tests/imports.nim b/tests/imports.nim index fbe642fc..19ab608a 100644 --- a/tests/imports.nim +++ b/tests/imports.nim @@ -2,17 +2,20 @@ import std/macros import std/os import std/strutils -macro importTests*(dir: static string): untyped = - ## imports all files in the specified directory whose filename - ## starts with "test" and ends in ".nim" +macro importTests*(dir: static string, exclude: static string): untyped = + ## imports every test*.nim file under `dir` (recursively). + ## `exclude`, when non-empty, skips files whose path contains it. let imports = newStmtList() for file in walkDirRec(dir): let (_, name, ext) = splitFile(file) - if name.startsWith("test") and ext == ".nim": - imports.add( - quote do: - import `file` - ) + if not (name.startsWith("test") and ext == ".nim"): + continue + if exclude.len > 0 and exclude in file: + continue + imports.add( + quote do: + import `file` + ) imports macro importAll*(paths: static seq[string]): untyped = diff --git a/tests/integration/multinodes.nim b/tests/integration/multinodes.nim index 9d4153bd..d5419255 100644 --- a/tests/integration/multinodes.nim +++ b/tests/integration/multinodes.nim @@ -127,11 +127,15 @@ template multinodesuite*(suiteName: string, body: untyped) = lastUsedStorageApiPort = apiPort lastUsedStorageDiscPort = discPort - for bootstrapNode in bootstrapNodes: - config.addCliOption("--bootstrap-node", bootstrapNode) + if bootstrapNodes.len == 0: + # Without this flag the node would bootstrap on the default + # network preset. + config.addCliOption("--no-bootstrap-node") + else: + for bootstrapNode in bootstrapNodes: + config.addCliOption("--bootstrap-node", bootstrapNode) config.addCliOption("--data-dir", datadir) - config.addCliOption("--nat", "none") except StorageConfigError as e: raiseMultiNodeSuiteError "invalid cli option, error: " & e.msg @@ -214,10 +218,14 @@ template multinodesuite*(suiteName: string, body: untyped) = trace "Setting up test", suite = suiteName, test = currentTestName, nodeConfigs if var clients =? nodeConfigs.clients: failAndTeardownOnError "failed to start client nodes": + # Only the first node (bootstrap) gets a known extip. Other nodes use + # nat=auto so AutoNAT can run and determine their reachability. + clients = clients.withExtIp(0).withAutonatServer(0) for config in clients.configs: let node = await startClientNode(config) running.add RunningNode(role: Role.Client, node: node) - await StorageProcess(node).updateBootstrapNodes() + if config.isBootstrapNode: + await StorageProcess(node).updateBootstrapNodes() teardown: await teardownImpl() diff --git a/tests/integration/nat/Dockerfile b/tests/integration/nat/Dockerfile new file mode 100644 index 00000000..4d297c65 --- /dev/null +++ b/tests/integration/nat/Dockerfile @@ -0,0 +1,52 @@ +# One image for every podman NAT scenario, built as localhost/storage-nat. +# Carries the storage binary + miniupnpd (for the upnp/pcp routers); scenarios +# differ only in their entrypoint scripts, which compose mounts. +# Build context = project root. +FROM ubuntu:24.04 + +ARG NIM_VERSION=2.2.10 + +RUN apt-get update && apt-get install -y --no-install-recommends \ + gcc g++ make cmake git curl ca-certificates xz-utils \ + libc-dev ccache pkg-config \ + iproute2 iptables nftables jq \ + libnftnl-dev libmnl-dev \ + && rm -rf /var/lib/apt/lists/* + +# miniupnpd with the real nftables backend (the iptables backend no longer builds +# against modern libiptc), used by the upnp/pcp routers: its mapping requests +# install a genuine DNAT on the router, so AutoNAT's dial-back reaches the node. +RUN git clone --depth=1 --branch miniupnpd_2_3_9 \ + https://github.com/miniupnp/miniupnp.git /tmp/miniupnp-nft \ + && cd /tmp/miniupnp-nft/miniupnpd \ + && ./configure --firewall=nftables \ + && make miniupnpd \ + && install -m 755 miniupnpd /usr/local/sbin/miniupnpd-nft \ + && rm -rf /tmp/miniupnp-nft + +RUN curl -fsSL "https://nim-lang.org/download/nim-${NIM_VERSION}-linux_x64.tar.xz" \ + | tar -xJ -C /opt +RUN ln -s "/opt/nim-${NIM_VERSION}/bin/nim" /usr/local/bin/nim + +WORKDIR /app + +# vendor/ already has the checked-out submodules, so no `make update` here. +COPY vendor/ vendor/ +COPY storage/ storage/ +COPY build.nims config.nims storage.nim ./ + +# libplum static lib, linked by nim-libplum. +RUN --mount=type=cache,target=/root/.ccache \ + export PATH="/usr/lib/ccache:$PATH" && \ + rm -rf vendor/nim-libplum/vendor/libplum/build && \ + cmake -B vendor/nim-libplum/vendor/libplum/build \ + -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF \ + vendor/nim-libplum/vendor/libplum && \ + make -j"$(nproc)" -C vendor/nim-libplum/vendor/libplum/build && \ + cp vendor/nim-libplum/vendor/libplum/build/libplum.a \ + vendor/nim-libplum/vendor/libplum/libplum.a + +RUN --mount=type=cache,target=/root/.ccache \ + export PATH="/usr/lib/ccache:$PATH" && \ + USE_SYSTEM_NIM=1 vendor/nimbus-build-system/scripts/env.sh \ + nim storage -d:disable_libbacktrace build.nims diff --git a/tests/integration/nat/composehelper.nim b/tests/integration/nat/composehelper.nim new file mode 100644 index 00000000..8386ffc3 --- /dev/null +++ b/tests/integration/nat/composehelper.nim @@ -0,0 +1,72 @@ +## Helpers shared by the compose-driven NAT scenario tests (real topology, not +## the in-process simulation). Each scenario provides its own compose.yml and +## the list of services whose logs should be collected. + +import std/[os, osproc] +import ../utils + +const + routerWanIp* = "7.7.7.2" ## public IP AutoNAT observes for a NATed node (masquerade) + bootstrapIp* = "7.7.7.10" ## relay + bootstrap public IP + +proc composeCmd(composeFile: string): string = + ## Prefer podman (where the Makefile builds the image), fall back to docker. + let base = + if findExe("podman-compose") != "": + "podman-compose" + elif findExe("podman") != "": + "podman compose" + elif findExe("docker") != "": + "docker compose" + else: + raise newException(IOError, "neither podman nor docker found") + base & " -f \"" & composeFile & "\"" + +proc compose*(composeFile, action: string) = + let cmd = composeCmd(composeFile) & " " & action + doAssert execShellCmd(cmd) == 0, "command failed: " & cmd + +proc serviceLogs*(composeFile, service: string): string = + ## Current logs (stdout+stderr) of a compose service, or "" on error. + try: + let + cmd = composeCmd(composeFile) & " logs " & service + (output, code) = execCmdEx(cmd) + if code != 0: + echo "warning: '", cmd, "' exited ", code + output + except CatchableError as e: + echo "could not read logs for ", service, ": ", e.msg + "" + +proc saveContainerLogs*( + composeFile, suiteName, testName, startTime: string, services: openArray[string] +) = + ## Writes each container's log via getLogFile, the same helper and layout as + ## the multinodes suite: tests/integration/logs/__/ + ## /.log. Must run before `down` destroys the containers. + for service in services: + try: + let logFile = getLogFile("", startTime, suiteName, testName, service) + writeFile(logFile, serviceLogs(composeFile, service)) + except CatchableError as e: + echo "could not save logs for ", service, ": ", e.msg + +template eventuallyInfo*(client, predicate: untyped): bool = + ## Poll `client.info()` until `predicate` holds, swallowing HttpError while the + ## node's API is still starting. The decoded info JsonNode is exposed as `info` + ## inside `predicate`. + eventuallySafe( + block: + var satisfied = false + try: + let res = await client.info() + if res.isOk: + let info {.inject.} = res.get + satisfied = predicate + except HttpError: + discard + satisfied, + timeout = 300000, + pollInterval = 5000, + ) diff --git a/tests/integration/nat/connection-reversal/README.md b/tests/integration/nat/connection-reversal/README.md new file mode 100644 index 00000000..459987d6 --- /dev/null +++ b/tests/integration/nat/connection-reversal/README.md @@ -0,0 +1,46 @@ +# NAT connection-reversal scenario + +## Scenario + +A node behind a NAT is reachable only through A's relay. When the reachable node +C dials it through the relay, the relayed node dials C back directly (C is +public) and the relayed connection is upgraded to a direct one. + +This is a unilateral reversal — only the NATed node dials, because C is public. +It is not a coordinated hole punch; see `../hole-punch` for the both-NATed case. + +## Topology + +``` +node B ──── lan ──── router (NAT) ──── wan ──── bootstrap A (relay) + └────── node C (reachable) +``` + +- **bootstrap A** — public node on the wan, autonat + relay server. +- **router** — `lan -> wan` masquerade and *no* inbound forward. +- **node B** — `nat=auto`, on the lan. NotReachable, takes a relay reservation + on A. When C reaches it through the relay, its hole-punching handler dials C + back directly and closes the relayed connection. +- **node C** — `nat=auto`, directly on the wan, so it is `Reachable`. It dials B + through the relay. + +## Run + +```bash +make testNatIntegration \ + STORAGE_INTEGRATION_TEST_INCLUDES=tests/integration/nat/connection-reversal/testconnectionreversal.nim +``` + +Builds the shared image and brings the compose topology up and down. Rootless, but +needs the host netfilter modules — if the router fails on iptables: +`sudo modprobe iptable_nat nf_conntrack`. + +## Expected result + +B is `NotReachable` behind the relay, C is `Reachable`. C downloads from B +through the relay, which opens a relayed connection; B then dials C back +directly. The direct upgrade has no REST surface, so the test asserts on B's log +line `Direct connection created.`. + +Per-run container logs (router, bootstrap, client, node) are written before teardown to +`tests/integration/logs/__NAT_connection_reversal//.log`. diff --git a/tests/integration/nat/connection-reversal/compose.yml b/tests/integration/nat/connection-reversal/compose.yml new file mode 100644 index 00000000..4a75ba32 --- /dev/null +++ b/tests/integration/nat/connection-reversal/compose.yml @@ -0,0 +1,106 @@ +# NAT connection-reversal scenario — see README.md. Run via testconnectionreversal.nim. +name: nat-connection-reversal + +# Topology addresses, named for their role (defined once, referenced below). +x-addresses: + # fake public internet; a routable range so B looks public to A + wan_subnet: &wan_subnet 7.7.7.0/24 + # private network behind the NAT + lan_subnet: &lan_subnet 10.99.0.0/24 + # A: public bootstrap, autonat + relay server + bootstrap_ip: &bootstrap_ip 7.7.7.10 + # C: public node on the wan, reachable, the one that dials B through the relay + client_ip: &client_ip 7.7.7.20 + # router's public face + router_wan_ip: &router_wan_ip 7.7.7.2 + # router's private face = B's gateway + router_lan_ip: &router_lan_ip 10.99.0.2 + # B, behind the NAT + node_ip: &node_ip 10.99.0.10 + +networks: + wan: + # Keep the fake public range private, not exposed to the host + internal: true + ipam: + config: + - subnet: *wan_subnet + lan: + ipam: + config: + - subnet: *lan_subnet + +services: + router: + image: localhost/storage-nat + cap_add: [NET_ADMIN] + sysctls: + net.ipv4.ip_forward: 1 + networks: + wan: + ipv4_address: *router_wan_ip + lan: + ipv4_address: *router_lan_ip + environment: + ROUTER_WAN_IP: *router_wan_ip + LAN_SUBNET: *lan_subnet + # scripts mounted, so editing them needs no image rebuild + volumes: + - ../router-common.sh:/scripts/router-common.sh:ro,z + - ./router-entrypoint.sh:/scripts/router-entrypoint.sh:ro,z + entrypoint: ["bash", "/scripts/router-entrypoint.sh"] + + bootstrap: + image: localhost/storage-nat + networks: + wan: + ipv4_address: *bootstrap_ip + entrypoint: ["/app/build/storage"] + command: + - --listen-ip=0.0.0.0 + - --api-bindaddr=0.0.0.0 + - --listen-port=8070 + - --disc-port=8090 + - --api-port=8080 + # bootstrap_ip (anchors can't go inside a string) + - --nat=extip:7.7.7.10 + - --relay-server + - --autonat-server + - --no-bootstrap-node + - --data-dir=/data + - --log-level=DEBUG + + # C sits on the wan, directly reachable + client: + image: localhost/storage-nat + depends_on: [bootstrap] + networks: + wan: + ipv4_address: *client_ip + # C's API, published so the test can drive the download from C and poll it + ports: + - "127.0.0.1:18089:8080" + environment: + # C fetches A's SPR from this API at startup to join the network (bootstrap_ip) + BOOTSTRAP_API: http://7.7.7.10:8080 + volumes: + - ../node-entrypoint.sh:/scripts/node-entrypoint.sh:ro,z + entrypoint: ["bash", "/scripts/node-entrypoint.sh"] + + node: + image: localhost/storage-nat + cap_add: [NET_ADMIN] + depends_on: [router, bootstrap] + networks: + lan: + ipv4_address: *node_ip + # B's API, published so the test can upload to it and poll it + ports: + - "127.0.0.1:18088:8080" + environment: + ROUTER_LAN_IP: *router_lan_ip + # B fetches A's SPR from this API at startup to join the network (bootstrap_ip) + BOOTSTRAP_API: http://7.7.7.10:8080 + volumes: + - ../node-entrypoint.sh:/scripts/node-entrypoint.sh:ro,z + entrypoint: ["bash", "/scripts/node-entrypoint.sh"] diff --git a/tests/integration/nat/connection-reversal/router-entrypoint.sh b/tests/integration/nat/connection-reversal/router-entrypoint.sh new file mode 100755 index 00000000..21a8ef3b --- /dev/null +++ b/tests/integration/nat/connection-reversal/router-entrypoint.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +source "$(dirname "$0")/router-common.sh" + +echo "router ready (wan iface $wanif)" + +hold_until_stopped diff --git a/tests/integration/nat/connection-reversal/testconnectionreversal.nim b/tests/integration/nat/connection-reversal/testconnectionreversal.nim new file mode 100644 index 00000000..0c4c1306 --- /dev/null +++ b/tests/integration/nat/connection-reversal/testconnectionreversal.nim @@ -0,0 +1,62 @@ +## NAT connection-reversal scenario. See README.md. + +import std/[json, os, sequtils, strutils, times] +import pkg/chronos +import pkg/questionable/results + +import ../../../asynctest +import ../../../checktest +import ../../storageclient +import ../composehelper + +const directConnLog = "Direct connection created." + +proc announcesCircuitAddr(info: JsonNode): bool = + ## A node behind the relay announces its circuit (p2p-circuit) address. + info{"announceAddresses"}.getElems.anyIt("p2p-circuit" in it.getStr) + +asyncchecksuite "NAT connection reversal": + let + composeFile = currentSourcePath.parentDir / "compose.yml" + nodeApiUrl = "http://127.0.0.1:18088/api/storage/v1" + clientApiUrl = "http://127.0.0.1:18089/api/storage/v1" + suiteName = "NAT connection reversal" + testName = "a relayed node is upgraded to a direct connection" + services = ["router", "bootstrap", "client", "node"] + startTime = now().format("yyyy-MM-dd'_'HH:mm:ss") + var + nodeClient: StorageClient + clientC: StorageClient + + setup: + compose(composeFile, "up -d") + nodeClient = StorageClient.new(nodeApiUrl) + clientC = StorageClient.new(clientApiUrl) + + teardown: + await nodeClient.close() + await clientC.close() + saveContainerLogs(composeFile, suiteName, testName, startTime, services) + compose(composeFile, "down -v") + + test testName: + # B is NotReachable behind the relay, C is reachable + check eventuallyInfo( + nodeClient, + info{"nat"}{"reachability"}.getStr == "NotReachable" and + info.announcesCircuitAddr(), + ) + + # C is Reachable + check eventuallyInfo(clientC, info{"nat"}{"reachability"}.getStr == "Reachable") + + # C dials B through the relay; a download is enough to open the connection + let cid = (await nodeClient.upload("hole punch me")).get + check (await clientC.download(cid)).isOk + + # B sees the relayed peer C join and dials it back directly + check eventuallySafe( + directConnLog in serviceLogs(composeFile, "node"), + timeout = 60_000, + pollInterval = 2_000, + ) diff --git a/tests/integration/nat/hole-punch/README.md b/tests/integration/nat/hole-punch/README.md new file mode 100644 index 00000000..b0c25fd7 --- /dev/null +++ b/tests/integration/nat/hole-punch/README.md @@ -0,0 +1,42 @@ +# NAT hole-punch scenario + +## Scenario + +Two nodes are each behind their **own** NAT, so neither can reach the other on a +shared lan. Both are `NotReachable` and take a relay reservation on A. When D +downloads from B through the relay, B drives a **coordinated** DCUtR +simultaneous-open and starts hole-punching. + +## Topology + +``` +node B ── lan1 ── router1 (NAT) ──┐ + ├── wan ── bootstrap A (relay + autonat) +node D ── lan2 ── router2 (NAT) ──┘ +``` + +- **bootstrap A** — public node on the wan, autonat + relay server. +- **router1 / router2** — `lan -> wan` masquerade, *no* inbound forward. +- **node B** — behind router1, NotReachable, uploaded to and downloaded from. + Holds the inbound relayed connection, so it is the DCUtR initiator. +- **node D** — behind router2, NotReachable, downloads from B through the relay. + +## Run + +```bash +make testNatIntegration \ + STORAGE_INTEGRATION_TEST_INCLUDES=tests/integration/nat/hole-punch/testholepunch.nim +``` + +Rootless, but needs the host netfilter modules — if a router fails on iptables: +`sudo modprobe iptable_nat nf_conntrack`. + +## Expected result + +Both nodes are `NotReachable`. D downloads from B through the relay, opening a +relayed connection; B then runs DCUtR and the connection is upgraded to a direct +one. The test polls B's `/debug/info` and asserts its connection to D becomes +non-relayed (`connections[].direct == true` for D's peerId). + +Per-run container logs are written before teardown to +`tests/integration/logs/__NAT_hole_punching//.log`. diff --git a/tests/integration/nat/hole-punch/compose.yml b/tests/integration/nat/hole-punch/compose.yml new file mode 100644 index 00000000..9a3b641c --- /dev/null +++ b/tests/integration/nat/hole-punch/compose.yml @@ -0,0 +1,126 @@ +# NAT hole-punch scenario — see README.md. Run via testholepunch.nim. +name: nat-hole-punch + +# Two NATed nodes, one behind each router (separate lans), so neither can reach +# the other on a shared lan: the only direct path is a coordinated hole punch. +x-addresses: + wan_subnet: &wan_subnet 7.7.7.0/24 + lan1_subnet: &lan1_subnet 10.99.1.0/24 + lan2_subnet: &lan2_subnet 10.99.2.0/24 + # A: public bootstrap, autonat + relay server + bootstrap_ip: &bootstrap_ip 7.7.7.10 + # router1 (B's NAT) + router1_wan_ip: &router1_wan_ip 7.7.7.3 + router1_lan_ip: &router1_lan_ip 10.99.1.2 + # router2 (D's NAT) + router2_wan_ip: &router2_wan_ip 7.7.7.4 + router2_lan_ip: &router2_lan_ip 10.99.2.2 + # B behind router1, D behind router2 + node_ip: &node_ip 10.99.1.10 + peer_ip: &peer_ip 10.99.2.10 + +networks: + wan: + internal: true + ipam: + config: + - subnet: *wan_subnet + lan1: + ipam: + config: + - subnet: *lan1_subnet + lan2: + ipam: + config: + - subnet: *lan2_subnet + +services: + router1: + image: localhost/storage-nat + cap_add: [NET_ADMIN] + sysctls: + net.ipv4.ip_forward: 1 + networks: + wan: + ipv4_address: *router1_wan_ip + lan1: + ipv4_address: *router1_lan_ip + environment: + ROUTER_WAN_IP: *router1_wan_ip + LAN_SUBNET: *lan1_subnet + volumes: + - ../router-common.sh:/scripts/router-common.sh:ro,z + - ./router-entrypoint.sh:/scripts/router-entrypoint.sh:ro,z + entrypoint: ["bash", "/scripts/router-entrypoint.sh"] + + router2: + image: localhost/storage-nat + cap_add: [NET_ADMIN] + sysctls: + net.ipv4.ip_forward: 1 + networks: + wan: + ipv4_address: *router2_wan_ip + lan2: + ipv4_address: *router2_lan_ip + environment: + ROUTER_WAN_IP: *router2_wan_ip + LAN_SUBNET: *lan2_subnet + volumes: + - ../router-common.sh:/scripts/router-common.sh:ro,z + - ./router-entrypoint.sh:/scripts/router-entrypoint.sh:ro,z + entrypoint: ["bash", "/scripts/router-entrypoint.sh"] + + bootstrap: + image: localhost/storage-nat + networks: + wan: + ipv4_address: *bootstrap_ip + entrypoint: ["/app/build/storage"] + command: + - --listen-ip=0.0.0.0 + - --api-bindaddr=0.0.0.0 + - --listen-port=8070 + - --disc-port=8090 + - --api-port=8080 + # bootstrap_ip (anchors can't go inside a string) + - --nat=extip:7.7.7.10 + - --relay-server + - --autonat-server + - --no-bootstrap-node + - --data-dir=/data + - --log-level=DEBUG + + # B: behind router1, uploaded to and downloaded from + node: + image: localhost/storage-nat + cap_add: [NET_ADMIN] + depends_on: [router1, bootstrap] + networks: + lan1: + ipv4_address: *node_ip + ports: + - "127.0.0.1:18090:8080" + environment: + ROUTER_LAN_IP: *router1_lan_ip + BOOTSTRAP_API: http://7.7.7.10:8080 + volumes: + - ../node-entrypoint.sh:/scripts/node-entrypoint.sh:ro,z + entrypoint: ["bash", "/scripts/node-entrypoint.sh"] + + # D: behind router2, the one that downloads from B through the relay + peer: + image: localhost/storage-nat + cap_add: [NET_ADMIN] + depends_on: [router2, bootstrap] + networks: + lan2: + ipv4_address: *peer_ip + ports: + - "127.0.0.1:18091:8080" + environment: + ROUTER_LAN_IP: *router2_lan_ip + BOOTSTRAP_API: http://7.7.7.10:8080 + volumes: + - ../node-entrypoint.sh:/scripts/node-entrypoint.sh:ro,z + entrypoint: ["bash", "/scripts/node-entrypoint.sh"] diff --git a/tests/integration/nat/hole-punch/router-entrypoint.sh b/tests/integration/nat/hole-punch/router-entrypoint.sh new file mode 100644 index 00000000..11d6d17d --- /dev/null +++ b/tests/integration/nat/hole-punch/router-entrypoint.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +source "$(dirname "$0")/router-common.sh" + +# Drop early punch SYN so TCP retransmits until the +# pinhole is open and the SYN gets forwarded. +iptables -A INPUT -i "$wanif" -p tcp --dport 8070 -j DROP + +echo "router ready (wan iface $wanif)" + +hold_until_stopped diff --git a/tests/integration/nat/hole-punch/testholepunch.nim b/tests/integration/nat/hole-punch/testholepunch.nim new file mode 100644 index 00000000..5a4db7ad --- /dev/null +++ b/tests/integration/nat/hole-punch/testholepunch.nim @@ -0,0 +1,64 @@ +## Coordinated DCUtR hole-punching scenario (both peers NATed). See README.md. + +import std/[json, os, sequtils, strutils, times] +import pkg/chronos +import pkg/questionable/results + +import ../../../asynctest +import ../../../checktest +import ../../storageclient +import ../composehelper + +proc announcesCircuitAddr(info: JsonNode): bool = + info{"announceAddresses"}.getElems.anyIt("p2p-circuit" in it.getStr) + +asyncchecksuite "NAT hole punching": + let + composeFile = currentSourcePath.parentDir / "compose.yml" + nodeApiUrl = "http://127.0.0.1:18090/api/storage/v1" + peerApiUrl = "http://127.0.0.1:18091/api/storage/v1" + suiteName = "NAT hole punching" + testName = "two NATed nodes upgrade a relayed connection to a direct one" + services = ["router1", "router2", "bootstrap", "node", "peer"] + startTime = now().format("yyyy-MM-dd'_'HH:mm:ss") + var + nodeClient: StorageClient + peerClient: StorageClient + + setup: + compose(composeFile, "up -d") + nodeClient = StorageClient.new(nodeApiUrl) + peerClient = StorageClient.new(peerApiUrl) + + teardown: + await nodeClient.close() + await peerClient.close() + saveContainerLogs(composeFile, suiteName, testName, startTime, services) + compose(composeFile, "down -v") + + test testName: + # Both nodes are NotReachable behind their own NAT and take a relay reservation. + check eventuallyInfo( + nodeClient, + info{"nat"}{"reachability"}.getStr == "NotReachable" and + info.announcesCircuitAddr(), + ) + check eventuallyInfo( + peerClient, + info{"nat"}{"reachability"}.getStr == "NotReachable" and + info.announcesCircuitAddr(), + ) + + # D downloads from B through the relay; that opens the relayed connection. + let cid = (await nodeClient.upload("punch me for real")).get + check (await peerClient.download(cid)).isOk + + # B should upgrade the relayed connection to a direct one: its connection to D + # becomes non-relayed + let peerId = (await peerClient.info()).get{"id"}.getStr + check eventuallyInfo( + nodeClient, + info{"connections"}.getElems.anyIt( + it{"peerId"}.getStr == peerId and it{"direct"}.getBool + ), + ) diff --git a/tests/integration/nat/node-entrypoint.sh b/tests/integration/nat/node-entrypoint.sh new file mode 100644 index 00000000..118bca81 --- /dev/null +++ b/tests/integration/nat/node-entrypoint.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# Redirect the traffic to our router instead of podman's own gateway to put the +# node behind the NAT. A node on the wan (reachable) leaves ROUTER_LAN_IP unset +# and keeps its default route. +if [[ -n "${ROUTER_LAN_IP:-}" ]]; then + ip route replace default via "$ROUTER_LAN_IP" +fi + +# Fetch the bootstrap SPR (retry: the bootstrap may still be starting). +echo "fetching bootstrap SPR from $BOOTSTRAP_API ..." +spr="" +for _ in $(seq 1 60); do + spr=$(curl -fsS -H 'Accept: text/plain' "$BOOTSTRAP_API/api/storage/v1/spr" || true) + [[ -n "$spr" ]] && break + sleep 1 +done +[[ -n "$spr" ]] || { echo "ERROR: could not fetch bootstrap SPR" >&2; exit 1; } + +# api-bindaddr=0.0.0.0 so the published host port reaches the REST API. +exec /app/build/storage \ + --listen-ip=0.0.0.0 --api-bindaddr=0.0.0.0 \ + --listen-port=8070 --disc-port=8090 --api-port=8080 \ + --bootstrap-node="$spr" \ + --nat-num-peers-to-ask=1 --nat-max-queue-size=1 \ + --nat-min-confidence=1.0 --nat-schedule-interval=30s \ + --data-dir=/data --log-level=DEBUG \ + ${EXTRA_STORAGE_ARGS:-} diff --git a/tests/integration/nat/not-downloadable/README.md b/tests/integration/nat/not-downloadable/README.md new file mode 100644 index 00000000..5581a52a --- /dev/null +++ b/tests/integration/nat/not-downloadable/README.md @@ -0,0 +1,42 @@ +# NAT not-downloadable scenario + +## Scenario + +A node behind a NAT with no relay is `NotReachable` and announces no dialable +address. A remote peer can never dial it, so a download from it fails. + +## Topology + +``` +node B ──── lan ──── router (NAT) ──── wan ──── bootstrap A + └────── node C (reachable) +``` + +- **bootstrap A** — public node on the wan, autonat server, started with + `--nat=extip`. Unlike not-reachable, it runs *without* `--relay-server`, so B + has no relay to fall back to. +- **router** — `lan -> wan` masquerade and *no* inbound forward, so B can dial + out but nothing can dial back in. +- **node B** — `nat=auto`, on the lan. It joins via A, AutoNAT finds it + unreachable, and with no relay it ends up announcing nothing dialable. +- **node C** — `nat=auto`, directly on the wan, so AutoNAT finds it + `Reachable`. It is the peer that tries (and fails) to download from B. + +## Run + +```bash +make testNatIntegration \ + STORAGE_INTEGRATION_TEST_INCLUDES=tests/integration/nat/not-downloadable/testnotdownloadable.nim +``` + +Builds the shared image and brings the compose topology up and down. Rootless, but +needs the host netfilter modules — if the router fails on iptables: +`sudo modprobe iptable_nat nf_conntrack`. + +## Expected result + +B is `NotReachable` and announces no address, while C is `Reachable`. B uploads +a file, then C tries to fetch its manifest over the network and fails. + +Per-run container logs (router, bootstrap, client, node) are written before teardown to +`tests/integration/logs/__NAT_not_downloadable//.log`. diff --git a/tests/integration/nat/not-downloadable/compose.yml b/tests/integration/nat/not-downloadable/compose.yml new file mode 100644 index 00000000..414f127c --- /dev/null +++ b/tests/integration/nat/not-downloadable/compose.yml @@ -0,0 +1,105 @@ +# NAT not-downloadable scenario — see README.md. Run via testnotdownloadable.nim. +name: nat-not-downloadable + +# Topology addresses, named for their role (defined once, referenced below). +x-addresses: + # fake public internet; a routable range so B looks public to A + wan_subnet: &wan_subnet 7.7.7.0/24 + # private network behind the NAT + lan_subnet: &lan_subnet 10.99.0.0/24 + # A: public bootstrap, autonat server (no relay) + bootstrap_ip: &bootstrap_ip 7.7.7.10 + # C: public node on the wan, reachable, the one that tries to download from B + client_ip: &client_ip 7.7.7.20 + # router's public face + router_wan_ip: &router_wan_ip 7.7.7.2 + # router's private face = B's gateway + router_lan_ip: &router_lan_ip 10.99.0.2 + # B, behind the NAT + node_ip: &node_ip 10.99.0.10 + +networks: + wan: + # Keep the fake public range private, not exposed to the host + internal: true + ipam: + config: + - subnet: *wan_subnet + lan: + ipam: + config: + - subnet: *lan_subnet + +services: + router: + image: localhost/storage-nat + cap_add: [NET_ADMIN] + sysctls: + net.ipv4.ip_forward: 1 + networks: + wan: + ipv4_address: *router_wan_ip + lan: + ipv4_address: *router_lan_ip + environment: + ROUTER_WAN_IP: *router_wan_ip + LAN_SUBNET: *lan_subnet + # scripts mounted, so editing them needs no image rebuild + volumes: + - ../router-common.sh:/scripts/router-common.sh:ro,z + - ./router-entrypoint.sh:/scripts/router-entrypoint.sh:ro,z + entrypoint: ["bash", "/scripts/router-entrypoint.sh"] + + bootstrap: + image: localhost/storage-nat + networks: + wan: + ipv4_address: *bootstrap_ip + entrypoint: ["/app/build/storage"] + command: + - --listen-ip=0.0.0.0 + - --api-bindaddr=0.0.0.0 + - --listen-port=8070 + - --disc-port=8090 + - --api-port=8080 + # bootstrap_ip (anchors can't go inside a string) + - --nat=extip:7.7.7.10 + - --autonat-server + - --no-bootstrap-node + - --data-dir=/data + - --log-level=DEBUG + + # C sits on the wan, directly reachable + client: + image: localhost/storage-nat + depends_on: [bootstrap] + networks: + wan: + ipv4_address: *client_ip + # C's API, published so the test can drive the download from C and poll it + ports: + - "127.0.0.1:18085:8080" + environment: + # C fetches A's SPR from this API at startup to join the network (bootstrap_ip) + BOOTSTRAP_API: http://7.7.7.10:8080 + volumes: + - ../node-entrypoint.sh:/scripts/node-entrypoint.sh:ro,z + entrypoint: ["bash", "/scripts/node-entrypoint.sh"] + + node: + image: localhost/storage-nat + cap_add: [NET_ADMIN] + depends_on: [router, bootstrap] + networks: + lan: + ipv4_address: *node_ip + # B's API, published so the test can upload to it and poll it + ports: + - "127.0.0.1:18084:8080" + environment: + ROUTER_LAN_IP: *router_lan_ip + # B fetches A's SPR from this API at startup to join the network (bootstrap_ip) + BOOTSTRAP_API: http://7.7.7.10:8080 + volumes: + - ../node-entrypoint.sh:/scripts/node-entrypoint.sh:ro,z + entrypoint: ["bash", "/scripts/node-entrypoint.sh"] diff --git a/tests/integration/nat/not-downloadable/router-entrypoint.sh b/tests/integration/nat/not-downloadable/router-entrypoint.sh new file mode 100755 index 00000000..21a8ef3b --- /dev/null +++ b/tests/integration/nat/not-downloadable/router-entrypoint.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +source "$(dirname "$0")/router-common.sh" + +echo "router ready (wan iface $wanif)" + +hold_until_stopped diff --git a/tests/integration/nat/not-downloadable/testnotdownloadable.nim b/tests/integration/nat/not-downloadable/testnotdownloadable.nim new file mode 100644 index 00000000..79045b56 --- /dev/null +++ b/tests/integration/nat/not-downloadable/testnotdownloadable.nim @@ -0,0 +1,60 @@ +## NAT not-downloadable scenario. See README.md. + +import std/[json, os, times] +import pkg/chronos +import pkg/questionable/results + +import ../../../asynctest +import ../../../checktest +import ../../storageclient +import ../composehelper + +proc announcesNothing(info: JsonNode): bool = + ## An unreachable node with no relay has no dialable address to announce. + info{"announceAddresses"}.getElems.len == 0 + +asyncchecksuite "NAT not downloadable": + let + composeFile = currentSourcePath.parentDir / "compose.yml" + nodeApiUrl = "http://127.0.0.1:18084/api/storage/v1" + clientApiUrl = "http://127.0.0.1:18085/api/storage/v1" + suiteName = "NAT not downloadable" + testName = "a NAT'd node without relay cannot be downloaded from" + services = ["router", "bootstrap", "client", "node"] + startTime = now().format("yyyy-MM-dd'_'HH:mm:ss") + var + nodeClient: StorageClient + clientC: StorageClient + + setup: + compose(composeFile, "up -d") + nodeClient = StorageClient.new(nodeApiUrl) + clientC = StorageClient.new(clientApiUrl) + + teardown: + await nodeClient.close() + await clientC.close() + saveContainerLogs(composeFile, suiteName, testName, startTime, services) + compose(composeFile, "down -v") + + test testName: + # Make sure nodeClient is not reachable + check eventuallyInfo( + nodeClient, + info{"nat"}{"reachability"}.getStr == "NotReachable" and info.announcesNothing(), + ) + + let info = (await nodeClient.info()).get + # Double check to make sure nodeClient is not reachable and has + # nothing to announce + check info.announcesNothing() + + # C is reachable + check eventuallyInfo(clientC, info{"nat"}{"reachability"}.getStr == "Reachable") + + # B uploads a file + let cid = (await nodeClient.upload("hello from behind the NAT")).get + + # C cannot download the manifest, as B is not reachable + let res = await clientC.downloadManifestOnly(cid) + check res.isErr diff --git a/tests/integration/nat/not-reachable/README.md b/tests/integration/nat/not-reachable/README.md new file mode 100644 index 00000000..32d9e008 --- /dev/null +++ b/tests/integration/nat/not-reachable/README.md @@ -0,0 +1,64 @@ +# NAT not-reachable scenario + +## Scenario + +A node behind a NAT that cannot be reached from outside must be detected +`NotReachable` and fall back to bootstrap A's relay. + +## Topology + +``` +node B ──── lan ──── router (NAT) ──── wan ──── bootstrap A +``` + +- **bootstrap A** — public node on the wan, runs the relay + autonat server, + started with `--nat=extip` so it advertises its own public address. +- **router** — two interfaces (lan + wan). Does `lan -> wan` masquerade and *no* + inbound forward, so B can dial out but nothing can dial back in. +- **node B** — `nat=auto`, on the lan. Its default route points at the router, + so all wan-bound traffic is NATed. It fetches A's SPR over A's API to join, + then AutoNAT probes A and finds itself unreachable. + +The wan uses a real public range because our address policy keeps only public +dialable addresses: a private observed address would be filtered out and AutoNAT +would stay `Unknown` instead of `NotReachable`. The wan is `internal` so that +range never leaks to host routes. + +## Run + +Every NAT scenario: + +```bash +make testNatIntegration +``` + +Just this one — same `STORAGE_INTEGRATION_TEST_INCLUDES` filter as testIntegration, +with the test file path: + +```bash +make testNatIntegration \ + STORAGE_INTEGRATION_TEST_INCLUDES=tests/integration/nat/not-reachable/testnotreachable.nim +``` + +Builds the shared image and brings the compose topology up and down. Rootless, but +needs the host netfilter modules — if the router fails on iptables: +`sudo modprobe iptable_nat nf_conntrack`. + +## Expected result + +B ends up `NotReachable` with the relay running, announcing only its circuit +(relay) address — never a direct one. Its `debug/info`: + +```json +{ + "nat": { + "reachability": "NotReachable", + "clientMode": true, + "relayRunning": true, + "portMapping": "none" + } +} +``` + +Per-run container logs (router, bootstrap, node) are written before teardown to +`tests/integration/logs/__NAT_not_reachable//.log`. diff --git a/tests/integration/nat/not-reachable/compose.yml b/tests/integration/nat/not-reachable/compose.yml new file mode 100644 index 00000000..80294483 --- /dev/null +++ b/tests/integration/nat/not-reachable/compose.yml @@ -0,0 +1,86 @@ +# NAT not-reachable scenario — see README.md. Run via testnotreachable.nim. +name: nat-not-reachable + +# Topology addresses, named for their role (defined once, referenced below). +x-addresses: + # fake public internet; a routable range so B looks public to A + wan_subnet: &wan_subnet 7.7.7.0/24 + # private network behind the NAT + lan_subnet: &lan_subnet 10.99.0.0/24 + # A: public bootstrap, relay + autonat server + bootstrap_ip: &bootstrap_ip 7.7.7.10 + # router's public face + router_wan_ip: &router_wan_ip 7.7.7.2 + # router's private face = B's gateway + router_lan_ip: &router_lan_ip 10.99.0.2 + # B, behind the NAT + node_ip: &node_ip 10.99.0.10 + +networks: + wan: + # Keep the fake public range private, not exposed to the host + internal: true + ipam: + config: + - subnet: *wan_subnet + lan: + ipam: + config: + - subnet: *lan_subnet + +services: + router: + image: localhost/storage-nat + cap_add: [NET_ADMIN] + sysctls: + net.ipv4.ip_forward: 1 + networks: + wan: + ipv4_address: *router_wan_ip + lan: + ipv4_address: *router_lan_ip + environment: + ROUTER_WAN_IP: *router_wan_ip + LAN_SUBNET: *lan_subnet + # scripts mounted, so editing them needs no image rebuild + volumes: + - ../router-common.sh:/scripts/router-common.sh:ro,z + - ./router-entrypoint.sh:/scripts/router-entrypoint.sh:ro,z + entrypoint: ["bash", "/scripts/router-entrypoint.sh"] + + bootstrap: + image: localhost/storage-nat + networks: + wan: + ipv4_address: *bootstrap_ip + entrypoint: ["/app/build/storage"] + command: + - --listen-ip=0.0.0.0 + - --api-bindaddr=0.0.0.0 + - --listen-port=8070 + - --disc-port=8090 + - --api-port=8080 + # bootstrap_ip (anchors can't go inside a string) + - --nat=extip:7.7.7.10 + - --relay-server + - --autonat-server + - --no-bootstrap-node + - --data-dir=/data + - --log-level=DEBUG + + node: + image: localhost/storage-nat + cap_add: [NET_ADMIN] + depends_on: [router, bootstrap] + networks: + lan: + ipv4_address: *node_ip + ports: + - "127.0.0.1:18080:8080" + environment: + ROUTER_LAN_IP: *router_lan_ip + # B fetches A's SPR from this API at startup to join the network (bootstrap_ip) + BOOTSTRAP_API: http://7.7.7.10:8080 + volumes: + - ../node-entrypoint.sh:/scripts/node-entrypoint.sh:ro,z + entrypoint: ["bash", "/scripts/node-entrypoint.sh"] diff --git a/tests/integration/nat/not-reachable/router-entrypoint.sh b/tests/integration/nat/not-reachable/router-entrypoint.sh new file mode 100755 index 00000000..21a8ef3b --- /dev/null +++ b/tests/integration/nat/not-reachable/router-entrypoint.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +source "$(dirname "$0")/router-common.sh" + +echo "router ready (wan iface $wanif)" + +hold_until_stopped diff --git a/tests/integration/nat/not-reachable/testnotreachable.nim b/tests/integration/nat/not-reachable/testnotreachable.nim new file mode 100644 index 00000000..f485fb90 --- /dev/null +++ b/tests/integration/nat/not-reachable/testnotreachable.nim @@ -0,0 +1,50 @@ +## NAT not-reachable scenario. See README.md. + +import std/[json, os, sequtils, strutils, times] +import pkg/chronos +import pkg/questionable/results + +import ../../../asynctest +import ../../../checktest +import ../../storageclient +import ../composehelper + +proc announcesCircuitAddr(info: JsonNode): bool = + info{"announceAddresses"}.getElems.anyIt("p2p-circuit" in it.getStr) + +asyncchecksuite "NAT not reachable": + let + composeFile = currentSourcePath.parentDir / "compose.yml" + nodeApiUrl = "http://127.0.0.1:18080/api/storage/v1" + suiteName = "NAT not reachable" + testName = "node behind NAT is NotReachable and falls back to relay" + services = ["router", "bootstrap", "node"] + startTime = now().format("yyyy-MM-dd'_'HH:mm:ss") + var client: StorageClient + + setup: + compose(composeFile, "up -d") + client = StorageClient.new(nodeApiUrl) + + teardown: + await client.close() + saveContainerLogs(composeFile, suiteName, testName, startTime, services) + compose(composeFile, "down -v") + + test testName: + # Wait for the announcements, after the relay reservation is created. + check eventuallyInfo(client, info.announcesCircuitAddr()) + + let info = (await client.info()).get + let nat = info{"nat"} + check nat{"reachability"}.getStr == "NotReachable" + check nat{"relayRunning"}.getBool + check nat{"portMapping"}.getStr == "none" + check info.announcesCircuitAddr() + let announced = info{"announceAddresses"}.getElems.mapIt(it.getStr) + # the announced circuit address points at the bootstrap's relay + check announced.anyIt( + ("/ip4/" & bootstrapIp & "/tcp/8070" in it) and ("p2p-circuit" in it) + ) + # relay addresses go only into the provider record, never the DHT routing record + check info{"dhtAddresses"}.getElems.len == 0 diff --git a/tests/integration/nat/pcp/README.md b/tests/integration/nat/pcp/README.md new file mode 100644 index 00000000..848156e4 --- /dev/null +++ b/tests/integration/nat/pcp/README.md @@ -0,0 +1,64 @@ +# NAT pcp scenario + +## Scenario + +A node behind a NAT becomes `Reachable` by mapping its port over PCP — the router +forwards nothing on its own, the node asks for the mapping and no relay is needed. + +## Topology + +``` +node B ──── lan ──── router (NAT + miniupnpd/PCP) ──── wan ──── bootstrap A +``` + +- **bootstrap A** — public node on the wan, runs the relay + autonat server. +- **router** — `lan -> wan` masquerade and *no* static forward. It runs + `miniupnpd` (real nftables backend) with PCP/NAT-PMP enabled. libplum tries PCP + first, so the mapping request goes over PCP and installs a real DNAT into the + nft chains the entrypoint pre-creates. +- **node B** — `nat=auto`, on the lan. First detected `NotReachable`, it maps its + TCP listen (8070) and UDP disc (8090) ports over PCP; the resulting DNAT lets + A's dial-back reach it, so the next AutoNAT round flips it to `Reachable`. + +The wan public range and `internal` flag work as in +[not-reachable](../not-reachable/README.md); the public wan IP also keeps +miniupnpd from refusing PCP/NAT-PMP as double-NAT. + +## Run + +Every NAT scenario: + +```bash +make testNatIntegration +``` + +Just this one — same `STORAGE_INTEGRATION_TEST_INCLUDES` filter as testIntegration, +with the test file path: + +```bash +make testNatIntegration \ + STORAGE_INTEGRATION_TEST_INCLUDES=tests/integration/nat/pcp/testpcp.nim +``` + +Builds the shared image and brings the compose topology up and down. Rootless, but +needs the host netfilter modules — if the router fails on iptables: +`sudo modprobe iptable_nat nf_conntrack`. + +## Expected result + +B ends up `Reachable`, the relay not running, announcing its direct address with +an active PCP mapping. Its `debug/info`: + +```json +{ + "nat": { + "reachability": "Reachable", + "clientMode": false, + "relayRunning": false, + "portMapping": "pcp" + } +} +``` + +Per-run container logs (router, bootstrap, node) are written before teardown to +`tests/integration/logs/__NAT_pcp//.log`. diff --git a/tests/integration/nat/pcp/compose.yml b/tests/integration/nat/pcp/compose.yml new file mode 100644 index 00000000..19d27558 --- /dev/null +++ b/tests/integration/nat/pcp/compose.yml @@ -0,0 +1,91 @@ +# NAT pcp scenario — see README.md. Run via testpcp.nim. +name: nat-pcp + +# Topology addresses, named for their role (defined once, referenced below). +x-addresses: + # fake public internet; a routable range so B looks public to A + wan_subnet: &wan_subnet 7.7.7.0/24 + # private network behind the NAT + lan_subnet: &lan_subnet 10.99.0.0/24 + # A: public bootstrap, relay + autonat server + bootstrap_ip: &bootstrap_ip 7.7.7.10 + # router's public face + router_wan_ip: &router_wan_ip 7.7.7.2 + # router's private face = B's gateway, and the PCP/NAT-PMP control point + router_lan_ip: &router_lan_ip 10.99.0.2 + # B, behind the NAT + node_ip: &node_ip 10.99.0.10 + +networks: + wan: + # Keep the fake public range private, not exposed to the host + internal: true + ipam: + config: + - subnet: *wan_subnet + lan: + ipam: + config: + - subnet: *lan_subnet + +services: + router: + image: localhost/storage-nat + cap_add: [NET_ADMIN] + sysctls: + net.ipv4.ip_forward: 1 + networks: + wan: + ipv4_address: *router_wan_ip + lan: + ipv4_address: *router_lan_ip + environment: + ROUTER_WAN_IP: *router_wan_ip + ROUTER_LAN_IP: *router_lan_ip + LAN_SUBNET: *lan_subnet + # scripts mounted, not baked, so editing them needs no image rebuild + volumes: + - ../router-common.sh:/scripts/router-common.sh:ro,z + - ./router-entrypoint.sh:/scripts/router-entrypoint.sh:ro,z + entrypoint: ["bash", "/scripts/router-entrypoint.sh"] + + bootstrap: + image: localhost/storage-nat + networks: + wan: + ipv4_address: *bootstrap_ip + entrypoint: ["/app/build/storage"] + command: + - --listen-ip=0.0.0.0 + - --api-bindaddr=0.0.0.0 + - --listen-port=8070 + - --disc-port=8090 + - --api-port=8080 + # bootstrap_ip (anchors can't go inside a string) + - --nat=extip:7.7.7.10 + - --relay-server + - --autonat-server + - --no-bootstrap-node + - --data-dir=/data + - --log-level=DEBUG + + node: + image: localhost/storage-nat + cap_add: [NET_ADMIN] + depends_on: [router, bootstrap] + networks: + lan: + ipv4_address: *node_ip + # B's API, published so the test can poll it + ports: + - "127.0.0.1:18083:8080" + environment: + ROUTER_LAN_IP: *router_lan_ip + # B fetches A's SPR from this API at startup to join the network (bootstrap_ip) + BOOTSTRAP_API: http://7.7.7.10:8080 + EXTRA_STORAGE_ARGS: >- + --nat-port-mapping-discover-timeout=5000 + --nat-port-mapping-timeout=5000 + volumes: + - ../node-entrypoint.sh:/scripts/node-entrypoint.sh:ro,z + entrypoint: ["bash", "/scripts/node-entrypoint.sh"] diff --git a/tests/integration/nat/pcp/router-entrypoint.sh b/tests/integration/nat/pcp/router-entrypoint.sh new file mode 100644 index 00000000..257ce6cb --- /dev/null +++ b/tests/integration/nat/pcp/router-entrypoint.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + +source "$(dirname "$0")/router-common.sh" + +# miniupnpd serves PCP/NAT-PMP (UDP 5351) on the lan face, so find it by its IP +# like router-common.sh finds the wan one. +lanif=$(ip -o -4 addr show | awk -v ip="$ROUTER_LAN_IP" '$0 ~ ip {print $2; exit}') + +# Reuse miniupnpd's chains (as nft_init.sh sets them up) without its forward drop +# policy. +nft -f - <<'EOF' +table inet filter { + chain prerouting_miniupnpd {} + chain postrouting_miniupnpd {} + chain miniupnpd {} + chain prerouting { + type nat hook prerouting priority -100; policy accept; + jump prerouting_miniupnpd + } + chain postrouting { + type nat hook postrouting priority 100; policy accept; + jump postrouting_miniupnpd + } +} +EOF + +conf=/tmp/miniupnpd.conf +cat > "$conf" </dev/null \ + || { echo "ERROR: miniupnpd failed to start" >&2; exit 1; } + +echo "router ready (wan iface $wanif, miniupnpd on $lanif)" + +hold_until_stopped diff --git a/tests/integration/nat/pcp/testpcp.nim b/tests/integration/nat/pcp/testpcp.nim new file mode 100644 index 00000000..612aec67 --- /dev/null +++ b/tests/integration/nat/pcp/testpcp.nim @@ -0,0 +1,51 @@ +## NAT pcp scenario. See README.md. + +import std/[json, os, sequtils, strutils, times] +import pkg/chronos +import pkg/questionable/results + +import ../../../asynctest +import ../../../checktest +import ../../storageclient +import ../composehelper + +proc announcesDirectAddr(info: JsonNode): bool = + ## A reachable node announces at least one direct (non-circuit) address. + info{"announceAddresses"}.getElems.anyIt("p2p-circuit" notin it.getStr) + +asyncchecksuite "NAT pcp": + let + composeFile = currentSourcePath.parentDir / "compose.yml" + nodeApiUrl = "http://127.0.0.1:18083/api/storage/v1" + suiteName = "NAT pcp" + testName = "node behind NAT maps its port over PCP and is Reachable" + services = ["router", "bootstrap", "node"] + startTime = now().format("yyyy-MM-dd'_'HH:mm:ss") + var client: StorageClient + + setup: + compose(composeFile, "up -d") + client = StorageClient.new(nodeApiUrl) + + teardown: + await client.close() + saveContainerLogs(composeFile, suiteName, testName, startTime, services) + compose(composeFile, "down -v") + + test testName: + # Reachable is the settling signal: wait for it, then assert each expected + # property separately so a failure points at the exact condition. + check eventuallyInfo(client, info{"nat"}{"reachability"}.getStr == "Reachable") + + let info = (await client.info()).get + let nat = info{"nat"} + check nat{"reachability"}.getStr == "Reachable" + check nat{"relayRunning"}.getBool == false + check nat{"portMapping"}.getStr == "pcp" + check info.announcesDirectAddr() + let announced = info{"announceAddresses"}.getElems.mapIt(it.getStr) + # PCP may map a port different from the listen port, so check the IP only + check announced.anyIt(("/ip4/" & routerWanIp & "/tcp/") in it) + # the public mapped address + # a reachable node announces its UDP address to the DHT routing record + check info{"dhtAddresses"}.getElems.len > 0 diff --git a/tests/integration/nat/reachable/README.md b/tests/integration/nat/reachable/README.md new file mode 100644 index 00000000..199a55b5 --- /dev/null +++ b/tests/integration/nat/reachable/README.md @@ -0,0 +1,63 @@ +# NAT reachable scenario + +## Scenario + +A node behind a NAT whose port is forwarded must be detected `Reachable` and +keep its direct address — no relay fallback. + +## Topology + +``` +node B ──── lan ──── router (NAT + port forward) ──── wan ──── bootstrap A +``` + +- **bootstrap A** — public node on the wan, runs the relay + autonat server. +- **router** — `lan -> wan` masquerade *plus* a static DNAT forwarding B's TCP + listen port (8070) and UDP disc port (8090) inbound. No miniupnpd: the router + opens the port itself, so B maps nothing. +- **node B** — `nat=auto`, on the lan, default route through the router. It dials + out from its listen port (8070) and the masquerade keeps that port, so A + observes it at `7.7.7.2:8070` — exactly what the DNAT forwards back, so the + dial-back reaches it. + +The wan public range and `internal` flag work as in +[not-reachable](../not-reachable/README.md). + +## Run + +Every NAT scenario: + +```bash +make testNatIntegration +``` + +Just this one — same `STORAGE_INTEGRATION_TEST_INCLUDES` filter as testIntegration, +with the test file path: + +```bash +make testNatIntegration \ + STORAGE_INTEGRATION_TEST_INCLUDES=tests/integration/nat/reachable/testreachable.nim +``` + +Builds the shared image and brings the compose topology up and down. Rootless, but +needs the host netfilter modules — if the router fails on iptables: +`sudo modprobe iptable_nat nf_conntrack`. + +## Expected result + +B ends up `Reachable`, the relay not running, announcing its direct address — +not a circuit one. Its `debug/info`: + +```json +{ + "nat": { + "reachability": "Reachable", + "clientMode": false, + "relayRunning": false, + "portMapping": "none" + } +} +``` + +Per-run container logs (router, bootstrap, node) are written before teardown to +`tests/integration/logs/__NAT_reachable//.log`. diff --git a/tests/integration/nat/reachable/compose.yml b/tests/integration/nat/reachable/compose.yml new file mode 100644 index 00000000..f5936baa --- /dev/null +++ b/tests/integration/nat/reachable/compose.yml @@ -0,0 +1,89 @@ +# NAT reachable scenario — see README.md. Run via testreachable.nim. +name: nat-reachable + +# Topology addresses, named for their role (defined once, referenced below). +x-addresses: + # fake public internet; a routable range so B looks public to A + wan_subnet: &wan_subnet 7.7.7.0/24 + # private network behind the NAT + lan_subnet: &lan_subnet 10.99.0.0/24 + # A: public bootstrap, relay + autonat server + bootstrap_ip: &bootstrap_ip 7.7.7.10 + # router's public face + router_wan_ip: &router_wan_ip 7.7.7.2 + # router's private face = B's gateway + router_lan_ip: &router_lan_ip 10.99.0.2 + # B, behind the NAT (also the DNAT target) + node_ip: &node_ip 10.99.0.10 + +networks: + wan: + # Keep the fake public range private, not exposed to the host + internal: true + ipam: + config: + - subnet: *wan_subnet + lan: + ipam: + config: + - subnet: *lan_subnet + +services: + router: + image: localhost/storage-nat + cap_add: [NET_ADMIN] + sysctls: + net.ipv4.ip_forward: 1 + networks: + wan: + ipv4_address: *router_wan_ip + lan: + ipv4_address: *router_lan_ip + environment: + ROUTER_WAN_IP: *router_wan_ip + LAN_SUBNET: *lan_subnet + # where the router forwards the port + NODE_IP: *node_ip + # scripts mounted, not baked, so editing them needs no image rebuild + volumes: + - ../router-common.sh:/scripts/router-common.sh:ro,z + - ./router-entrypoint.sh:/scripts/router-entrypoint.sh:ro,z + entrypoint: ["bash", "/scripts/router-entrypoint.sh"] + + bootstrap: + image: localhost/storage-nat + networks: + wan: + ipv4_address: *bootstrap_ip + entrypoint: ["/app/build/storage"] + command: + - --listen-ip=0.0.0.0 + - --api-bindaddr=0.0.0.0 + - --listen-port=8070 + - --disc-port=8090 + - --api-port=8080 + # bootstrap_ip (anchors can't go inside a string) + - --nat=extip:7.7.7.10 + - --relay-server + - --autonat-server + - --no-bootstrap-node + - --data-dir=/data + - --log-level=DEBUG + + node: + image: localhost/storage-nat + cap_add: [NET_ADMIN] + depends_on: [router, bootstrap] + networks: + lan: + ipv4_address: *node_ip + # B's API, published so the test can poll it + ports: + - "127.0.0.1:18081:8080" + environment: + ROUTER_LAN_IP: *router_lan_ip + # B fetches A's SPR from this API at startup to join the network (bootstrap_ip) + BOOTSTRAP_API: http://7.7.7.10:8080 + volumes: + - ../node-entrypoint.sh:/scripts/node-entrypoint.sh:ro,z + entrypoint: ["bash", "/scripts/node-entrypoint.sh"] diff --git a/tests/integration/nat/reachable/router-entrypoint.sh b/tests/integration/nat/reachable/router-entrypoint.sh new file mode 100755 index 00000000..1aa07e06 --- /dev/null +++ b/tests/integration/nat/reachable/router-entrypoint.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +source "$(dirname "$0")/router-common.sh" + +# Forward the node's TCP listen port (what AutoNAT dials back) and UDP disc port +# in order to simulate the port mapping. +iptables -t nat -A PREROUTING -i "$wanif" -p tcp --dport 8070 -j DNAT --to-destination "$NODE_IP:8070" +iptables -t nat -A PREROUTING -i "$wanif" -p udp --dport 8090 -j DNAT --to-destination "$NODE_IP:8090" + +echo "router ready (forwarding tcp/8070 + udp/8090 to $NODE_IP, wan iface $wanif)" + +hold_until_stopped diff --git a/tests/integration/nat/reachable/testreachable.nim b/tests/integration/nat/reachable/testreachable.nim new file mode 100644 index 00000000..d1f87456 --- /dev/null +++ b/tests/integration/nat/reachable/testreachable.nim @@ -0,0 +1,49 @@ +## NAT reachable scenario. See README.md. + +import std/[json, os, sequtils, strutils, times] +import pkg/chronos +import pkg/questionable/results + +import ../../../asynctest +import ../../../checktest +import ../../storageclient +import ../composehelper + +proc announcesDirectAddr(info: JsonNode): bool = + ## A reachable node announces at least one direct (non-circuit) address. + info{"announceAddresses"}.getElems.anyIt("p2p-circuit" notin it.getStr) + +asyncchecksuite "NAT reachable": + let + composeFile = currentSourcePath.parentDir / "compose.yml" + nodeApiUrl = "http://127.0.0.1:18081/api/storage/v1" + suiteName = "NAT reachable" + testName = "node behind NAT with a forwarded port is Reachable" + services = ["router", "bootstrap", "node"] + startTime = now().format("yyyy-MM-dd'_'HH:mm:ss") + var client: StorageClient + + setup: + compose(composeFile, "up -d") + client = StorageClient.new(nodeApiUrl) + + teardown: + await client.close() + saveContainerLogs(composeFile, suiteName, testName, startTime, services) + compose(composeFile, "down -v") + + test testName: + # Reachable is the settling signal: wait for it, then assert each expected + # property separately so a failure points at the exact condition. + check eventuallyInfo(client, info{"nat"}{"reachability"}.getStr == "Reachable") + + let info = (await client.info()).get + let nat = info{"nat"} + check nat{"reachability"}.getStr == "Reachable" + check nat{"relayRunning"}.getBool == false + check info.announcesDirectAddr() + let announced = info{"announceAddresses"}.getElems.mapIt(it.getStr) + check announced.anyIt(("/ip4/" & routerWanIp & "/tcp/8070") in it) + # public forwarded address + # a reachable node announces its UDP address to the DHT routing record + check info{"dhtAddresses"}.getElems.len > 0 diff --git a/tests/integration/nat/relay-download/README.md b/tests/integration/nat/relay-download/README.md new file mode 100644 index 00000000..dfa9048a --- /dev/null +++ b/tests/integration/nat/relay-download/README.md @@ -0,0 +1,43 @@ +# NAT relay-download scenario + +## Scenario + +A node behind a NAT falls back to bootstrap A's relay and announces its circuit +address. A reachable node C finds it as a provider and downloads its data +through the relay. + +## Topology + +``` +node B ──── lan ──── router (NAT) ──── wan ──── bootstrap A (relay) + └────── node C (reachable) +``` + +- **bootstrap A** — public node on the wan, autonat + relay server, started with + `--nat=extip`. +- **router** — `lan -> wan` masquerade and *no* inbound forward, so B can dial + out but nothing can dial back in. +- **node B** — `nat=auto`, on the lan. AutoNAT finds it unreachable, so it takes + a relay reservation on A and announces its circuit address. +- **node C** — `nat=auto`, directly on the wan, so AutoNAT finds it + `Reachable`. It is the peer that downloads from B through the relay. + +## Run + +```bash +make testNatIntegration \ + STORAGE_INTEGRATION_TEST_INCLUDES=tests/integration/nat/relay-download/testrelaydownload.nim +``` + +Builds the shared image and brings the compose topology up and down. Rootless, but +needs the host netfilter modules — if the router fails on iptables: +`sudo modprobe iptable_nat nf_conntrack`. + +## Expected result + +B is `NotReachable` and announces its circuit address, while C is `Reachable`. +B uploads a file, then C fetches it over the network through the relay and gets +the same content back. + +Per-run container logs (router, bootstrap, client, node) are written before teardown to +`tests/integration/logs/__NAT_relay_download//.log`. diff --git a/tests/integration/nat/relay-download/compose.yml b/tests/integration/nat/relay-download/compose.yml new file mode 100644 index 00000000..2e2652c7 --- /dev/null +++ b/tests/integration/nat/relay-download/compose.yml @@ -0,0 +1,107 @@ +# NAT relay-download scenario — see README.md. Run via testrelaydownload.nim. +name: nat-relay-download + +# Topology addresses, named for their role (defined once, referenced below). +x-addresses: + # fake public internet; a routable range so B looks public to A + wan_subnet: &wan_subnet 7.7.7.0/24 + # private network behind the NAT + lan_subnet: &lan_subnet 10.99.0.0/24 + # A: public bootstrap, autonat + relay server + bootstrap_ip: &bootstrap_ip 7.7.7.10 + # C: public node on the wan, reachable, the one that downloads from B + client_ip: &client_ip 7.7.7.20 + # router's public face + router_wan_ip: &router_wan_ip 7.7.7.2 + # router's private face = B's gateway + router_lan_ip: &router_lan_ip 10.99.0.2 + # B, behind the NAT + node_ip: &node_ip 10.99.0.10 + +networks: + wan: + # Keep the fake public range private, not exposed to the host + internal: true + ipam: + config: + - subnet: *wan_subnet + lan: + ipam: + config: + - subnet: *lan_subnet + +services: + router: + image: localhost/storage-nat + cap_add: [NET_ADMIN] + sysctls: + net.ipv4.ip_forward: 1 + networks: + wan: + ipv4_address: *router_wan_ip + lan: + ipv4_address: *router_lan_ip + environment: + ROUTER_WAN_IP: *router_wan_ip + LAN_SUBNET: *lan_subnet + # scripts mounted, so editing them needs no image rebuild + volumes: + - ../router-common.sh:/scripts/router-common.sh:ro,z + - ./router-entrypoint.sh:/scripts/router-entrypoint.sh:ro,z + entrypoint: ["bash", "/scripts/router-entrypoint.sh"] + + bootstrap: + image: localhost/storage-nat + networks: + wan: + ipv4_address: *bootstrap_ip + entrypoint: ["/app/build/storage"] + command: + - --listen-ip=0.0.0.0 + - --api-bindaddr=0.0.0.0 + - --listen-port=8070 + - --disc-port=8090 + - --api-port=8080 + # bootstrap_ip (anchors can't go inside a string) + - --nat=extip:7.7.7.10 + - --relay-server + - --autonat-server + - --no-bootstrap-node + - --data-dir=/data + - --log-level=DEBUG + + # C sits on the wan, directly reachable: no NAT, so it leaves ROUTER_LAN_IP + # unset and keeps its default route (see node-entrypoint.sh). + client: + image: localhost/storage-nat + depends_on: [bootstrap] + networks: + wan: + ipv4_address: *client_ip + # C's API, published so the test can drive the download from C and poll it + ports: + - "127.0.0.1:18087:8080" + environment: + # C fetches A's SPR from this API at startup to join the network (bootstrap_ip) + BOOTSTRAP_API: http://7.7.7.10:8080 + volumes: + - ../node-entrypoint.sh:/scripts/node-entrypoint.sh:ro,z + entrypoint: ["bash", "/scripts/node-entrypoint.sh"] + + node: + image: localhost/storage-nat + cap_add: [NET_ADMIN] + depends_on: [router, bootstrap] + networks: + lan: + ipv4_address: *node_ip + # B's API, published so the test can upload to it and poll it + ports: + - "127.0.0.1:18086:8080" + environment: + ROUTER_LAN_IP: *router_lan_ip + # B fetches A's SPR from this API at startup to join the network (bootstrap_ip) + BOOTSTRAP_API: http://7.7.7.10:8080 + volumes: + - ../node-entrypoint.sh:/scripts/node-entrypoint.sh:ro,z + entrypoint: ["bash", "/scripts/node-entrypoint.sh"] diff --git a/tests/integration/nat/relay-download/router-entrypoint.sh b/tests/integration/nat/relay-download/router-entrypoint.sh new file mode 100755 index 00000000..21a8ef3b --- /dev/null +++ b/tests/integration/nat/relay-download/router-entrypoint.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +source "$(dirname "$0")/router-common.sh" + +echo "router ready (wan iface $wanif)" + +hold_until_stopped diff --git a/tests/integration/nat/relay-download/testrelaydownload.nim b/tests/integration/nat/relay-download/testrelaydownload.nim new file mode 100644 index 00000000..0f7349b5 --- /dev/null +++ b/tests/integration/nat/relay-download/testrelaydownload.nim @@ -0,0 +1,62 @@ +## NAT relay-download scenario. See README.md. + +import std/[json, os, sequtils, strutils, times] +import pkg/chronos +import pkg/questionable/results + +import ../../../asynctest +import ../../../checktest +import ../../storageclient +import ../composehelper + +proc announcesCircuitAddr(info: JsonNode): bool = + ## A node behind the relay announces its circuit (p2p-circuit) address. + info{"announceAddresses"}.getElems.anyIt("p2p-circuit" in it.getStr) + +asyncchecksuite "NAT relay download": + let + composeFile = currentSourcePath.parentDir / "compose.yml" + nodeApiUrl = "http://127.0.0.1:18086/api/storage/v1" + clientApiUrl = "http://127.0.0.1:18087/api/storage/v1" + suiteName = "NAT relay download" + testName = "a NAT'd node behind a relay can be downloaded from" + services = ["router", "bootstrap", "client", "node"] + startTime = now().format("yyyy-MM-dd'_'HH:mm:ss") + var + nodeClient: StorageClient + clientC: StorageClient + + setup: + compose(composeFile, "up -d") + nodeClient = StorageClient.new(nodeApiUrl) + clientC = StorageClient.new(clientApiUrl) + + teardown: + await nodeClient.close() + await clientC.close() + saveContainerLogs(composeFile, suiteName, testName, startTime, services) + compose(composeFile, "down -v") + + test testName: + # B is NotReachable and falls back to the relay, announcing its circuit address + check eventuallyInfo( + nodeClient, + info{"nat"}{"reachability"}.getStr == "NotReachable" and + info.announcesCircuitAddr(), + ) + + let info = (await nodeClient.info()).get + # Double check B announces only its circuit address + check info.announcesCircuitAddr() + + # C is reachable + check eventuallyInfo(clientC, info{"nat"}{"reachability"}.getStr == "Reachable") + + # B uploads a file + let content = "hello from behind the relay" + let cid = (await nodeClient.upload(content)).get + + # C downloads it through the relay and gets the same content back + let res = await clientC.download(cid) + check res.isOk + check res.get == content diff --git a/tests/integration/nat/router-common.sh b/tests/integration/nat/router-common.sh new file mode 100644 index 00000000..66461f77 --- /dev/null +++ b/tests/integration/nat/router-common.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +# Shared router base, sourced by each scenario's router-entrypoint.sh + +set -euo pipefail + +# iptables needs the wan interface's name (eth0/eth1), but podman assigns those +# names arbitrarily — so look for the name using the wan IP, +# defined in the compose file. +wanif=$(ip -o -4 addr show | awk -v ip="$ROUTER_WAN_IP" '$0 ~ ip {print $2; exit}') + +if ! iptables -t nat -A POSTROUTING -s "$LAN_SUBNET" -o "$wanif" -j MASQUERADE; then + echo "ERROR: iptables NAT failed. Load netfilter modules on the host:" >&2 + echo " sudo modprobe iptable_nat nf_conntrack" >&2 + exit 1 +fi +iptables -P FORWARD ACCEPT + +# Block until `compose down`. sleep runs in the background so the SIGTERM trap +# fires immediately instead of waiting for sleep to return. +hold_until_stopped() { + trap 'exit 0' TERM INT + sleep infinity & + wait +} diff --git a/tests/integration/nat/upnp/README.md b/tests/integration/nat/upnp/README.md new file mode 100644 index 00000000..083316bf --- /dev/null +++ b/tests/integration/nat/upnp/README.md @@ -0,0 +1,64 @@ +# NAT upnp scenario + +## Scenario + +A node behind a NAT becomes `Reachable` by mapping its port over UPnP — the +router forwards nothing on its own, the node asks for the mapping and no relay is +needed. + +## Topology + +``` +node B ──── lan ──── router (NAT + miniupnpd) ──── wan ──── bootstrap A +``` + +- **bootstrap A** — public node on the wan, runs the relay + autonat server. +- **router** — `lan -> wan` masquerade and *no* static forward. It runs + `miniupnpd` (real nftables backend) as the UPnP gateway, with PCP/NAT-PMP + disabled so libplum falls back to UPnP. +- **node B** — `nat=auto`, on the lan. First detected `NotReachable`, it maps its + TCP listen (8070) and UDP disc (8090) ports over UPnP; the resulting DNAT lets + A's dial-back reach it, so the next AutoNAT round flips it to `Reachable`. + +The wan public range and `internal` flag work as in +[not-reachable](../not-reachable/README.md); the public wan IP also keeps +miniupnpd from treating the setup as double-NAT and refusing to forward. + +## Run + +Every NAT scenario: + +```bash +make testNatIntegration +``` + +Just this one — same `STORAGE_INTEGRATION_TEST_INCLUDES` filter as testIntegration, +with the test file path: + +```bash +make testNatIntegration \ + STORAGE_INTEGRATION_TEST_INCLUDES=tests/integration/nat/upnp/testupnp.nim +``` + +Builds the shared image and brings the compose topology up and down. Rootless, but +needs the host netfilter modules — if the router fails on iptables: +`sudo modprobe iptable_nat nf_conntrack`. + +## Expected result + +B ends up `Reachable`, the relay not running, announcing its direct address with +an active UPnP mapping. Its `debug/info`: + +```json +{ + "nat": { + "reachability": "Reachable", + "clientMode": false, + "relayRunning": false, + "portMapping": "upnp" + } +} +``` + +Per-run container logs (router, bootstrap, node) are written before teardown to +`tests/integration/logs/__NAT_upnp//.log`. diff --git a/tests/integration/nat/upnp/compose.yml b/tests/integration/nat/upnp/compose.yml new file mode 100644 index 00000000..8de469f1 --- /dev/null +++ b/tests/integration/nat/upnp/compose.yml @@ -0,0 +1,91 @@ +# NAT upnp scenario — see README.md. Run via testupnp.nim. +name: nat-upnp + +# Topology addresses, named for their role (defined once, referenced below). +x-addresses: + # fake public internet; a routable range so B looks public to A + wan_subnet: &wan_subnet 7.7.7.0/24 + # private network behind the NAT + lan_subnet: &lan_subnet 10.99.0.0/24 + # A: public bootstrap, relay + autonat server + bootstrap_ip: &bootstrap_ip 7.7.7.10 + # router's public face + router_wan_ip: &router_wan_ip 7.7.7.2 + # router's private face = B's gateway, and the UPnP control point + router_lan_ip: &router_lan_ip 10.99.0.2 + # B, behind the NAT + node_ip: &node_ip 10.99.0.10 + +networks: + wan: + # Keep the fake public range private, not exposed to the host + internal: true + ipam: + config: + - subnet: *wan_subnet + lan: + ipam: + config: + - subnet: *lan_subnet + +services: + router: + image: localhost/storage-nat + cap_add: [NET_ADMIN] + sysctls: + net.ipv4.ip_forward: 1 + networks: + wan: + ipv4_address: *router_wan_ip + lan: + ipv4_address: *router_lan_ip + environment: + ROUTER_WAN_IP: *router_wan_ip + ROUTER_LAN_IP: *router_lan_ip + LAN_SUBNET: *lan_subnet + # scripts mounted, so editing them needs no image rebuild + volumes: + - ../router-common.sh:/scripts/router-common.sh:ro,z + - ./router-entrypoint.sh:/scripts/router-entrypoint.sh:ro,z + entrypoint: ["bash", "/scripts/router-entrypoint.sh"] + + bootstrap: + image: localhost/storage-nat + networks: + wan: + ipv4_address: *bootstrap_ip + entrypoint: ["/app/build/storage"] + command: + - --listen-ip=0.0.0.0 + - --api-bindaddr=0.0.0.0 + - --listen-port=8070 + - --disc-port=8090 + - --api-port=8080 + # bootstrap_ip (anchors can't go inside a string) + - --nat=extip:7.7.7.10 + - --relay-server + - --autonat-server + - --no-bootstrap-node + - --data-dir=/data + - --log-level=DEBUG + + node: + image: localhost/storage-nat + cap_add: [NET_ADMIN] + depends_on: [router, bootstrap] + networks: + lan: + ipv4_address: *node_ip + # B's API, published so the test can poll it + ports: + - "127.0.0.1:18082:8080" + environment: + ROUTER_LAN_IP: *router_lan_ip + # B fetches A's SPR from this API at startup to join the network (bootstrap_ip) + BOOTSTRAP_API: http://7.7.7.10:8080 + EXTRA_STORAGE_ARGS: >- + --nat-port-mapping-discover-timeout=5000 + --nat-port-mapping-timeout=5000 + volumes: + - ../node-entrypoint.sh:/scripts/node-entrypoint.sh:ro,z + entrypoint: ["bash", "/scripts/node-entrypoint.sh"] diff --git a/tests/integration/nat/upnp/router-entrypoint.sh b/tests/integration/nat/upnp/router-entrypoint.sh new file mode 100644 index 00000000..64b66598 --- /dev/null +++ b/tests/integration/nat/upnp/router-entrypoint.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +source "$(dirname "$0")/router-common.sh" + +# miniupnpd listens for UPnP (SSDP) on the lan face, so find it by its IP like +# router-common.sh finds the wan one. +lanif=$(ip -o -4 addr show | awk -v ip="$ROUTER_LAN_IP" '$0 ~ ip {print $2; exit}') + +# Reuse miniupnpd's chains (as nft_init.sh sets them up) without its forward drop +# policy. +nft -f - <<'EOF' +table inet filter { + chain prerouting_miniupnpd {} + chain postrouting_miniupnpd {} + chain miniupnpd {} + chain prerouting { + type nat hook prerouting priority -100; policy accept; + jump prerouting_miniupnpd + } + chain postrouting { + type nat hook postrouting priority 100; policy accept; + jump postrouting_miniupnpd + } +} +EOF + +conf=/tmp/miniupnpd.conf +cat > "$conf" </dev/null \ + || { echo "ERROR: miniupnpd failed to start" >&2; exit 1; } + +echo "router ready (wan iface $wanif, miniupnpd on $lanif)" + +hold_until_stopped diff --git a/tests/integration/nat/upnp/testupnp.nim b/tests/integration/nat/upnp/testupnp.nim new file mode 100644 index 00000000..8e72e5de --- /dev/null +++ b/tests/integration/nat/upnp/testupnp.nim @@ -0,0 +1,50 @@ +## NAT upnp scenario. See README.md. + +import std/[json, os, sequtils, strutils, times] +import pkg/chronos +import pkg/questionable/results + +import ../../../asynctest +import ../../../checktest +import ../../storageclient +import ../composehelper + +proc announcesDirectAddr(info: JsonNode): bool = + ## A reachable node announces at least one direct (non-circuit) address. + info{"announceAddresses"}.getElems.anyIt("p2p-circuit" notin it.getStr) + +asyncchecksuite "NAT upnp": + let + composeFile = currentSourcePath.parentDir / "compose.yml" + nodeApiUrl = "http://127.0.0.1:18082/api/storage/v1" + suiteName = "NAT upnp" + testName = "node behind NAT maps its port over UPnP and is Reachable" + services = ["router", "bootstrap", "node"] + startTime = now().format("yyyy-MM-dd'_'HH:mm:ss") + var client: StorageClient + + setup: + compose(composeFile, "up -d") + client = StorageClient.new(nodeApiUrl) + + teardown: + await client.close() + saveContainerLogs(composeFile, suiteName, testName, startTime, services) + compose(composeFile, "down -v") + + test testName: + # Reachable is the settling signal: wait for it, then assert each expected + # property separately so a failure points at the exact condition. + check eventuallyInfo(client, info{"nat"}{"reachability"}.getStr == "Reachable") + + let info = (await client.info()).get + let nat = info{"nat"} + check nat{"reachability"}.getStr == "Reachable" + check nat{"relayRunning"}.getBool == false + check nat{"portMapping"}.getStr == "upnp" + check info.announcesDirectAddr() + let announced = info{"announceAddresses"}.getElems.mapIt(it.getStr) + check announced.anyIt(("/ip4/" & routerWanIp & "/tcp/8070") in it) + # public mapped address + # a reachable node announces its UDP address to the DHT routing record + check info{"dhtAddresses"}.getElems.len > 0 diff --git a/tests/integration/storageclient.nim b/tests/integration/storageclient.nim index ec990bb9..17931f34 100644 --- a/tests/integration/storageclient.nim +++ b/tests/integration/storageclient.nim @@ -1,4 +1,5 @@ import std/strutils +import std/sequtils from pkg/libp2p import Cid, `$`, init import pkg/questionable/results @@ -68,16 +69,6 @@ proc delete( .} = return self.request(MethodDelete, url, headers = headers) -proc patch*( - self: StorageClient, - url: string, - body: string = "", - headers: seq[HttpHeaderTuple] = @[], -): Future[HttpClientResponseRef] {. - async: (raw: true, raises: [CancelledError, HttpError]) -.} = - return self.request(MethodPatch, url, headers = headers, body = body) - proc body*( response: HttpClientResponseRef ): Future[string] {.async: (raises: [CancelledError, HttpError]).} = @@ -229,20 +220,6 @@ proc list*( RestContentList.fromJson(await response.body) -proc space*( - client: StorageClient -): Future[?!RestRepoStore] {.async: (raises: [CancelledError, HttpError]).} = - let url = client.baseurl & "/space" - let response = await client.get(url) - - if response.status != 200: - return failure($response.status) - - RestRepoStore.fromJson(await response.body) - -proc buildUrl*(client: StorageClient, path: string): string = - return client.baseurl & path - proc hasBlock*( client: StorageClient, cid: Cid ): Future[?!bool] {.async: (raises: [CancelledError, HttpError]).} = diff --git a/tests/integration/storageconfig.nim b/tests/integration/storageconfig.nim index 4aeb6d60..8c50fa0d 100644 --- a/tests/integration/storageconfig.nim +++ b/tests/integration/storageconfig.nim @@ -6,7 +6,6 @@ import std/sugar import std/tables from pkg/chronicles import LogLevel import pkg/storage/conf -import pkg/storage/units import pkg/confutils import pkg/confutils/defs import libp2p except setup @@ -234,49 +233,33 @@ proc withBlockMaintenanceInterval*( config.addCliOption("--block-mi", $interval) return startConfig -proc logLevelWithTopics( - config: StorageConfig, topics: varargs[string] -): string {.raises: [StorageConfigError].} = - convertError: - var logLevel = LogLevel.INFO - let built = config.buildConfig("Invalid storage config cli params") - logLevel = parseEnum[LogLevel](built.logLevel.toUpperAscii) - let level = $logLevel & ";TRACE: " & topics.join(",") - return level - -proc withLogTopics*( - self: StorageConfigs, idx: int, topics: varargs[string] -): StorageConfigs {.raises: [StorageConfigError].} = - self.checkBounds idx - - convertError: - let config = self.configs[idx] - let level = config.logLevelWithTopics(topics) - var startConfig = self - return startConfig.withLogLevel(idx, level) - -proc withLogTopics*( - self: StorageConfigs, topics: varargs[string] -): StorageConfigs {.raises: [StorageConfigError].} = - var startConfig = self - for config in startConfig.configs.mitems: - let level = config.logLevelWithTopics(topics) - config = config.withLogLevel(level) - return startConfig - -proc withStorageQuota*( - self: StorageConfigs, idx: int, quota: NBytes +proc withExtIp*( + self: StorageConfigs, idx: int, ip = "127.0.0.1" ): StorageConfigs {.raises: [StorageConfigError].} = self.checkBounds idx var startConfig = self - startConfig.configs[idx].addCliOption("--storage-quota", $quota) + startConfig.configs[idx].addCliOption("--nat", "extip:" & ip) return startConfig -proc withStorageQuota*( - self: StorageConfigs, quota: NBytes +# For testing, a node with extip (not behind nat) with autonat server +# enabled is a bootstrap node +proc isBootstrapNode*(config: StorageConfig): bool {.raises: [].} = + let opts = config.cliOptions.getOrDefault(StartUpCmd.noCmd) + + try: + if "--nat" in opts and "extip" in opts["--nat"].value and "--autonat-server" in opts: + return true + except KeyError: + warn "Failed to look at the extip config" + return false + + return false + +proc withAutonatServer*( + self: StorageConfigs, idx: int ): StorageConfigs {.raises: [StorageConfigError].} = + self.checkBounds idx var startConfig = self - for config in startConfig.configs.mitems: - config.addCliOption("--storage-quota", $quota) + startConfig.configs[idx].addCliOption("--autonat-server") return startConfig diff --git a/tests/integration/twonodes.nim b/tests/integration/twonodes.nim index 7fa55244..e2553aa6 100644 --- a/tests/integration/twonodes.nim +++ b/tests/integration/twonodes.nim @@ -12,7 +12,10 @@ export multinodes template twonodessuite*(name: string, body: untyped) = multinodesuite name: let twoNodesConfig {.inject, used.} = - NodeConfigs(clients: StorageConfigs.init(nodes = 2).some) + # Disable Autonat for this suite + NodeConfigs( + clients: StorageConfigs.init(nodes = 2).withExtIp(1).withAutonatServer(0).some + ) var node1 {.inject, used.}: StorageProcess var node2 {.inject, used.}: StorageProcess diff --git a/tests/storage/helpers/nodeutils.nim b/tests/storage/helpers/nodeutils.nim index fdaf6162..97d43add 100644 --- a/tests/storage/helpers/nodeutils.nim +++ b/tests/storage/helpers/nodeutils.nim @@ -11,8 +11,6 @@ import pkg/storage/stores import pkg/storage/blocktype as bt import pkg/storage/blockexchange import pkg/storage/systemclock -import pkg/storage/nat -import pkg/storage/utils/natutils import pkg/storage/merkletree import pkg/storage/manifest @@ -226,15 +224,10 @@ proc generateNodes*( if config.enableBootstrap: waitFor switch.peerInfo.update() - let (announceAddrs, discoveryAddrs) = nattedAddress( - NatConfig(hasExtIp: false, nat: NatNone), - switch.peerInfo.addrs, - bindPort.Port, + blockDiscovery.announceDirectAddrs( + switch.peerInfo.addrs, udpPort = bindPort.Port ) - blockDiscovery.updateAnnounceRecord(announceAddrs) - blockDiscovery.updateDhtRecord(discoveryAddrs) - if blockDiscovery.dhtRecord.isSome: - bootstrapNodes.add !blockDiscovery.dhtRecord + bootstrapNodes.add blockDiscovery.getSpr() fullNode else: diff --git a/tests/storage/natsimulation.nim b/tests/storage/natsimulation.nim new file mode 100644 index 00000000..19ec0a33 --- /dev/null +++ b/tests/storage/natsimulation.nim @@ -0,0 +1,187 @@ +# NAT simulation for integration testing. +# +# It simulates the filtering behaviors (endpoint-independent, address-dependent, +# address-and-port-dependent, double NAT) at the connection level, so the full +# AutoNAT detection and relay stack can be exercised without actual NAT hardware. + +{.push raises: [].} + +import std/[options, sequtils] +import pkg/chronos +import pkg/chronicles +import pkg/results +import pkg/libp2p +import pkg/libp2p/transports/tcptransport +import pkg/libp2p/transports/transport +import pkg/libp2p/wire + +import ../../storage/nat + +logScope: + topics = "nat simulation" + +type FilteringBehavior* = enum + EndpointIndependent + AddressDependent + AddressAndPortDependent + DoubleNat + +type NatRouter* = ref object + filtering*: FilteringBehavior + conntrack: seq[TransportAddress] # remote addrs we dialed; allows them to connect back + natMapper*: Option[NatPortMapper] + +type NatTransport* = ref object of Transport + tcp: TcpTransport + router: NatRouter + +proc fromString*( + T: type FilteringBehavior, s: string +): Result[FilteringBehavior, string] = + case s + of "endpoint-independent": + ok(EndpointIndependent) + of "address-dependent": + ok(AddressDependent) + of "address-and-port-dependent": + ok(AddressAndPortDependent) + of "double-nat": + ok(DoubleNat) + else: + err("Unknown filtering behavior: " & s) + +proc new*(T: type NatRouter, filtering: FilteringBehavior): T = + T(filtering: filtering) + +proc setFiltering*(r: NatRouter, filtering: FilteringBehavior) = + debug "NAT filtering changed", previous = r.filtering, next = filtering + + r.filtering = filtering + r.conntrack = @[] + +proc allowInbound(r: NatRouter, remote: TransportAddress, localPort: Port): bool = + case r.filtering + of DoubleNat: + return + false + # always blocks: simulates a scenario where inbound connections are never possible + of EndpointIndependent: + return true + else: + discard + + if r.natMapper.isSome and r.natMapper.get.portMapping.isSome and + r.natMapper.get.portMapping.get.activeTcpPort == localPort: + return true + + case r.filtering + of AddressDependent: + r.conntrack.anyIt( + try: + it.address == remote.address + except ValueError: + false + ) + of AddressAndPortDependent: + remote in r.conntrack + else: + false + +proc new*( + T: type NatTransport, + router: NatRouter, + upgrade: Upgrade, + flags: set[ServerFlags] = {}, +): T = + let self = T(tcp: TcpTransport.new(flags, upgrade), upgrader: upgrade, router: router) + procCall Transport(self).initialize() + return self + +method start*( + self: NatTransport, addrs: seq[MultiAddress] +) {.async: (raises: [LPError, transport.TransportError, CancelledError]).} = + await self.tcp.start(addrs) + self.addrs = self.tcp.addrs + self.running = true + self.onRunning.fire() + +method stop*(self: NatTransport) {.async: (raises: []).} = + await self.tcp.stop() + self.running = false + self.onStop.fire() + +method dial*( + self: NatTransport, + hostname: string, + address: MultiAddress, + peerId: Opt[PeerId] = Opt.none(PeerId), +): Future[Connection] {.async: (raises: [transport.TransportError, CancelledError]).} = + ## establishes an outgoing TCP connection and records the remote address + ## so it can connect back to us later + let conn = await self.tcp.dial(hostname, address) + + if conn.observedAddr.isSome: + let transportAddr = initTAddress(conn.observedAddr.get) + if transportAddr.isOk: + let remote = transportAddr.get + self.router.conntrack.add(remote) + proc cleanupConntrack() {.async: (raises: []).} = + await noCancel conn.closeEvent.wait() + self.router.conntrack.keepItIf(it != remote) + + asyncSpawn cleanupConntrack() + + return conn + +method accept*( + self: NatTransport +): Future[Connection] {.async: (raises: [transport.TransportError, CancelledError]).} = + ## waits for an incoming TCP connection and applies the NAT filtering rules + while true: + let conn = await self.tcp.accept() + + if self.router.filtering == EndpointIndependent: + return conn + + if conn.observedAddr.isNone: + await conn.close() + continue + + let transportAddr = initTAddress(conn.observedAddr.get) + if transportAddr.isErr: + debug "Dropping inbound connection: invalid observed address", + address = conn.observedAddr.get + await conn.close() + continue + + var localPort = Port(0) + if conn.localAddr.isSome: + # Local address read from the accepted socket. + let localAddr = initTAddress(conn.localAddr.get) + if localAddr.isOk: + localPort = localAddr.get.port + + if not self.router.allowInbound(transportAddr.get, localPort): + # The rejected connection is not closed here: tcp.stop() closes all + # accepted TCP connections on teardown. + continue + + debug "Inbound connection accepted", + remote = transportAddr.get, filtering = self.router.filtering + return conn + +method handles*( + self: NatTransport, address: MultiAddress +): bool {.gcsafe, raises: [].} = + ## returns true if this transport handles the given address (TCP only) + if procCall Transport(self).handles(address): + if address.protocols.isOk: + return TCP.match(address) + +proc withNatTransport*( + b: SwitchBuilder, router: NatRouter, flags: set[ServerFlags] = {} +): SwitchBuilder = + b.withTransport( + proc(config: TransportConfig): Transport = + NatTransport.new(router, config.upgr, flags) + ) diff --git a/tests/storage/node/helpers.nim b/tests/storage/node/helpers.nim index af4012b8..d5e9678f 100644 --- a/tests/storage/node/helpers.nim +++ b/tests/storage/node/helpers.nim @@ -7,7 +7,6 @@ import pkg/storage/chunker import pkg/storage/stores import ../../asynctest -import ../helpers/switchutils type CountingStore* = ref object of NetworkStore lookups*: Table[Cid, int] diff --git a/tests/storage/testaddrutils.nim b/tests/storage/testaddrutils.nim new file mode 100644 index 00000000..d9a4e91e --- /dev/null +++ b/tests/storage/testaddrutils.nim @@ -0,0 +1,79 @@ +import std/[net, options] +import pkg/libp2p/multiaddress +import ../asynctest +import ../../storage/utils/addrutils + +const relayId = "16Uiu2HAkyRvHo1AyyQY1xiHC8QbYjXCHkZbneVC8dBtJjp1SZcGD" + +proc circuitAddr(relayIp: string): MultiAddress = + MultiAddress + .init("/ip4/" & relayIp & "/tcp/8070/p2p/" & relayId & "/p2p-circuit") + .expect("valid") + +suite "addrutils - getTcpPort": + test "extracts port from ipv4 tcp address": + let ma = MultiAddress.init("/ip4/1.2.3.4/tcp/5000").expect("valid") + check getTcpPort(ma) == some(Port(5000)) + + test "extracts port from ipv6 tcp address": + let ma = MultiAddress.init("/ip6/::1/tcp/8080").expect("valid") + check getTcpPort(ma) == some(Port(8080)) + + test "returns none for udp address": + let ma = MultiAddress.init("/ip4/1.2.3.4/udp/5000").expect("valid") + check getTcpPort(ma) == Port.none + + test "extracts port 0": + let ma = MultiAddress.init("/ip4/0.0.0.0/tcp/0").expect("valid") + check getTcpPort(ma) == some(Port(0)) + +suite "addrutils - remapAddr": + test "replaces protocol tcp with udp": + let ma = MultiAddress.init("/ip4/1.2.3.4/tcp/5000").expect("valid") + let remapped = ma.remapAddr(protocol = some("udp"), port = some(Port(9000))) + check remapped == MultiAddress.init("/ip4/1.2.3.4/udp/9000").expect("valid") + + test "replaces only port, keeping protocol": + let ma = MultiAddress.init("/ip4/1.2.3.4/tcp/5000").expect("valid") + let remapped = ma.remapAddr(port = some(Port(9000))) + check remapped == MultiAddress.init("/ip4/1.2.3.4/tcp/9000").expect("valid") + + test "replaces only ip, keeping protocol and port": + let ma = MultiAddress.init("/ip4/1.2.3.4/tcp/5000").expect("valid") + let remapped = ma.remapAddr(ip = some(parseIpAddress("8.8.8.8"))) + check remapped == MultiAddress.init("/ip4/8.8.8.8/tcp/5000").expect("valid") + +suite "addrutils - hasPublicRelayTransport": + test "true when the relay has a public ip": + check circuitAddr("204.168.234.45").hasPublicRelayTransport() + + test "false when the relay is loopback": + check not circuitAddr("127.0.0.1").hasPublicRelayTransport() + + test "false when the relay is a private ip": + check not circuitAddr("172.17.0.1").hasPublicRelayTransport() + +suite "addrutils - dialableAddressPolicy": + test "keeps a public direct address": + check MultiAddress + .init("/ip4/204.168.234.45/tcp/8070") + .expect("valid") + .dialableAddressPolicy() + + test "drops a loopback direct address": + check not MultiAddress + .init("/ip4/127.0.0.1/tcp/8070") + .expect("valid") + .dialableAddressPolicy() + + test "drops a private direct address": + check not MultiAddress + .init("/ip4/192.168.100.103/tcp/8070") + .expect("valid") + .dialableAddressPolicy() + + test "keeps a circuit address through a public relay": + check circuitAddr("204.168.234.45").dialableAddressPolicy() + + test "drops a circuit address through a private relay": + check not circuitAddr("172.17.0.1").dialableAddressPolicy() diff --git a/tests/storage/testconf.nim b/tests/storage/testconf.nim new file mode 100644 index 00000000..836135d5 --- /dev/null +++ b/tests/storage/testconf.nim @@ -0,0 +1,149 @@ +import std/net +import ../asynctest +import ./helpers +import ../../storage/conf +import ../../storage/nat + +proc validConfig(): StorageConf = + StorageConf( + nat: defaultNatConfig(), + natMaxQueueSize: 3, + natNumPeersToAsk: 5, + natMinConfidence: 0.7, + natObservedAddrMinCount: 1, + natScheduleInterval: DefaultNatScheduleInterval, + natMaxRelays: 2, + natPortMappingDiscoverTimeout: 500, + natPortMappingTimeout: 500, + natPortMappingRecheckPeriod: 300000, + ) + +suite "Conf - validateAutonatConfig": + test "accepts a valid config": + check validConfig().validateAutonatConfig().isOk + + test "rejects autonat server without extip": + var config = validConfig() + config.autonatServer = true + + check config.validateAutonatConfig().isErr + + test "accepts autonat server with extip": + var config = validConfig() + config.autonatServer = true + config.nat = nat.NatConfig(hasExtIp: true, extIp: parseIpAddress("1.2.3.4")) + + check config.validateAutonatConfig().isOk + + test "rejects relay server without extip": + var config = validConfig() + config.isRelayServer = true + + check config.validateAutonatConfig().isErr + + test "accepts relay server with extip": + var config = validConfig() + config.isRelayServer = true + config.nat = nat.NatConfig(hasExtIp: true, extIp: parseIpAddress("1.2.3.4")) + + check config.validateAutonatConfig().isOk + + test "rejects no-bootstrap-node without extip": + var config = validConfig() + config.noBootstrapNode = true + + check config.validateAutonatConfig().isErr + + test "accepts no-bootstrap-node with extip": + var config = validConfig() + config.noBootstrapNode = true + config.nat = nat.NatConfig(hasExtIp: true, extIp: parseIpAddress("1.2.3.4")) + + check config.validateAutonatConfig().isOk + + test "rejects nat-max-queue-size below 1": + var config = validConfig() + config.natMaxQueueSize = 0 + + check config.validateAutonatConfig().isErr + + test "accepts nat-max-queue-size of 1": + var config = validConfig() + config.natMaxQueueSize = 1 + + check config.validateAutonatConfig().isOk + + test "rejects nat-num-peers-to-ask below 1": + var config = validConfig() + config.natNumPeersToAsk = 0 + + check config.validateAutonatConfig().isErr + + test "accepts nat-num-peers-to-ask of 1": + var config = validConfig() + config.natNumPeersToAsk = 1 + + check config.validateAutonatConfig().isOk + + test "rejects nat-observed-addr-min-count below 1": + var config = validConfig() + config.natObservedAddrMinCount = 0 + + check config.validateAutonatConfig().isErr + + test "accepts nat-observed-addr-min-count of 1": + var config = validConfig() + config.natObservedAddrMinCount = 1 + + check config.validateAutonatConfig().isOk + + test "rejects negative nat-min-confidence": + var config = validConfig() + config.natMinConfidence = -0.1 + + check config.validateAutonatConfig().isErr + + test "rejects nat-min-confidence above 1": + var config = validConfig() + config.natMinConfidence = 1.1 + + check config.validateAutonatConfig().isErr + + test "accepts nat-min-confidence bounds": + var config = validConfig() + + config.natMinConfidence = 0.0 + check config.validateAutonatConfig().isOk + + config.natMinConfidence = 1.0 + check config.validateAutonatConfig().isOk + + test "rejects nat-schedule-interval of zero": + var config = validConfig() + config.natScheduleInterval = 0.seconds + + check config.validateAutonatConfig().isErr + + test "rejects nat-max-relays below 1": + var config = validConfig() + config.natMaxRelays = 0 + + check config.validateAutonatConfig().isErr + + test "rejects nat-port-mapping-discover-timeout of zero": + var config = validConfig() + config.natPortMappingDiscoverTimeout = 0 + + check config.validateAutonatConfig().isErr + + test "rejects nat-port-mapping-timeout of zero": + var config = validConfig() + config.natPortMappingTimeout = 0 + + check config.validateAutonatConfig().isErr + + test "rejects nat-port-mapping-recheck-period of zero": + var config = validConfig() + config.natPortMappingRecheckPeriod = 0 + + check config.validateAutonatConfig().isErr diff --git a/tests/storage/testdiscovery.nim b/tests/storage/testdiscovery.nim new file mode 100644 index 00000000..f0cb5a3a --- /dev/null +++ b/tests/storage/testdiscovery.nim @@ -0,0 +1,40 @@ +import std/[net, sequtils] +import pkg/libp2p/[multiaddress, routing_record] + +import ../asynctest +import ./helpers +import ../../storage/discovery +import ../../storage/rng + +suite "Discovery - SPR record logic": + var key: PrivateKey + var disc: Discovery + + let + directAddr = MultiAddress.init("/ip4/1.2.3.4/tcp/4001").expect("valid") + relayAddr = MultiAddress + .init( + "/ip4/5.6.7.8/tcp/4002/p2p/16Uiu2HAmQu456Ae52JqPuqog6wCex47LLvNY8oHMBC4GRRtaStHs/p2p-circuit" + ) + .expect("valid") + udpPort = Port(8090) + + setup: + key = PrivateKey.random(Rng.instance().libp2pRng).get() + disc = Discovery.new(key, announceAddrs = @[]) + + test "announceDirectAddrs sets the SPR with both TCP and UDP addresses": + disc.announceDirectAddrs(@[directAddr], udpPort) + + let spr = disc.getSpr() + let addrs = spr.data.addresses.mapIt($it.address) + check addrs.anyIt(it.contains("/tcp/")) + check addrs.anyIt(it.contains("/udp/")) + + test "announceRelayAddrs updates the SPR with the announce addresses": + disc.announceDirectAddrs(@[directAddr], udpPort) + + disc.announceRelayAddrs(@[relayAddr]) + + let addrs = disc.getSpr().data.addresses.mapIt($it.address) + check addrs == @[$relayAddr] diff --git a/tests/storage/testnat.nim b/tests/storage/testnat.nim deleted file mode 100644 index 21faa156..00000000 --- a/tests/storage/testnat.nim +++ /dev/null @@ -1,43 +0,0 @@ -import std/[unittest, net] -import pkg/chronos -import pkg/libp2p/[multiaddress, multihash, multicodec] -import pkg/results - -import ../../storage/nat -import ../../storage/utils - -suite "NAT Address Tests": - test "nattedAddress with local addresses": - # Setup test data - let - udpPort = Port(1234) - natConfig = NatConfig(hasExtIp: true, extIp: parseIpAddress("8.8.8.8")) - - # Create test addresses - localAddr = MultiAddress.init("/ip4/127.0.0.1/tcp/5000").expect("valid multiaddr") - anyAddr = MultiAddress.init("/ip4/0.0.0.0/tcp/5000").expect("valid multiaddr") - publicAddr = - MultiAddress.init("/ip4/192.168.1.1/tcp/5000").expect("valid multiaddr") - - # Expected results - let - expectedDiscoveryAddrs = @[ - MultiAddress.init("/ip4/8.8.8.8/udp/1234").expect("valid multiaddr"), - MultiAddress.init("/ip4/8.8.8.8/udp/1234").expect("valid multiaddr"), - MultiAddress.init("/ip4/8.8.8.8/udp/1234").expect("valid multiaddr"), - ] - expectedlibp2pAddrs = @[ - MultiAddress.init("/ip4/8.8.8.8/tcp/5000").expect("valid multiaddr"), - MultiAddress.init("/ip4/8.8.8.8/tcp/5000").expect("valid multiaddr"), - MultiAddress.init("/ip4/8.8.8.8/tcp/5000").expect("valid multiaddr"), - ] - - #ipv6Addr = MultiAddress.init("/ip6/::1/tcp/5000").expect("valid multiaddr") - addrs = @[localAddr, anyAddr, publicAddr] - - # Test address remapping - let (libp2pAddrs, discoveryAddrs) = nattedAddress(natConfig, addrs, udpPort) - - # Verify results - check(discoveryAddrs == expectedDiscoveryAddrs) - check(libp2pAddrs == expectedlibp2pAddrs) diff --git a/tests/storage/testnatdetection.nim b/tests/storage/testnatdetection.nim new file mode 100644 index 00000000..ddd19fdf --- /dev/null +++ b/tests/storage/testnatdetection.nim @@ -0,0 +1,262 @@ +## NAT detection unit tests: real AutoNAT v2 detecting +## through the NAT simulation, feeding storage's handleNatStatus, which drives +## the relay and client mode. +## +## The MockNatPortMapper simulates a failing port mapping. This is not the aspect +## of the NAT detection that is being tested: we want to test the NAT detection +## logic itself, not the port mapping logic. + +import std/options +import pkg/chronos +import pkg/libp2p except setup +import pkg/libp2p/protocols/connectivity/autonatv2/service except setup +import pkg/libp2p/protocols/connectivity/autonatv2/client except setup +import pkg/libp2p/protocols/connectivity/autonatv2/types as autonatv2Types +import pkg/libp2p/protocols/connectivity/relay/client as relayClientModule +import pkg/libp2p/services/autorelayservice except setup +import pkg/libp2p/observedaddrmanager + +import ./helpers +import ./natsimulation +import ../asynctest +import ../../storage/utils/natutils +import ../../storage/nat +import ../../storage/discovery +import ../../storage/rng + +const + flags = {ServerFlags.ReuseAddr} + listenAddr = "/ip4/127.0.0.1/tcp/0" + discoveryPort = Port(8090) + # ms — AutoNAT probe + confidence + reaction + detectTimeout = 20000 + mockMappedTcpPort = Port(40000) + mockMappedUdpPort = Port(40001) + +type MockNatPortMapper = ref object of NatPortMapper + +method mapNatPorts*( + m: MockNatPortMapper +): Future[Option[(Port, Port, MappingProtocol)]] {. + async: (raises: [CancelledError]), gcsafe +.} = + none((Port, Port, MappingProtocol)) + +# Simulates a successful PCP mapping +type MockMappingNatPortMapper = ref object of NatPortMapper + +method mapNatPorts*( + m: MockMappingNatPortMapper +): Future[Option[(Port, Port, MappingProtocol)]] {. + async: (raises: [CancelledError]), gcsafe +.} = + m.portMapping = some( + PortMapping( + activeMappingProtocol: MappingProtocol.PCP, + activeTcpPort: mockMappedTcpPort, + activeUdpPort: mockMappedUdpPort, + ) + ) + some((mockMappedTcpPort, mockMappedUdpPort, MappingProtocol.PCP)) + +# Captures the candidate addresses the service sends and answers Reachable, so +# the service flips to reachable and runs its address mapper — without dialing. +type MockAutonatV2Client = ref object of AutonatV2Client + reqAddrs: seq[MultiAddress] + +method sendDialRequest*( + self: MockAutonatV2Client, pid: PeerId, testAddrs: seq[MultiAddress] +): Future[AutonatV2Response] {. + async: (raises: [AutonatV2Error, CancelledError, DialFailedError, LPStreamError]) +.} = + self.reqAddrs = testAddrs + AutonatV2Response(reachability: Reachable) + +proc serverSwitch(): Switch = + SwitchBuilder + .new() + .withRng(Rng.instance().libp2pRng) + .withPrivateKey(PrivateKey.random(Rng.instance().libp2pRng).get()) + .withAddresses(@[MultiAddress.init(listenAddr).get()]) + .withTcpTransport(flags) + .withNoise() + .withYamux() + .withAutonatV2Server() + .build() + +asyncchecksuite "NAT detection - simulated NAT": + var + natNode: Switch + autonat: AutonatV2Service + relay: AutoRelayService + disc: Discovery + server: Switch + + proc setupTopology(router: NatRouter) {.async.} = + let relayClient = relayClientModule.RelayClient.new() + natNode = SwitchBuilder + .new() + .withRng(Rng.instance().libp2pRng) + .withPrivateKey(PrivateKey.random(Rng.instance().libp2pRng).get()) + .withAddresses(@[MultiAddress.init(listenAddr).get()]) + .withNatTransport(router, flags) + .withNoise() + .withYamux() + .withCircuitRelay(relayClient) + .build() + + relay = AutoRelayService.new(1, relayClient, nil, Rng.instance().libp2pRng) + autorelayservice.setup(relay, natNode) + disc = Discovery.new( + PrivateKey.random(Rng.instance().libp2pRng).get(), announceAddrs = @[] + ) + # nodes start in client mode until Reachable + disc.protocol.clientMode = true + + # Setup real AutoNAT v2 client using nat simulation + let autonatClient = AutonatV2Client.new(natNode.rng) + client.setup(autonatClient, natNode) + natNode.mount(autonatClient) + + # Setup AutoNAT v2 service with maxQueueSize=1 and minConfidence=0.5, + # so a single dial-back answer (confidence 1.0) is needed. + let config = AutonatV2ServiceConfig.new( + scheduleInterval = Opt.some(1.seconds), + askNewConnectedPeers = true, + numPeersToAsk = 1, + maxQueueSize = 1, + minConfidence = 0.5, + ) + autonat = AutonatV2Service.new(natNode.rng, autonatClient, config) + service.setup(autonat, natNode) + + autonat.setStatusAndConfidenceHandler( + proc( + reachability: NetworkReachability, + confidence: Opt[float], + addrs: Opt[MultiAddress], + ) {.async: (raises: [CancelledError]).} = + # One call to our handleNatStatus handler + await MockNatPortMapper().handleNatStatus( + reachability, addrs, discoveryPort, disc, natNode, relay + ) + ) + + # Create and start one Autonat server (maxQueueSize=1 and minConfidence=0.5) + server = serverSwitch() + await server.start() + + # Start the NAT node and connect to the Autonat server (bootstrap node in our network). + # Then start the Autonat service on the NAT node. + await natNode.start() + await natNode.connect(server.peerInfo.peerId, server.peerInfo.addrs) + await autonat.start(natNode) + + teardown: + await autonat.stop(natNode) + + if relay.isRunning: + await relay.stop(natNode) + + await natNode.stop() + await server.stop() + + test "node behind EIF nat ends up reachable: no relay, not in client mode": + await setupTopology(NatRouter.new(EndpointIndependent)) + check eventually( + not relay.isRunning and not disc.protocol.clientMode, timeout = detectTimeout + ) + + test "node behind APDF nat ends up not reachable: relay running, client mode": + await setupTopology(NatRouter.new(AddressAndPortDependent)) + check eventually( + relay.isRunning and disc.protocol.clientMode, timeout = detectTimeout + ) + + test "node behind double NAT ends up not reachable: relay running, client mode": + await setupTopology(NatRouter.new(DoubleNat)) + check eventually( + relay.isRunning and disc.protocol.clientMode, timeout = detectTimeout + ) + + test "node recovers (relay stops) when nat switches from APDF to EIF": + let router = NatRouter.new(AddressAndPortDependent) + await setupTopology(router) + check eventually( + relay.isRunning and disc.protocol.clientMode, timeout = detectTimeout + ) + + router.setFiltering(EndpointIndependent) + check eventually( + not relay.isRunning and not disc.protocol.clientMode, timeout = detectTimeout + ) + + test "node degrades (relay starts) when nat switches from EIF to APDF": + let router = NatRouter.new(EndpointIndependent) + await setupTopology(router) + check eventually( + not relay.isRunning and not disc.protocol.clientMode, timeout = detectTimeout + ) + + router.setFiltering(AddressAndPortDependent) + check eventually( + relay.isRunning and disc.protocol.clientMode, timeout = detectTimeout + ) + +asyncchecksuite "NAT detection - dial request candidates": + # This detects is useful to detect behaviour changes in libp2p + # that may break our autonat dial request candidate handling. + # + # By default, Autonat V2 dials the addresses passed to peerInfo.addrs and + # uses the first attempt to dial as the primary candidate. + # + # With enableDialableCandidates, Autonat V2 also dials the guessDialableAddress + # as first candidate and use the most observed address as the fallback. + var sw: Switch + + setup: + sw = newStandardSwitch() + await sw.start() + + teardown: + await sw.stop() + + test "autonat handles the observed dialable address": + let mockClient = MockAutonatV2Client() + let autonat = AutonatV2Service.new( + Rng.instance().libp2pRng, + mockClient, + AutonatV2ServiceConfig.new( + enableDialableCandidates = true, maxQueueSize = 1, minConfidence = 0.5 + ), + ) + service.setup(autonat, sw) + await autonat.start(sw) # registers the address mapper on peerInfo + + # observations before the manager trusts an addr in libp2p; the observed + # port (4001) differs from our listen port on purpose (see dialable below) + let quorum = 3 + let observed = MultiAddress.init("/ip4/8.8.8.8/tcp/4001").expect("valid") + for _ in 0 ..< quorum: + discard sw.peerStore.identify.observedAddrManager.addObservation(observed) + + let sw2 = newStandardSwitch() + await sw2.start() + await sw.connect(sw2.peerInfo.peerId, sw2.peerInfo.addrs) + + # The dialable candidate keeps our real listen port and swaps in the + # observed IP. It must be reached via AutoNAT and peerInfo, so it can only + # come from guessDialableAddr. + let tcpPart = sw.peerInfo.listenAddrs[0][1].expect("valid") + let dialable = + concat(MultiAddress.init("/ip4/8.8.8.8").expect("valid"), tcpPart).expect("valid") + + # Phase 1: it is submitted as a dial candidate, not 127.0.0.1 from + # newStandardSwitch. + check eventually(dialable in mockClient.reqAddrs) + + # Phase 2: the address mapper promotes it into peerInfo. + check eventually(dialable in sw.peerInfo.addrs) + + await autonat.stop(sw) + await sw2.stop() diff --git a/tests/storage/testnatreaction.nim b/tests/storage/testnatreaction.nim new file mode 100644 index 00000000..5e5de7bc --- /dev/null +++ b/tests/storage/testnatreaction.nim @@ -0,0 +1,345 @@ +import std/[importutils, net] +import pkg/chronos +import pkg/libp2p/[multiaddress, multihash, multicodec] +import pkg/libp2p/protocols/connectivity/autonat/types +import pkg/libp2p/protocols/connectivity/relay/client as relayClientModule +import pkg/libp2p/services/autorelayservice except setup +import pkg/results + +import ./helpers +import ../asynctest +import ../../storage/utils/natutils +import ../../storage/nat +import ../../storage/discovery +import ../../storage/rng +import ../../storage/utils + +type MockNatPortMapper = ref object of NatPortMapper + mappedPorts: Option[(Port, Port, MappingProtocol)] + +method mapNatPorts*( + m: MockNatPortMapper +): Future[Option[(Port, Port, MappingProtocol)]] {. + async: (raises: [CancelledError]), gcsafe +.} = + m.mappedPorts + +method destroyMappingFor(m: MockNatPortMapper, id: cint) {.gcsafe.} = + discard + +type MockMapNatPortMapper = ref object of NatPortMapper + tcpResult: Result[MappingResult, string] + udpResult: Result[MappingResult, string] + live: bool + createAttempts: seq[PlumProtocol] + destroyed: seq[cint] + +method initPlum(m: MockMapNatPortMapper): Result[void, string] {.gcsafe.} = + ok() + +method hasLivePortMapping(m: MockMapNatPortMapper): bool {.gcsafe.} = + m.portMapping.isSome and m.live + +method createMappingFor( + m: MockMapNatPortMapper, protocol: PlumProtocol, port: uint16 +): Future[Result[MappingResult, string]] {.async: (raises: [CancelledError]), gcsafe.} = + m.createAttempts.add(protocol) + if protocol == TCP: m.tcpResult else: m.udpResult + +method destroyMappingFor(m: MockMapNatPortMapper, id: cint) {.gcsafe.} = + m.destroyed.add(id) + +proc mappingOk(id: cint, port: uint16): Result[MappingResult, string] = + Result[MappingResult, string].ok( + MappingResult( + id: id, + mapping: PlumMapping(mappingProtocol: MappingProtocol.UPnP, externalPort: port), + ) + ) + +const relayId = "16Uiu2HAmQu456Ae52JqPuqog6wCex47LLvNY8oHMBC4GRRtaStHs" + +proc circuitAddr(relayIp: string): MultiAddress = + MultiAddress + .init("/ip4/" & relayIp & "/tcp/8070/p2p/" & relayId & "/p2p-circuit") + .expect("valid") + +asyncchecksuite "NAT reaction - port mapping": + var sw: Switch + var key: PrivateKey + var disc: Discovery + var autoRelay: AutoRelayService + + setup: + autoRelay = AutoRelayService.new( + 1, relayClientModule.RelayClient.new(), nil, Rng.instance().libp2pRng + ) + key = PrivateKey.random(Rng.instance().libp2pRng).get() + disc = Discovery.new(key, announceAddrs = @[]) + sw = newStandardSwitch() + await sw.start() + + teardown: + await sw.stop() + + if autoRelay.isRunning: + await autoRelay.stop(sw) + + let discoveryPort = Port(8090) + + test "handleNatStatus keeps relay off when NotReachable and mapping succeeds": + let dialBack = MultiAddress.init("/ip4/1.2.3.4/tcp/8080").expect("valid") + let mapper = MockNatPortMapper( + mappedPorts: some((Port(9000), Port(9001), MappingProtocol.UPnP)) + ) + + autorelayservice.setup(autoRelay, sw) + await mapper.handleNatStatus( + NotReachable, Opt.some(dialBack), discoveryPort, disc, sw, autoRelay + ) + + # A mapping doesn't guarantee reachability, so the relay stays off until + # AutoNAT confirms Reachable. + check not autoRelay.isRunning + check disc.protocol.clientMode + + test "handleNatStatus starts autoRelay when NotReachable with no mapped ports": + let mapper = MockNatPortMapper(mappedPorts: none((Port, Port, MappingProtocol))) + + autorelayservice.setup(autoRelay, sw) + await mapper.handleNatStatus( + NotReachable, Opt.none(MultiAddress), discoveryPort, disc, sw, autoRelay + ) + + check autoRelay.isRunning + check disc.announceAddrs == newSeq[MultiAddress]() + check disc.protocol.clientMode + + test "handleNatStatus keeps a live mapping and starts relay when NotReachable": + privateAccess(PortMapping) + let dialBack = MultiAddress.init("/ip4/1.2.3.4/tcp/8080").expect("valid") + let mapper = MockMapNatPortMapper(live: true) + mapper.portMapping = some( + PortMapping( + tcpMappingId: cint(1), + udpMappingId: cint(2), + activeMappingProtocol: MappingProtocol.UPnP, + activeTcpPort: Port(9000), + activeUdpPort: Port(9001), + ) + ) + + autorelayservice.setup(autoRelay, sw) + await mapper.handleNatStatus( + NotReachable, Opt.some(dialBack), discoveryPort, disc, sw, autoRelay + ) + + check autoRelay.isRunning + check disc.announceAddrs == newSeq[MultiAddress]() + check disc.protocol.clientMode + check mapper.portMapping.isSome # the live mapping is kept + check mapper.destroyed.len == 0 # never torn down + + test "handleNatStatus recreates a dead mapping instead of pinning it": + privateAccess(PortMapping) + let dialBack = MultiAddress.init("/ip4/1.2.3.4/tcp/8080").expect("valid") + let mapper = MockMapNatPortMapper( + live: false, + tcpResult: mappingOk(cint(10), 9000), + udpResult: mappingOk(cint(20), 9001), + ) + mapper.portMapping = some( + PortMapping( + tcpMappingId: cint(1), + udpMappingId: cint(2), + activeMappingProtocol: MappingProtocol.UPnP, + activeTcpPort: Port(9000), + activeUdpPort: Port(9001), + ) + ) + + autorelayservice.setup(autoRelay, sw) + await mapper.handleNatStatus( + NotReachable, Opt.some(dialBack), discoveryPort, disc, sw, autoRelay + ) + + check mapper.destroyed == @[cint(1), cint(2)] # the dead mapping is torn down + check mapper.portMapping.isSome # replaced by a fresh one + check not autoRelay.isRunning # direct path kept, no relay + + test "handleNatStatus stops relay and exits client mode when mapping is created and node is Reachable": + let dialBack = MultiAddress.init("/ip4/1.2.3.4/tcp/8080").expect("valid") + let mapper = MockNatPortMapper(mappedPorts: none((Port, Port, MappingProtocol))) + + disc.protocol.clientMode = true + autorelayservice.setup(autoRelay, sw) + await mapper.handleNatStatus( + Reachable, Opt.some(dialBack), discoveryPort, disc, sw, autoRelay + ) + + check not autoRelay.isRunning + check not disc.protocol.clientMode + + test "handleNatStatus does nothing after the mapper is stopped": + let dialBack = MultiAddress.init("/ip4/1.2.3.4/tcp/8080").expect("valid") + let mapper = MockNatPortMapper( + mappedPorts: some((Port(9000), Port(9001), MappingProtocol.UPnP)) + ) + mapper.stop() + + autorelayservice.setup(autoRelay, sw) + await mapper.handleNatStatus( + NotReachable, Opt.some(dialBack), discoveryPort, disc, sw, autoRelay + ) + + check not autoRelay.isRunning + check disc.announceAddrs == newSeq[MultiAddress]() + +asyncchecksuite "NAT reaction - address announcing": + var sw: Switch + var key: PrivateKey + var disc: Discovery + var autoRelay: AutoRelayService + + setup: + autoRelay = AutoRelayService.new( + 1, relayClientModule.RelayClient.new(), nil, Rng.instance().libp2pRng + ) + key = PrivateKey.random(Rng.instance().libp2pRng).get() + disc = Discovery.new(key, announceAddrs = @[]) + sw = newStandardSwitch() + await sw.start() + + teardown: + await sw.stop() + if autoRelay.isRunning: + await autoRelay.stop(sw) + + let discoveryPort = Port(8090) + + test "handleNatStatus announces the dial-back address when Reachable": + let dialBack = MultiAddress.init("/ip4/1.2.3.4/tcp/9000").expect("valid") + + let mapper = NatPortMapper(discoveryPort: discoveryPort) + await mapper.handleNatStatus( + Reachable, Opt.some(dialBack), discoveryPort, disc, sw, autoRelay + ) + + check disc.announceAddrs == @[dialBack] + + test "handleNatStatus does not announce when Reachable without a dial-back address": + let mapper = NatPortMapper(discoveryPort: discoveryPort) + await mapper.handleNatStatus( + Reachable, Opt.none(MultiAddress), discoveryPort, disc, sw, autoRelay + ) + + check disc.announceAddrs == newSeq[MultiAddress]() + + test "handleNatStatus clears the DHT routing addresses when it becomes NotReachable": + let dialBack = MultiAddress.init("/ip4/1.2.3.4/tcp/9000").expect("valid") + let mapper = MockNatPortMapper(mappedPorts: none((Port, Port, MappingProtocol))) + + autorelayservice.setup(autoRelay, sw) + + # Reachable: the node announces direct addresses, including UDP for the DHT. + await mapper.handleNatStatus( + Reachable, Opt.some(dialBack), discoveryPort, disc, sw, autoRelay + ) + check disc.dhtAddrs.len > 0 + + # NotReachable: the DHT routing addresses are cleared + await mapper.handleNatStatus( + NotReachable, Opt.some(dialBack), discoveryPort, disc, sw, autoRelay + ) + check disc.dhtAddrs.len == 0 + + test "announceRelayReservation announces only the publicly dialable circuit address": + disc.announceRelayReservation( + @[circuitAddr("127.0.0.1"), circuitAddr("204.168.234.45")] + ) + + check disc.announceAddrs == @[circuitAddr("204.168.234.45")] + + test "announceRelayReservation does not announce a private circuit address": + disc.announceRelayReservation(@[circuitAddr("127.0.0.1")]) + + check disc.announceAddrs.len == 0 + +proc mapperWith(protocol: MappingProtocol): Option[NatPortMapper] = + some(NatPortMapper(portMapping: some(PortMapping(activeMappingProtocol: protocol)))) + +asyncchecksuite "NAT - portMappingStr": + test "no mapper is none": + check portMappingStr(none(NatPortMapper)) == "none" + + test "mapper without an active protocol is none": + check portMappingStr(some(NatPortMapper())) == "none" + + test "UPnP maps to upnp": + check portMappingStr(mapperWith(MappingProtocol.UPnP)) == "upnp" + + test "NAT-PMP maps to pmp": + check portMappingStr(mapperWith(MappingProtocol.NatPmp)) == "pmp" + + test "PCP maps to pcp": + check portMappingStr(mapperWith(MappingProtocol.PCP)) == "pcp" + + test "Direct maps to direct": + check portMappingStr(mapperWith(MappingProtocol.Direct)) == "direct" + + test "Unknown maps to none": + check portMappingStr(mapperWith(MappingProtocol.Unknown)) == "none" + +asyncchecksuite "NAT - mapNatPorts": + test "returns the mapped ports when both mappings succeed": + let mapper = MockMapNatPortMapper( + tcpResult: mappingOk(cint(1), 9000), udpResult: mappingOk(cint(2), 9001) + ) + + check (await mapper.mapNatPorts()) == + some((Port(9000), Port(9001), MappingProtocol.UPnP)) + check mapper.destroyed.len == 0 + + test "destroys the TCP mapping when the UDP mapping fails": + let mapper = MockMapNatPortMapper( + tcpResult: mappingOk(cint(42), 9000), + udpResult: Result[MappingResult, string].err("udp mapping failed"), + ) + + check (await mapper.mapNatPorts()).isNone + check mapper.destroyed == @[cint(42)] + + test "gives up without touching UDP when the TCP mapping fails": + let mapper = MockMapNatPortMapper( + tcpResult: Result[MappingResult, string].err("tcp mapping failed"), + udpResult: mappingOk(cint(2), 9001), + ) + + check (await mapper.mapNatPorts()).isNone + check mapper.createAttempts == @[PlumProtocol.TCP] # UDP never attempted + check mapper.destroyed.len == 0 # nothing to clean up + + test "does not map when configured with an external IP": + let mapper = MockMapNatPortMapper( + natConfig: nat.NatConfig(hasExtIp: true, extIp: parseIpAddress("1.2.3.4")) + ) + + check (await mapper.mapNatPorts()).isNone + check mapper.createAttempts.len == 0 # short-circuits before any mapping + + test "reuses the existing mapping when both are still live": + privateAccess(PortMapping) + let mapper = MockMapNatPortMapper(live: true) + mapper.portMapping = some( + PortMapping( + tcpMappingId: cint(1), + udpMappingId: cint(2), + activeMappingProtocol: MappingProtocol.UPnP, + activeTcpPort: Port(9000), + activeUdpPort: Port(9001), + ) + ) + + check (await mapper.mapNatPorts()) == + some((Port(9000), Port(9001), MappingProtocol.UPnP)) + check mapper.createAttempts.len == 0 diff --git a/tests/storage/testnatsimulation.nim b/tests/storage/testnatsimulation.nim new file mode 100644 index 00000000..077f8d62 --- /dev/null +++ b/tests/storage/testnatsimulation.nim @@ -0,0 +1,180 @@ +import std/net +import pkg/chronos +import pkg/libp2p/wire + +import ./helpers +import ../asynctest +import ../../storage/rng as storage_rng +import ../../storage/nat +import ./natsimulation + +const flags = {ServerFlags.ReuseAddr} +const listenAddr = "/ip4/127.0.0.1/tcp/0" +const filterTimeout = 500.millis + +proc cannotConnect(a, b: Switch): Future[bool] {.async.} = + let completed = + try: + await a.connect(b.peerInfo.peerId, b.peerInfo.addrs).withTimeout(filterTimeout) + except LPError: + false + if completed: + return false + return not a.isConnected(b.peerInfo.peerId) + +proc newSwitch(rng: storage_rng.Rng): Switch = + SwitchBuilder + .new() + .withRng(rng.libp2pRng) + .withPrivateKey(PrivateKey.random(rng.libp2pRng).get()) + .withAddresses(@[MultiAddress.init(listenAddr).get()]) + .withTcpTransport(flags) + .withNoise() + .withYamux() + .build() + +proc newNatSwitch(router: NatRouter, rng: storage_rng.Rng): Switch = + SwitchBuilder + .new() + .withRng(rng.libp2pRng) + .withPrivateKey(PrivateKey.random(rng.libp2pRng).get()) + .withAddresses(@[MultiAddress.init(listenAddr).get()]) + .withNatTransport(router, flags) + .withNoise() + .withYamux() + .build() + +asyncchecksuite "Nat transport - Endpoint-Independent Filtering": + var bootstrap, natNode: Switch + + setup: + let router = NatRouter.new(EndpointIndependent) + bootstrap = newSwitch(storage_rng.Rng.instance()) + natNode = newNatSwitch(router, storage_rng.Rng.instance()) + await bootstrap.start() + await natNode.start() + + teardown: + await bootstrap.stop() + await natNode.stop() + + test "bootstrap can connect to nat node without any prior outbound": + await bootstrap.connect(natNode.peerInfo.peerId, natNode.peerInfo.addrs) + check bootstrap.isConnected(natNode.peerInfo.peerId) + +asyncchecksuite "Nat transport - Address-Dependent Filtering": + var bootstrap, thirdNode, natNode: Switch + + setup: + let router = NatRouter.new(AddressDependent) + bootstrap = newSwitch(storage_rng.Rng.instance()) + thirdNode = newSwitch(storage_rng.Rng.instance()) + natNode = newNatSwitch(router, storage_rng.Rng.instance()) + await bootstrap.start() + await thirdNode.start() + await natNode.start() + + teardown: + await bootstrap.stop() + await thirdNode.stop() + await natNode.stop() + + test "bootstrap can connect to nat node with a pre-existing connection": + await natNode.connect(bootstrap.peerInfo.peerId, bootstrap.peerInfo.addrs) + check natNode.isConnected(bootstrap.peerInfo.peerId) + + await bootstrap.connect(natNode.peerInfo.peerId, natNode.peerInfo.addrs) + check bootstrap.isConnected(natNode.peerInfo.peerId) + + test "third node can connect to nat node after nat node connected to bootstrap": + await natNode.connect(bootstrap.peerInfo.peerId, bootstrap.peerInfo.addrs) + await thirdNode.connect(natNode.peerInfo.peerId, natNode.peerInfo.addrs) + check thirdNode.isConnected(natNode.peerInfo.peerId) + + test "bootstrap cannot connect to nat node without a pre-existing connection": + check await cannotConnect(bootstrap, natNode) + +asyncchecksuite "Nat transport - Address-and-Port-Dependent Filtering": + var bootstrap, thirdNode, natNode: Switch + + setup: + let router = NatRouter.new(AddressAndPortDependent) + bootstrap = newSwitch(storage_rng.Rng.instance()) + thirdNode = newSwitch(storage_rng.Rng.instance()) + natNode = newNatSwitch(router, storage_rng.Rng.instance()) + await bootstrap.start() + await thirdNode.start() + await natNode.start() + + teardown: + await bootstrap.stop() + await thirdNode.stop() + await natNode.stop() + + test "bootstrap can connect to nat node with a pre-existing connection": + await natNode.connect(bootstrap.peerInfo.peerId, bootstrap.peerInfo.addrs) + check natNode.isConnected(bootstrap.peerInfo.peerId) + + await bootstrap.connect(natNode.peerInfo.peerId, natNode.peerInfo.addrs) + check bootstrap.isConnected(natNode.peerInfo.peerId) + + test "bootstrap cannot connect to nat node without a pre-existing connection": + check await cannotConnect(bootstrap, natNode) + + test "third node cannot connect to nat node even after nat node connected to bootstrap": + await natNode.connect(bootstrap.peerInfo.peerId, bootstrap.peerInfo.addrs) + check await cannotConnect(thirdNode, natNode) + +asyncchecksuite "Nat transport - Double NAT": + var bootstrap, natNode: Switch + var router: NatRouter + + setup: + router = NatRouter.new(DoubleNat) + bootstrap = newSwitch(storage_rng.Rng.instance()) + natNode = newNatSwitch(router, storage_rng.Rng.instance()) + await bootstrap.start() + await natNode.start() + + teardown: + await bootstrap.stop() + await natNode.stop() + + test "bootstrap cannot connect to nat node regardless of port mapping": + let actualPort = initTAddress(natNode.peerInfo.addrs[0]).get().port + let natMapper = NatPortMapper() + natMapper.portMapping = some(PortMapping(activeTcpPort: actualPort)) + router.natMapper = some(natMapper) + + check await cannotConnect(bootstrap, natNode) + +asyncchecksuite "Nat transport - Port Mapping": + var bootstrap, natNode: Switch + var router: NatRouter + + setup: + router = NatRouter.new(AddressAndPortDependent) + bootstrap = newSwitch(storage_rng.Rng.instance()) + natNode = newNatSwitch(router, storage_rng.Rng.instance()) + await bootstrap.start() + await natNode.start() + + teardown: + await bootstrap.stop() + await natNode.stop() + + test "bootstrap can connect to nat node when port mapping matches listen port": + let actualPort = initTAddress(natNode.peerInfo.addrs[0]).get().port + let natMapper = NatPortMapper() + natMapper.portMapping = some(PortMapping(activeTcpPort: actualPort)) + router.natMapper = some(natMapper) + + await bootstrap.connect(natNode.peerInfo.peerId, natNode.peerInfo.addrs) + check bootstrap.isConnected(natNode.peerInfo.peerId) + + test "bootstrap cannot connect to nat node when port mapping does not match": + let natMapper = NatPortMapper() + natMapper.portMapping = some(PortMapping(activeTcpPort: Port(1))) + router.natMapper = some(natMapper) + + check await cannotConnect(bootstrap, natNode) diff --git a/tests/testIntegration.nim b/tests/testIntegration.nim index 5e289ff5..c5f576ef 100644 --- a/tests/testIntegration.nim +++ b/tests/testIntegration.nim @@ -11,7 +11,8 @@ when includes != "": # import only the specified tests importAll(includes.split(",")) else: - # import all tests in the integration/ directory - importTests(currentSourcePath().parentDir() / "integration") + # all tests in integration/, except the nat/ real-topology scenarios, which + # need podman + the storage-nat image and run via testNatIntegration instead + importTests(currentSourcePath().parentDir() / "integration", "/nat/") {.warning[UnusedImport]: off.} diff --git a/tests/testNatIntegration.nim b/tests/testNatIntegration.nim new file mode 100644 index 00000000..5e16eee5 --- /dev/null +++ b/tests/testNatIntegration.nim @@ -0,0 +1,15 @@ +import std/os +import std/strutils +import ./imports + +## Real-topology NAT scenarios (need podman + the storage-nat image). +## Limit which scenarios run with STORAGE_INTEGRATION_TEST_INCLUDES, listing test +## file paths, exactly as testIntegration does. +const includes = getEnv("STORAGE_INTEGRATION_TEST_INCLUDES") + +when includes != "": + importAll(includes.split(",")) +else: + importTests(currentSourcePath().parentDir() / "integration" / "nat", "") + +{.warning[UnusedImport]: off.} diff --git a/tests/testStorage.nim b/tests/testStorage.nim index 609a50ac..30549b32 100644 --- a/tests/testStorage.nim +++ b/tests/testStorage.nim @@ -1,6 +1,6 @@ import std/os import ./imports -importTests(currentSourcePath().parentDir() / "storage") +importTests(currentSourcePath().parentDir() / "storage", "") {.warning[UnusedImport]: off.} diff --git a/tools/scripts/ci-job-matrix.sh b/tools/scripts/ci-job-matrix.sh index e55968bf..9047df42 100755 --- a/tools/scripts/ci-job-matrix.sh +++ b/tools/scripts/ci-job-matrix.sh @@ -95,9 +95,10 @@ integration_test () { integration_test_job $tests done - # fail when there are integration tests with an unknown duration + # fail when there are integration tests with an unknown duration. nat/ is + # excluded: those real-topology scenarios run via the nat-integration job. local filter='1_minute\|5_minutes\|30_minutes' - local unknown=$(find_tests tests/integration | grep -v "$filter") + local unknown=$(find_tests tests/integration | grep -v "$filter" | grep -v '/nat/') if [ "$unknown" != "" ]; then echo "Error: Integration tests need to be in either the 1_minute," >&2 echo " 5_minutes, or 30_minutes directory, based on the maximum" >&2 @@ -117,11 +118,22 @@ libstorage_test () { job } +# outputs the NAT real-topology integration job (all scenarios). +# Linux-only: needs network-namespace + iptables manipulation. +nat_integration_tests () { + job_tests="nat-integration" + job_includes="" + job +} + # outputs jobs for all test types all_tests () { unit_test integration_test libstorage_test + if [ "$job_os" = "linux" ]; then + nat_integration_tests + fi } # outputs jobs for the specified operating systems and all test types diff --git a/vendor/nim-boringssl b/vendor/nim-boringssl index f8111056..e77caaba 160000 --- a/vendor/nim-boringssl +++ b/vendor/nim-boringssl @@ -1 +1 @@ -Subproject commit f8111056182cf6abd9e35de77a919e873ef94652 +Subproject commit e77caabae78fbc9aa5b78a0a521181b077c82571 diff --git a/vendor/nim-libp2p b/vendor/nim-libp2p index c4319937..c470b114 160000 --- a/vendor/nim-libp2p +++ b/vendor/nim-libp2p @@ -1 +1 @@ -Subproject commit c43199378f46d0aaf61be1cad1ee1d63e8f665d6 +Subproject commit c470b1146fa2ef23ab88c5a0940923cf7645e9c5 diff --git a/vendor/nim-libplum b/vendor/nim-libplum new file mode 160000 index 00000000..433e4878 --- /dev/null +++ b/vendor/nim-libplum @@ -0,0 +1 @@ +Subproject commit 433e48789dfef0a1435db97946b4fee8595d8fb9 diff --git a/vendor/nim-lsquic b/vendor/nim-lsquic index 00e4b7df..2f01046b 160000 --- a/vendor/nim-lsquic +++ b/vendor/nim-lsquic @@ -1 +1 @@ -Subproject commit 00e4b7dfaa197cd120267aa897b33b0914166b45 +Subproject commit 2f01046bf1d513de8b5f8296c3d8bec819ab0cb9 diff --git a/vendor/nim-protobuf-serialization b/vendor/nim-protobuf-serialization index f45476a3..d9aa950b 160000 --- a/vendor/nim-protobuf-serialization +++ b/vendor/nim-protobuf-serialization @@ -1 +1 @@ -Subproject commit f45476a3c1f4e7bff73845e6450d686be040ddeb +Subproject commit d9aa950b9d9e8bfc8a201740042b5e8ea5880875