From 281d5e3bcca08cf3849dc6da02b7040b2de37265 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Fri, 10 Apr 2026 18:26:27 +0400 Subject: [PATCH 001/167] Add autonat conf --- storage/conf.nim | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/storage/conf.nim b/storage/conf.nim index 2937bad4..300d7c0a 100644 --- a/storage/conf.nim +++ b/storage/conf.nim @@ -53,6 +53,8 @@ export DefaultQuotaBytes, DefaultBlockTtl, DefaultBlockInterval, DefaultNumBlocksPerInterval, DefaultBlockRetries +const DefaultNatScheduleInterval* = 5.minutes + type ThreadCount* = distinct Natural proc `==`*(a, b: ThreadCount): bool {.borrow.} @@ -309,6 +311,31 @@ type desc: "Logs to file", defaultValue: string.none, name: "log-file", hidden .}: Option[string] + natScheduleInterval* {. + desc: "Interval between AutoNAT reachability checks", + defaultValue: DefaultNatScheduleInterval, + defaultValueDesc: $DefaultNatScheduleInterval, + name: "nat-schedule-interval" + .}: Duration + + natNumPeersToAsk* {. + desc: "Number of peers to ask per AutoNAT round", + defaultValue: 3, + name: "nat-num-peers-to-ask" + .}: int + + natMaxQueueSize* {. + desc: "Number of past AutoNAT results kept to calculate confidence", + defaultValue: 3, + name: "nat-max-queue-size" + .}: int + + natMinConfidence* {. + desc: "Minimum confidence threshold to confirm reachability", + defaultValue: 0.7, + name: "nat-min-confidence" + .}: float + func defaultAddress*(conf: StorageConf): IpAddress = result = static parseIpAddress("127.0.0.1") From d0c02bbf10b4eeea4c2e54998900c5a8b8fce957 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Fri, 10 Apr 2026 18:27:21 +0400 Subject: [PATCH 002/167] Better support for ivp6 and more tests --- storage/nat.nim | 7 ++- storage/utils/natutils.nim | 80 ++++++++++++++++++---------------- tests/storage/testnat.nim | 45 ++++++++++++++----- tests/storage/testnatutils.nim | 67 ++++++++++++++++++++++++++++ 4 files changed, 151 insertions(+), 48 deletions(-) create mode 100644 tests/storage/testnatutils.nim diff --git a/storage/nat.nim b/storage/nat.nim index 60dedc49..f11d9786 100644 --- a/storage/nat.nim +++ b/storage/nat.nim @@ -137,7 +137,12 @@ proc getRoutePrefSrc(bindIp: IpAddress): (Option[IpAddress], PrefSrcStatus) = let bindAddress = initTAddress(bindIp, Port(0)) if bindAddress.isAnyLocal(): - let ip = getRouteIpv4() + let ip = + if bindIp.family == IpAddressFamily.IPv6: + getRouteIpv6() + else: + getRouteIpv4() + if ip.isErr(): # No route was found, log error and continue without IP. error "No routable IP address found, check your network connection", diff --git a/storage/utils/natutils.nim b/storage/utils/natutils.nim index 45ad7589..37228236 100644 --- a/storage/utils/natutils.nim +++ b/storage/utils/natutils.nim @@ -1,6 +1,6 @@ {.push raises: [].} -import std/[net, tables, hashes], pkg/results, chronos, chronicles +import std/[net, tables, hashes, options], pkg/results, chronos, chronicles import pkg/libp2p @@ -10,32 +10,6 @@ type NatStrategy* = enum NatPmp NatNone -type IpLimits* = object - limit*: uint - ips: Table[IpAddress, uint] - -func hash*(ip: IpAddress): Hash = - case ip.family - of IpAddressFamily.IPv6: - hash(ip.address_v6) - of IpAddressFamily.IPv4: - hash(ip.address_v4) - -func inc*(ipLimits: var IpLimits, ip: IpAddress): bool = - let val = ipLimits.ips.getOrDefault(ip, 0) - if val < ipLimits.limit: - ipLimits.ips[ip] = val + 1 - true - else: - false - -func dec*(ipLimits: var IpLimits, ip: IpAddress) = - let val = ipLimits.ips.getOrDefault(ip, 0) - if val == 1: - ipLimits.ips.del(ip) - elif val > 1: - ipLimits.ips[ip] = val - 1 - func isGlobalUnicast*(address: TransportAddress): bool = if address.isGlobal() and address.isUnicast(): true else: false @@ -43,18 +17,11 @@ func isGlobalUnicast*(address: IpAddress): bool = let a = initTAddress(address, Port(0)) a.isGlobalUnicast() -proc getRouteIpv4*(): Result[IpAddress, cstring] = - # Avoiding Exception with initTAddress and can't make it work with static. - # Note: `publicAddress` is only used an "example" IP to find the best route, - # no data is send over the network to this IP! - let - publicAddress = TransportAddress( - family: AddressFamily.IPv4, address_v4: [1'u8, 1, 1, 1], port: Port(0) - ) - route = getBestRoute(publicAddress) +proc getRoute(publicAddress: TransportAddress): Result[IpAddress, cstring] = + let route = getBestRoute(publicAddress) if route.source.isUnspecified(): - err("No best ipv4 route found") + err("No best route found") else: let ip = try: @@ -64,3 +31,42 @@ proc getRouteIpv4*(): Result[IpAddress, cstring] = error "Address conversion error", exception = e.name, msg = e.msg return err("Invalid IP address") ok(ip) + +proc getRouteIpv4*(): Result[IpAddress, cstring] = + # Avoiding Exception with initTAddress and can't make it work with static. + # Note: `publicAddress` is only used an "example" IP to find the best route, + # no data is send over the network to this IP! + let publicAddress = TransportAddress( + family: AddressFamily.IPv4, address_v4: [1'u8, 1, 1, 1], port: Port(0) + ) + + return getRoute(publicAddress) + +proc getRouteIpv6*(): Result[IpAddress, cstring] = + # Note: `googleDnsIpv6` is only used as an "example" IP to find the best route, + # no data is sent over the network to this IP! + const googleDnsIpv6 = TransportAddress( + family: AddressFamily.IPv6, + # 2001:4860:4860::8888 + address_v6: [32'u8, 1, 72, 96, 72, 96, 0, 0, 0, 0, 0, 0, 0, 0, 136, 136], + port: Port(0), + ) + + return getRoute(googleDnsIpv6) + +# If bindIp is a anyLocal address (0.0.0.0 or ::), +# the function will find the best ip address. +# Otherwise, it will just return the ip as it is. +proc getBestLocalAddress*(bindIp: IpAddress): Option[IpAddress] = + let bindAddress = initTAddress(bindIp, Port(0)) + if bindAddress.isAnyLocal(): + let ip = + if bindIp.family == IpAddressFamily.IPv6: + getRouteIpv6() + else: + getRouteIpv4() + if ip.isOk(): + return some(ip.get()) + return none(IpAddress) + else: + return some(bindIp) diff --git a/tests/storage/testnat.nim b/tests/storage/testnat.nim index 21faa156..eb93c1ff 100644 --- a/tests/storage/testnat.nim +++ b/tests/storage/testnat.nim @@ -21,16 +21,18 @@ suite "NAT Address Tests": # Expected results let - expectedDiscoveryAddrs = @[ - MultiAddress.init("/ip4/8.8.8.8/udp/1234").expect("valid multiaddr"), - MultiAddress.init("/ip4/8.8.8.8/udp/1234").expect("valid multiaddr"), - MultiAddress.init("/ip4/8.8.8.8/udp/1234").expect("valid multiaddr"), - ] - expectedlibp2pAddrs = @[ - MultiAddress.init("/ip4/8.8.8.8/tcp/5000").expect("valid multiaddr"), - MultiAddress.init("/ip4/8.8.8.8/tcp/5000").expect("valid multiaddr"), - MultiAddress.init("/ip4/8.8.8.8/tcp/5000").expect("valid multiaddr"), - ] + expectedDiscoveryAddrs = + @[ + MultiAddress.init("/ip4/8.8.8.8/udp/1234").expect("valid multiaddr"), + MultiAddress.init("/ip4/8.8.8.8/udp/1234").expect("valid multiaddr"), + MultiAddress.init("/ip4/8.8.8.8/udp/1234").expect("valid multiaddr"), + ] + expectedlibp2pAddrs = + @[ + MultiAddress.init("/ip4/8.8.8.8/tcp/5000").expect("valid multiaddr"), + MultiAddress.init("/ip4/8.8.8.8/tcp/5000").expect("valid multiaddr"), + MultiAddress.init("/ip4/8.8.8.8/tcp/5000").expect("valid multiaddr"), + ] #ipv6Addr = MultiAddress.init("/ip6/::1/tcp/5000").expect("valid multiaddr") addrs = @[localAddr, anyAddr, publicAddr] @@ -41,3 +43,26 @@ suite "NAT Address Tests": # Verify results check(discoveryAddrs == expectedDiscoveryAddrs) check(libp2pAddrs == expectedlibp2pAddrs) + +suite "setupAddress": + test "public bind IP with NatNone returns bind IP": + let + bindIp = parseIpAddress("8.8.8.8") + natConfig = NatConfig(hasExtIp: false, nat: NatStrategy.NatNone) + (ip, tcpPort, udpPort) = + setupAddress(natConfig, bindIp, Port(5000), Port(5001), "test") + + check ip == some(bindIp) + check tcpPort == some(Port(5000)) + check udpPort == some(Port(5001)) + + test "private bind IP with NatNone returns no IP": + let + bindIp = parseIpAddress("192.168.1.1") + natConfig = NatConfig(hasExtIp: false, nat: NatStrategy.NatNone) + (ip, tcpPort, udpPort) = + setupAddress(natConfig, bindIp, Port(5000), Port(5001), "test") + + check ip == none(IpAddress) + check tcpPort == some(Port(5000)) + check udpPort == some(Port(5001)) diff --git a/tests/storage/testnatutils.nim b/tests/storage/testnatutils.nim new file mode 100644 index 00000000..fcc48561 --- /dev/null +++ b/tests/storage/testnatutils.nim @@ -0,0 +1,67 @@ +import std/[unittest, net, options] +import pkg/chronos +import ../../storage/utils/natutils + +suite "isGlobalUnicast": + test "localhost IPv4 is not global unicast": + check not isGlobalUnicast(parseIpAddress("127.0.0.1")) + + test "unspecified IPv4 is not global unicast": + check not isGlobalUnicast(parseIpAddress("0.0.0.0")) + + test "link-local IPv4 is not global unicast": + check not isGlobalUnicast(parseIpAddress("169.254.1.1")) + + test "private IPv4 is not global unicast": + check not isGlobalUnicast(parseIpAddress("10.0.0.1")) + + test "public IPv4 is global unicast": + check isGlobalUnicast(parseIpAddress("8.8.8.8")) + + test "localhost IPv6 is not global unicast": + check not isGlobalUnicast(parseIpAddress("::1")) + + test "unspecified IPv6 is not global unicast": + check not isGlobalUnicast(parseIpAddress("::")) + + test "link-local IPv6 is not global unicast": + check not isGlobalUnicast(parseIpAddress("fe80::1")) + + test "private IPv6 is not global unicast": + check not isGlobalUnicast(parseIpAddress("fc00::1")) + + test "public IPv6 is global unicast": + check isGlobalUnicast(parseIpAddress("2606:4700::1")) + +suite "getRoute": + test "getRouteIpv4 returns a valid IPv4": + let res = getRouteIpv4() + + check res.isOk + check res.get().family == IpAddressFamily.IPv4 + + test "getRouteIpv6 returns a valid IPv6": + let res = getRouteIpv6() + # If the machine does not have a global route because + # it is not configured for IPv6, the test will fail + # because it didn't find the best route. In that case, + # we can just skip the test, because it is not a problem + # with the test itself but the machine configuration. + if res.isErr: + check res.error == "No best route found" + else: + check res.get().family == IpAddressFamily.IPv6 + +suite "getBestLocalAddress": + test "specific IPv4 is returned as it is": + let ip = parseIpAddress("192.168.1.1") + check getBestLocalAddress(ip) == some(ip) + + test "specific IPv6 is returned as it is": + let ip = parseIpAddress("2606:4700::1") + check getBestLocalAddress(ip) == some(ip) + + test "0.0.0.0 resolves to a local IPv4": + let res = getBestLocalAddress(parseIpAddress("0.0.0.0")) + check res.isSome + check res.get().family == IpAddressFamily.IPv4 From aac617ace5185b36ae4fa96aecf30102d37619cb Mon Sep 17 00:00:00 2001 From: Arnaud Date: Fri, 10 Apr 2026 18:28:21 +0400 Subject: [PATCH 003/167] Autonat integration and reachability info in debug api --- storage/rest/api.nim | 12 ++++++++-- storage/storage.nim | 55 ++++++++++++++++++++++++++++++++++++++------ 2 files changed, 58 insertions(+), 9 deletions(-) diff --git a/storage/rest/api.nim b/storage/rest/api.nim index 865591fc..af4b009e 100644 --- a/storage/rest/api.nim +++ b/storage/rest/api.nim @@ -23,6 +23,7 @@ import pkg/confutils import pkg/libp2p import pkg/libp2p/routing_record +import pkg/libp2p/protocols/connectivity/autonat/service import pkg/codexdht/discv5/spr as spr import ../logutils @@ -557,7 +558,12 @@ proc initNodeApi(node: StorageNodeRef, conf: StorageConf, router: var RestRouter return RestApiResponse.error(Http500, "Unknown error dialling peer", headers = headers) -proc initDebugApi(node: StorageNodeRef, conf: StorageConf, router: var RestRouter) = +proc initDebugApi( + node: StorageNodeRef, + conf: StorageConf, + autonat: AutonatService, + router: var RestRouter, +) = let allowedOrigin = router.allowedOrigin router.api(MethodGet, "/api/storage/v1/debug/info") do() -> RestApiResponse: @@ -577,6 +583,7 @@ proc initDebugApi(node: StorageNodeRef, conf: StorageConf, router: var RestRoute "announceAddresses": node.discovery.announceAddrs, "table": table, "storage": {"version": $storageVersion, "revision": $storageRevision}, + "nat": {"reachability": $autonat.networkReachability}, } # return pretty json for human readability @@ -637,12 +644,13 @@ proc initRestApi*( node: StorageNodeRef, conf: StorageConf, repoStore: RepoStore, + autonat: AutonatService, corsAllowedOrigin: ?string, ): RestRouter = var router = RestRouter.init(validate, corsAllowedOrigin) initDataApi(node, repoStore, router) initNodeApi(node, conf, router) - initDebugApi(node, conf, router) + initDebugApi(node, conf, autonat, router) return router diff --git a/storage/storage.nim b/storage/storage.nim index 678f5f87..23b4e626 100644 --- a/storage/storage.nim +++ b/storage/storage.nim @@ -17,6 +17,7 @@ import pkg/chronos import pkg/taskpools import pkg/presto import pkg/libp2p +import pkg/libp2p/protocols/connectivity/autonat/[service, client] import pkg/confutils import pkg/confutils/defs import pkg/stew/io2 @@ -51,6 +52,7 @@ type repoStore: RepoStore maintenance: BlockMaintainer taskpool: Taskpool + autonatService*: AutonatService isStarted: bool StoragePrivateKey* = libp2p.PrivateKey # alias @@ -76,9 +78,25 @@ proc start*(s: StorageServer) {.async.} = await s.storageNode.switch.start() - let (announceAddrs, discoveryAddrs) = nattedAddress( - s.config.nat, s.storageNode.switch.peerInfo.addrs, s.config.discoveryPort - ) + let announceIp = + if s.config.nat.hasExtIp: + some(s.config.nat.extIp) + else: + getBestLocalAddress(s.config.listenIp) + + if announceIp.isNone: + # We should have an IP, even at private IP + raise newException(StorageError, "Unable to determine an IP address to announce") + + # Remap switch addresses to the resolved IP (replaces 0.0.0.0 or :: with the actual address), + # keeping unique entries only. + let announceAddrs = s.storageNode.switch.peerInfo.addrs + .mapIt(it.remapAddr(ip = announceIp, port = none(Port))) + .deduplicate() + let discoveryAddrs = + @[getMultiAddrWithIPAndUDPPort(announceIp.get, s.config.discoveryPort)] + s.storageNode.discovery.updateDhtRecord(discoveryAddrs) + s.storageNode.discovery.updateAnnounceRecord(announceAddrs) var hasPublicAddr = false for announceAddr in announceAddrs: @@ -90,9 +108,6 @@ proc start*(s: StorageServer) {.async.} = if not hasPublicAddr: warn "Unable to determine a public IP address. This node will only be reachable on a private network." - s.storageNode.discovery.updateAnnounceRecord(announceAddrs) - s.storageNode.discovery.updateDhtRecord(discoveryAddrs) - await s.storageNode.start() if s.restServer != nil: @@ -171,6 +186,16 @@ proc new*( ## create StorageServer including setting up datastore, repostore, etc let listenMultiAddr = getMultiAddrWithIpAndTcpPort(config.listenIp, config.listenPort) + let autonatService = AutonatService.new( + autonatClient = AutonatClient.new(), + rng = random.Rng.instance(), + scheduleInterval = Opt.some(config.natScheduleInterval), + askNewConnectedPeers = true, + numPeersToAsk = config.natNumPeersToAsk, + maxQueueSize = config.natMaxQueueSize, + minConfidence = config.natMinConfidence, + ) + let switch = SwitchBuilder .new() .withPrivateKey(privateKey) @@ -183,6 +208,8 @@ proc new*( .withAgentVersion(config.agentString) .withSignedPeerRecord(true) .withTcpTransport({ServerFlags.ReuseAddr, ServerFlags.TcpNoDelay}) + .withAutonat() + .withServices(@[Service(autonatService)]) .build() var taskPool: Taskpool @@ -304,7 +331,9 @@ proc new*( if config.apiBindAddress.isSome: restServer = RestServerRef .new( - storageNode.initRestApi(config, repoStore, config.apiCorsAllowedOrigin), + storageNode.initRestApi( + config, repoStore, autonatService, config.apiCorsAllowedOrigin + ), initTAddress(config.apiBindAddress.get(), config.apiPort), bufferSize = (1024 * 64), maxRequestBodySize = int.high, @@ -314,6 +343,17 @@ proc new*( switch.mount(network) switch.mount(manifestProto) + autonatService.statusAndConfidenceHandler( + proc( + networkReachability: NetworkReachability, confidence: Opt[float] + ) {.async: (raises: [CancelledError]).} = + if networkReachability == NotReachable: + let (announceAddrs, discoveryAddrs) = + nattedAddress(config.nat, switch.peerInfo.addrs, config.discoveryPort) + discovery.updateAnnounceRecord(announceAddrs) + discovery.updateDhtRecord(discoveryAddrs) + ) + StorageServer( config: config, storageNode: storageNode, @@ -322,4 +362,5 @@ proc new*( maintenance: maintenance, taskPool: taskPool, logFile: logFile, + autonatService: autonatService, ) From 1567540b31fe751629947468fb0cda71fb978d39 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Fri, 10 Apr 2026 18:28:37 +0400 Subject: [PATCH 004/167] Autonat tests --- tests/integration/1_minute/testnat.nim | 44 ++++++++++++++++++++++++++ tests/integration/storageclient.nim | 43 ++++++++++++++++++------- tests/integration/storageconfig.nim | 41 ++++++++++++++++++++++++ 3 files changed, 117 insertions(+), 11 deletions(-) create mode 100644 tests/integration/1_minute/testnat.nim diff --git a/tests/integration/1_minute/testnat.nim b/tests/integration/1_minute/testnat.nim new file mode 100644 index 00000000..d7b9288f --- /dev/null +++ b/tests/integration/1_minute/testnat.nim @@ -0,0 +1,44 @@ +import std/json +import std/options +import std/sequtils +import pkg/chronos +import pkg/questionable/results + +import ../multinodes +import ../storageclient +import ../storageconfig + +multinodesuite "AutoNAT integration": + let natConfig = NodeConfigs( + clients: StorageConfigs + .init(nodes = 2) + .withNatNumPeersToAsk(1) + .withNatMinConfidence(0.5) + .withNatScheduleInterval(10.seconds) + .withNatMaxQueueSize(1) + .withLogFile() + .withLogLevel("DEBUG").some + ) + + # Reminder: multinodesuite setup the first node as bootstrap node + test "node is reachable when using bootstrap node on same network", natConfig: + let node1 = clients()[0] + let node2 = clients()[1] + + # Temporary + # DHT exposes only UDP information + # So we force temporary by connection the 2 node together + # to update the autonat reachability + let info = await node2.client.info() + + check not info.isErr + + await node1.client.connectPeer( + info.get()["id"].getStr(), info.get()["addrs"].getElems().mapIt(it.getStr()) + ) + + check eventuallySafe( + (await node1.client.natReachability()).get() == "Reachable", + timeout = 30_000, + pollInterval = 500, + ) diff --git a/tests/integration/storageclient.nim b/tests/integration/storageclient.nim index ec990bb9..c6ef5be0 100644 --- a/tests/integration/storageclient.nim +++ b/tests/integration/storageclient.nim @@ -1,4 +1,5 @@ import std/strutils +import std/sequtils from pkg/libp2p import Cid, `$`, init import pkg/questionable/results @@ -32,17 +33,17 @@ proc request( async: (raw: true, raises: [CancelledError, HttpError]) .} = HttpClientRequestRef - .new( - self.session, - url, - httpMethod, - version = HttpVersion11, - flags = {}, - maxResponseHeadersSize = HttpMaxHeadersSize, - headers = headers, - body = body.toOpenArrayByte(0, len(body) - 1), - ).get - .send() + .new( + self.session, + url, + httpMethod, + version = HttpVersion11, + flags = {}, + maxResponseHeadersSize = HttpMaxHeadersSize, + headers = headers, + body = body.toOpenArrayByte(0, len(body) - 1), + ).get + .send() proc post*( self: StorageClient, @@ -260,3 +261,23 @@ proc hasBlockRaw*( .} = let url = client.baseurl & "/data/" & cid & "/exists" return client.get(url) + +proc connectPeer*( + client: StorageClient, peerId: string, addrs: seq[string] +): Future[void] {.async: (raises: [CancelledError, HttpError]).} = + var url = client.baseurl & "/connect/" & peerId + if addrs.len > 0: + url &= "?" & addrs.mapIt("addrs=" & it).join("&") + let response = await client.get(url) + assert response.status == 200 + +proc natReachability*( + client: StorageClient +): Future[?!string] {.async: (raises: [CancelledError, HttpError]).} = + let info = await client.info() + if info.isErr: + return failure "Failed to get node info" + try: + return info.get()["nat"]["reachability"].getStr().success + except KeyError as e: + return failure e.msg diff --git a/tests/integration/storageconfig.nim b/tests/integration/storageconfig.nim index 4aeb6d60..240d44a2 100644 --- a/tests/integration/storageconfig.nim +++ b/tests/integration/storageconfig.nim @@ -5,6 +5,7 @@ import std/strutils import std/sugar import std/tables from pkg/chronicles import LogLevel +import pkg/chronos import pkg/storage/conf import pkg/storage/units import pkg/confutils @@ -280,3 +281,43 @@ proc withStorageQuota*( for config in startConfig.configs.mitems: config.addCliOption("--storage-quota", $quota) return startConfig + +proc withListenIp*( + self: StorageConfigs, ip: string +): StorageConfigs {.raises: [StorageConfigError].} = + var startConfig = self + for config in startConfig.configs.mitems: + config.addCliOption("--listen-ip", ip) + return startConfig + +proc withNatNumPeersToAsk*( + self: StorageConfigs, numPeersToAsk: int +): StorageConfigs {.raises: [StorageConfigError].} = + var startConfig = self + for config in startConfig.configs.mitems: + config.addCliOption("--nat-num-peers-to-ask", $numPeersToAsk) + return startConfig + +proc withNatMaxQueueSize*( + self: StorageConfigs, maxQueueSize: int +): StorageConfigs {.raises: [StorageConfigError].} = + var startConfig = self + for config in startConfig.configs.mitems: + config.addCliOption("--nat-max-queue-size", $maxQueueSize) + return startConfig + +proc withNatMinConfidence*( + self: StorageConfigs, minConfidence: float +): StorageConfigs {.raises: [StorageConfigError].} = + var startConfig = self + for config in startConfig.configs.mitems: + config.addCliOption("--nat-min-confidence", $minConfidence) + return startConfig + +proc withNatScheduleInterval*( + self: StorageConfigs, scheduleInterval: Duration +): StorageConfigs {.raises: [StorageConfigError].} = + var startConfig = self + for config in startConfig.configs.mitems: + config.addCliOption("--nat-schedule-interval", $scheduleInterval) + return startConfig From 5222d5dad11a7de8ba59b58d1c90ab73c8ffc19e Mon Sep 17 00:00:00 2001 From: Arnaud Date: Fri, 10 Apr 2026 19:43:50 +0400 Subject: [PATCH 005/167] Add reachability to debug info in lib --- .claude/settings.local.json | 23 ++ .../requests/node_debug_request.nim | 2 + nat-hackmd.md | 235 ++++++++++++++++++ 3 files changed, 260 insertions(+) create mode 100644 .claude/settings.local.json create mode 100644 nat-hackmd.md diff --git a/.claude/settings.local.json b/.claude/settings.local.json new file mode 100644 index 00000000..4494ff72 --- /dev/null +++ b/.claude/settings.local.json @@ -0,0 +1,23 @@ +{ + "permissions": { + "allow": [ + "Bash(grep -rn \"testnat\" /home/arnaud/Work/logos/logos-storage-nim/*.nims)", + "Bash(grep:*)", + "Bash(find:*)", + "Bash(curl -s http://127.0.0.1:8080/api/storage/v1/debug/info)", + "Bash(ls /home/arnaud/Work/logos/logos-storage-nim/tests/*.cfg /home/arnaud/Work/logos/logos-storage-nim/tests/*.nim)", + "Bash(nph --version)", + "Bash(make print-nph-path:*)", + "Bash(vendor/nimbus-build-system/vendor/Nim/bin/nph:*)", + "Bash(nimble build:*)", + "Bash(nim --version)", + "Bash(nimble --version)", + "Bash(/home/arnaud/.nimble/bin/nim --version)", + "Bash(make -n build-nph)", + "Bash(make build-nph:*)", + "Bash(/home/arnaud/.nimble/bin/nim c:*)", + "Bash(NIMFLAGS=\"--skipParentCfg\" nimble build)", + "Bash(NIMFLAGS=\"--skipParentCfg\" nimble build --verbose)" + ] + } +} diff --git a/library/storage_thread_requests/requests/node_debug_request.nim b/library/storage_thread_requests/requests/node_debug_request.nim index 8bf3106c..fe9bd389 100644 --- a/library/storage_thread_requests/requests/node_debug_request.nim +++ b/library/storage_thread_requests/requests/node_debug_request.nim @@ -9,6 +9,7 @@ import std/[options] import chronos import chronicles import codexdht/discv5/spr +import pkg/libp2p/protocols/connectivity/autonat/service import ../../alloc import ../../../storage/conf import ../../../storage/rest/json @@ -59,6 +60,7 @@ proc getDebug( if node.discovery.dhtRecord.isSome: node.discovery.dhtRecord.get.toURI else: "", "announceAddresses": node.discovery.announceAddrs, "table": table, + "nat": {"reachability": $storage[].autonatService.networkReachability}, } return ok($json) diff --git a/nat-hackmd.md b/nat-hackmd.md new file mode 100644 index 00000000..570d21f4 --- /dev/null +++ b/nat-hackmd.md @@ -0,0 +1,235 @@ +# NAT Traversal + +## Context + +A logos-storage-nim node needs to tell other peers how to reach it. This is harder than it sounds: most nodes are behind a NAT or a firewall. From the outside, the node looks unreachable. + +UPnP / NAT-PMP is already implemented in `nat.nim`. This document describes how we build on top of it using libp2p's AutoNAT to implement a full NAT traversal strategy. + +--- + +## Overview + +```mermaid +flowchart TD + A[Startup] --> B[Step 1: Collect address candidates] + B --> C[Step 2: AutoNAT] + C --> D{Result?} + D -- Reachable --> E[DHT server mode and announce direct address] + D -- Not Reachable --> F[Step 3: UPnP / NAT-PMP] + F --> G{Mapping OK?} + G -- Yes --> E + G -- No --> H[Step 4: AutoRelayService] + H --> E2[DHT client mode and announce relay address] + E --> I[AutoNAT every 5min] + E2 --> I + I --> D + F -- UPnp recheck --> F +``` + +--- + +## Step 1: Collecting address candidates + +At startup, the node builds a list of IP / port pairs it could announce to other peers. + +### IP addresses + +If `--listen-ip` is set to a specific address, only that address is used. Otherwise the node scans its network interfaces: + +| `--listen-ip` | What gets collected | +| --- | --- | +| `0.0.0.0` (default) | all local IPv4 addresses | +| `::` | all local IPv4 and IPv6 addresses | + +### Port + +The TCP port comes from `--listen-port`. If not set, a random free port is picked at startup. + +--- + +### Initial announcement + +At startup, the node resolves its initial addresses from the routing table and announces them. If `--nat:extip` is set, the static external IP is announced directly instead. + +No UPnP or NAT-PMP is attempted at this stage. That happens later in Step 3 if AutoNAT reports the node is not reachable. + +The node starts in DHT client mode. AutoNAT (Step 2) runs in the background to check if the node is reachable from the outside, and switches to server mode if confirmed. + +This ensures connectivity from the start, whether on a local or public network. If an address turns out to be unreachable, AutoNAT will detect it and update the announced addresses accordingly. + +--- + +### IPv6 specifics + +- Do not run UPnP or NAT-PMP on IPv6 addresses: they are directly routable, no port mapping needed. A node with a stable global IPv6 address can skip AutoNAT, UPnP, and relay entirely for that address. +- Some IPv6 addresses are temporary and change over time. If we announce one and it changes, the node becomes unreachable before the DHT records expire. We must only announce the stable address. +- chronos has no support for distinguishing stable addresses from temporary ones. chronos needs to be updated to expose address stability flags. + +--- + +### Initial DHT mode + +The DHT has two modes: + +- **Server mode**: the node appears in the routing tables of other nodes and answers their discovery requests ([`handleFindNode`](https://github.com/logos-storage/logos-storage-nim-dht/blob/6c7de036224724b064dcaa6b1d898be1c6d03242/codexdht/private/eth/p2p/discoveryv5/protocol.nim#L338), [`handlePing`](https://github.com/logos-storage/logos-storage-nim-dht/blob/6c7de036224724b064dcaa6b1d898be1c6d03242/codexdht/private/eth/p2p/discoveryv5/protocol.nim#L330)). Other nodes use it to route their own lookups. +- **Client mode**: the node can query the DHT and publish provider records (telling the network "I have this content at this address"), but it ignores inbound routing requests and does not appear in routing tables. A node that is not directly reachable must stay in client mode: if it were in server mode, other nodes would try to contact it, get timeouts, and that would degrade the DHT for everyone. + +The node starts in client mode. It switches to server mode only after AutoNAT confirms it is reachable (Step 2). + +--- + +## Step 2: AutoNAT + +After collecting and filtering addresses (Step 1), the node needs to check if it is actually reachable from the outside. It does this using AutoNAT: it asks a few connected peers to try to connect back to it. Bootstrap nodes are the first peers available for this, as they are dialed at startup. + +**Note:** AutoNAT tests TCP reachability via the libp2p switch, we infer UDP reachability from it. + +### How it works + +The node asks a few peers: "try to connect to me at this address". Each peer attempts the connection and reports success or failure. The results are collected and a confidence score is calculated. When confidence crosses a threshold, the node is marked `Reachable` or `NotReachable`. + +### Reference implementation + +logos-delivery already has a minimal setup at [`waku/discovery/autonat_service.nim`](https://github.com/logos-messaging/logos-delivery/blob/0b86093247da92060c503544d39e5d0a23922c15/waku/discovery/autonat_service.nim): + +```nim +AutonatService.new( + autonatClient = AutonatClient.new(), + rng = rng, + scheduleInterval = Opt.some(30.seconds), # logos-storage uses 5min + askNewConnectedPeers = true, # triggers a check on first bootstrap connection, avoids waiting 5min at startup + numPeersToAsk = 3, + maxQueueSize = 3, + minConfidence = 0.7, +) +``` + +### Parameters + +| Parameter | logos-delivery | logos-storage | libp2p default | Notes | +| --- | --- | --- | --- | --- | +| `numPeersToAsk` | 3 | 3 | 5 | how many peers are asked per round | +| `maxQueueSize` | 3 | 3 | 10 | how many past results are kept to calculate confidence | +| `minConfidence` | 0.7 | 0.7 | 0.3 | fraction of successful answers needed to confirm a state | +| `scheduleInterval` | 30s | 5min | none | reachability changes rarely in a storage network | +| `askNewConnectedPeers` | false | true | true | triggers a check on first bootstrap connection | + +logos-delivery uses a higher confidence threshold (0.7 vs 0.3) and a smaller history window (3 vs 10): fewer samples but a stricter bar to confirm reachability. + +libp2p default values are available [here](https://github.com/vacp2p/nim-libp2p/blob/e82080f7b1aa61c6d35fa5311b873f41eff4bb52/libp2p/protocols/connectivity/autonat/service.nim#L60-L64). + +## Step 3: UPnP / NAT-PMP (fallback) + +If AutoNAT says the node is not reachable, we try to ask the router to open a port using UPnP or NAT-PMP already implemented. + +### What already exists + +The implementation is already in `nat.nim`: +- [`getExternalIP()`](https://github.com/logos-storage/logos-storage-nim/blob/48f2508b07e51a222070ada72c254927da9c5806/storage/nat.nim#L67): tries UPnP first, then NAT-PMP, returns the external IP +- [`redirectPorts()`](https://github.com/logos-storage/logos-storage-nim/blob/48f2508b07e51a222070ada72c254927da9c5806/storage/nat.nim#L305): creates the port mapping on the router +- [`nattedAddress()`](https://github.com/logos-storage/logos-storage-nim/blob/48f2508b07e51a222070ada72c254927da9c5806/storage/nat.nim#L400): calls both and returns the updated addresses to announce + +### The problem + +[`nattedAddress()` is called unconditionally at startup](https://github.com/logos-storage/logos-storage-nim/blob/48f2508b07e51a222070ada72c254927da9c5806/storage/storage.nim#L79), before AutoNAT has run. It should only be called when AutoNAT returns `NotReachable`. + +### What needs to change + +Move the `nattedAddress()` call out of `start()` and into the AutoNAT `statusAndConfidenceHandler`: + +```nim +of NotReachable: + let (announceAddrs, discoveryAddrs) = nattedAddress( + config.nat, switch.peerInfo.addrs, config.discoveryPort + ) + discovery.updateAnnounceRecord(announceAddrs) + discovery.updateDhtRecord(discoveryAddrs) +``` + +`nattedAddress()` opens a port on the router and starts [`repeatPortMapping`](https://github.com/logos-storage/logos-storage-nim/blob/48f2508b07e51a222070ada72c254927da9c5806/storage/nat.nim#L231) in the background to renew it every [`20 min`](https://github.com/logos-storage/logos-storage-nim/blob/48f2508b07e51a222070ada72c254927da9c5806/storage/nat.nim#L28) (routers forget the mapping on reboot for UPnP, or after [`1 hour`](https://github.com/logos-storage/logos-storage-nim/blob/48f2508b07e51a222070ada72c254927da9c5806/storage/nat.nim#L29) for NAT-PMP). It only needs to be called once, if it fails, the node falls back to relay immediately. + +### States + +The `statusAndConfidenceHandler` needs to track what has already been tried to avoid re-running UPnP on every AutoNAT cycle: + +- `Unknown` + - `NotReachable`: try UPnP + - `Reachable`: switch DHT to server +- `UPnP` + - `NotReachable`: switch DHT to client, start relay + - `Reachable`: switch DHT to server + - background: nat.nim renews the mapping every 20 min and fires `onMappingRestored` on restoration +- `Relay` + - `NotReachable`: do nothing + - if `hasExtIp: true`: start a background task that periodically asks a peer to dial the static IP. On success, fires the same `onMappingRestored` callback + +When applying a state transition: always update announce records first, then change DHT mode, then start or stop the relay. If you set server mode before publishing the new addresses, peers will try to contact you before your records are up to date. + +**Note:** nim-libp2p's AutoNAT [filters out relay addresses](https://github.com/vacp2p/nim-libp2p/blob/e82080f7b1aa61c6d35fa5311b873f41eff4bb52/libp2p/protocols/connectivity/autonat/server.nim#L122-L126) before attempting dial-back. A node behind relay will always get `NotReachable` from AutoNAT. + +## Step 4: Relay and hole punching + +We do not use `HPService` because it starts the relay immediately on `NotReachable`, bypassing the UPnP step. Instead we wire `AutonatService` and `AutoRelayService` directly and control the relay from our own `statusAndConfidenceHandler`. + +A node in relay mode can receive inbound connections via the relay, so other peers can download data from it. The relay server acts as a middleman: the node connects to it, reserves a slot, and gets a relay address of the form `/ip4//tcp//p2p//p2p-circuit/p2p/`. It publishes this address in its DHT provider records so peers looking for its content can find and connect to it. It stays in DHT client mode (see Step 1): it can publish provider records but cannot act as a routing hop. + +Bootstrap nodes serve as the initial relay servers. `AutoRelayService` finds relay candidates among connected peers, so bootstrap nodes are the first ones used. + +### Setup + +```nim +let relayClient = RelayClient.new() +let autoRelayService = AutoRelayService.new(2, relayClient, onReservation, rng) + +let switch = SwitchBuilder + .new() + ... + .withServices(@[Service(autonatService)]) + .build() +``` + +The `onReservation` callback updates DHT addresses when relay reservations change: + +```nim +proc onReservation(addresses: seq[MultiAddress]) {.gcsafe, raises: [].} = + discovery.updateAnnounceRecord(addresses) + discovery.updateDhtRecord(addresses) +``` + +`updateAnnounceRecord` updates the libp2p peer record (multiaddr, used for content routing). `updateDhtRecord` updates the discv5 node record (SPR, used for peer discovery). Both replace the current set of addresses, they do not accumulate. + +### Mapping restored callback + +`repeatPortMapping` runs in the background even in `Relay` state, this is how we detect when the mapping comes back. It needs a callback added so we can stop the relay and update addresses when it fires. The external IP may have changed after a router reboot, so we always update regardless. + +```nim +proc onMappingRestored(addrs: seq[MultiAddress]) = + if natState == Relay: + await autoRelayService.stop(switch) + natState = UPnP + # addrs may differ from previously announced ones (e.g. router reboot changed external IP) + discovery.updateAnnounceRecord(addrs) + discovery.updateDhtRecord(addrs) +``` + +### Hole punching + +When a peer connects through a relay, libp2p automatically tries to establish a direct connection using dcutr (a protocol for hole punching). If it works, that specific connection bypasses the relay: lower latency and less load on the relay server. This does not change the node's reachability: new peers still need to go through the relay first. + +## Step 5: Periodic Re-evaluation + +`AutonatService` re-runs every 5 minutes. On each cycle it asks `numPeersToAsk` peers to dial back. If confidence crosses `minConfidence`, `statusAndConfidenceHandler` fires and our handler updates DHT mode, relay state, and announced addresses. The `onMappingRestored` callback from `nat.nim` can also fire independently between cycles. + +## What needs to be implemented + +- DHT client/server mode in `discovery.nim`: the node currently starts as a DHT server unconditionally +- Move `nattedAddress()` call out of `start()` and into the AutoNAT `statusAndConfidenceHandler` +- Expose a `onRestored` callback in `nat.nim`'s `repeatPortMapping` +- For `hasExtIp: true` (manual port forwarding): add a background task that periodically asks a peer to dial the static IP directly. If it succeeds, fire the same `onMappingRestored` callback. Without this, a node in `Relay` state with a restored manual port forwarding has no way to recover automatically. +- `statusAndConfidenceHandler` with `natState` tracking +- chronos needs to be updated to expose IPv6 address stability flags to distinguish stable SLAAC addresses from temporary ones + +## Open questions + +- [ ] Mobile nodes: UDP is often blocked on cellular networks. discv5 is UDP-only. How do we support mobile participation? From ef19eb134d47eb831a18ccedff165432b5367528 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Mon, 13 Apr 2026 10:06:19 +0400 Subject: [PATCH 006/167] Fix format --- tests/integration/storageclient.nim | 22 +++++++++++----------- tests/storage/testnat.nim | 22 ++++++++++------------ 2 files changed, 21 insertions(+), 23 deletions(-) diff --git a/tests/integration/storageclient.nim b/tests/integration/storageclient.nim index c6ef5be0..62d72917 100644 --- a/tests/integration/storageclient.nim +++ b/tests/integration/storageclient.nim @@ -33,17 +33,17 @@ proc request( async: (raw: true, raises: [CancelledError, HttpError]) .} = HttpClientRequestRef - .new( - self.session, - url, - httpMethod, - version = HttpVersion11, - flags = {}, - maxResponseHeadersSize = HttpMaxHeadersSize, - headers = headers, - body = body.toOpenArrayByte(0, len(body) - 1), - ).get - .send() + .new( + self.session, + url, + httpMethod, + version = HttpVersion11, + flags = {}, + maxResponseHeadersSize = HttpMaxHeadersSize, + headers = headers, + body = body.toOpenArrayByte(0, len(body) - 1), + ).get + .send() proc post*( self: StorageClient, diff --git a/tests/storage/testnat.nim b/tests/storage/testnat.nim index eb93c1ff..08d87c9c 100644 --- a/tests/storage/testnat.nim +++ b/tests/storage/testnat.nim @@ -21,18 +21,16 @@ suite "NAT Address Tests": # Expected results let - expectedDiscoveryAddrs = - @[ - MultiAddress.init("/ip4/8.8.8.8/udp/1234").expect("valid multiaddr"), - MultiAddress.init("/ip4/8.8.8.8/udp/1234").expect("valid multiaddr"), - MultiAddress.init("/ip4/8.8.8.8/udp/1234").expect("valid multiaddr"), - ] - expectedlibp2pAddrs = - @[ - MultiAddress.init("/ip4/8.8.8.8/tcp/5000").expect("valid multiaddr"), - MultiAddress.init("/ip4/8.8.8.8/tcp/5000").expect("valid multiaddr"), - MultiAddress.init("/ip4/8.8.8.8/tcp/5000").expect("valid multiaddr"), - ] + expectedDiscoveryAddrs = @[ + MultiAddress.init("/ip4/8.8.8.8/udp/1234").expect("valid multiaddr"), + MultiAddress.init("/ip4/8.8.8.8/udp/1234").expect("valid multiaddr"), + MultiAddress.init("/ip4/8.8.8.8/udp/1234").expect("valid multiaddr"), + ] + expectedlibp2pAddrs = @[ + MultiAddress.init("/ip4/8.8.8.8/tcp/5000").expect("valid multiaddr"), + MultiAddress.init("/ip4/8.8.8.8/tcp/5000").expect("valid multiaddr"), + MultiAddress.init("/ip4/8.8.8.8/tcp/5000").expect("valid multiaddr"), + ] #ipv6Addr = MultiAddress.init("/ip6/::1/tcp/5000").expect("valid multiaddr") addrs = @[localAddr, anyAddr, publicAddr] From 5c2581b87e5967985865557e5dd44f8b9ef43950 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Mon, 13 Apr 2026 11:04:21 +0400 Subject: [PATCH 007/167] Debug macos error --- storage/utils/natutils.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/storage/utils/natutils.nim b/storage/utils/natutils.nim index 37228236..6a7984d3 100644 --- a/storage/utils/natutils.nim +++ b/storage/utils/natutils.nim @@ -28,6 +28,7 @@ proc getRoute(publicAddress: TransportAddress): Result[IpAddress, cstring] = route.source.address() except ValueError as e: # This should not occur really. + echo "Address conversion error: ", e.name, " ", e.msg error "Address conversion error", exception = e.name, msg = e.msg return err("Invalid IP address") ok(ip) From 2f290eb03404b910c5746d55af75c29c994ee199 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Mon, 13 Apr 2026 11:22:14 +0400 Subject: [PATCH 008/167] Fix no route detection when AddressFamily is non --- storage/utils/natutils.nim | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/storage/utils/natutils.nim b/storage/utils/natutils.nim index 6a7984d3..fa63a9cc 100644 --- a/storage/utils/natutils.nim +++ b/storage/utils/natutils.nim @@ -20,7 +20,7 @@ func isGlobalUnicast*(address: IpAddress): bool = proc getRoute(publicAddress: TransportAddress): Result[IpAddress, cstring] = let route = getBestRoute(publicAddress) - if route.source.isUnspecified(): + if route.source == AddressFamily.None or route.source.isUnspecified(): err("No best route found") else: let ip = @@ -28,7 +28,6 @@ proc getRoute(publicAddress: TransportAddress): Result[IpAddress, cstring] = route.source.address() except ValueError as e: # This should not occur really. - echo "Address conversion error: ", e.name, " ", e.msg error "Address conversion error", exception = e.name, msg = e.msg return err("Invalid IP address") ok(ip) From 1a7de3dc17b21ab707305ec3d1d22a292448d2a5 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Mon, 13 Apr 2026 11:38:23 +0400 Subject: [PATCH 009/167] Fix route family condition --- storage/utils/natutils.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/utils/natutils.nim b/storage/utils/natutils.nim index fa63a9cc..04447d26 100644 --- a/storage/utils/natutils.nim +++ b/storage/utils/natutils.nim @@ -20,7 +20,7 @@ func isGlobalUnicast*(address: IpAddress): bool = proc getRoute(publicAddress: TransportAddress): Result[IpAddress, cstring] = let route = getBestRoute(publicAddress) - if route.source == AddressFamily.None or route.source.isUnspecified(): + if route.source.family == AddressFamily.None or route.source.isUnspecified(): err("No best route found") else: let ip = From ec9052ef468ae6ee760542e8206d440aac3a6cca Mon Sep 17 00:00:00 2001 From: Arnaud Date: Mon, 13 Apr 2026 17:46:42 +0400 Subject: [PATCH 010/167] Fix testnat import --- tests/storage/testnat.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/storage/testnat.nim b/tests/storage/testnat.nim index 08d87c9c..71155540 100644 --- a/tests/storage/testnat.nim +++ b/tests/storage/testnat.nim @@ -5,6 +5,7 @@ import pkg/results import ../../storage/nat import ../../storage/utils +import ../../storage/utils/natutils suite "NAT Address Tests": test "nattedAddress with local addresses": From 94762deb89214080b7a49458a7e19914fb5688e5 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Wed, 15 Apr 2026 09:07:14 +0400 Subject: [PATCH 011/167] Remove doc --- nat-hackmd.md | 235 -------------------------------------------------- 1 file changed, 235 deletions(-) delete mode 100644 nat-hackmd.md diff --git a/nat-hackmd.md b/nat-hackmd.md deleted file mode 100644 index 570d21f4..00000000 --- a/nat-hackmd.md +++ /dev/null @@ -1,235 +0,0 @@ -# NAT Traversal - -## Context - -A logos-storage-nim node needs to tell other peers how to reach it. This is harder than it sounds: most nodes are behind a NAT or a firewall. From the outside, the node looks unreachable. - -UPnP / NAT-PMP is already implemented in `nat.nim`. This document describes how we build on top of it using libp2p's AutoNAT to implement a full NAT traversal strategy. - ---- - -## Overview - -```mermaid -flowchart TD - A[Startup] --> B[Step 1: Collect address candidates] - B --> C[Step 2: AutoNAT] - C --> D{Result?} - D -- Reachable --> E[DHT server mode and announce direct address] - D -- Not Reachable --> F[Step 3: UPnP / NAT-PMP] - F --> G{Mapping OK?} - G -- Yes --> E - G -- No --> H[Step 4: AutoRelayService] - H --> E2[DHT client mode and announce relay address] - E --> I[AutoNAT every 5min] - E2 --> I - I --> D - F -- UPnp recheck --> F -``` - ---- - -## Step 1: Collecting address candidates - -At startup, the node builds a list of IP / port pairs it could announce to other peers. - -### IP addresses - -If `--listen-ip` is set to a specific address, only that address is used. Otherwise the node scans its network interfaces: - -| `--listen-ip` | What gets collected | -| --- | --- | -| `0.0.0.0` (default) | all local IPv4 addresses | -| `::` | all local IPv4 and IPv6 addresses | - -### Port - -The TCP port comes from `--listen-port`. If not set, a random free port is picked at startup. - ---- - -### Initial announcement - -At startup, the node resolves its initial addresses from the routing table and announces them. If `--nat:extip` is set, the static external IP is announced directly instead. - -No UPnP or NAT-PMP is attempted at this stage. That happens later in Step 3 if AutoNAT reports the node is not reachable. - -The node starts in DHT client mode. AutoNAT (Step 2) runs in the background to check if the node is reachable from the outside, and switches to server mode if confirmed. - -This ensures connectivity from the start, whether on a local or public network. If an address turns out to be unreachable, AutoNAT will detect it and update the announced addresses accordingly. - ---- - -### IPv6 specifics - -- Do not run UPnP or NAT-PMP on IPv6 addresses: they are directly routable, no port mapping needed. A node with a stable global IPv6 address can skip AutoNAT, UPnP, and relay entirely for that address. -- Some IPv6 addresses are temporary and change over time. If we announce one and it changes, the node becomes unreachable before the DHT records expire. We must only announce the stable address. -- chronos has no support for distinguishing stable addresses from temporary ones. chronos needs to be updated to expose address stability flags. - ---- - -### Initial DHT mode - -The DHT has two modes: - -- **Server mode**: the node appears in the routing tables of other nodes and answers their discovery requests ([`handleFindNode`](https://github.com/logos-storage/logos-storage-nim-dht/blob/6c7de036224724b064dcaa6b1d898be1c6d03242/codexdht/private/eth/p2p/discoveryv5/protocol.nim#L338), [`handlePing`](https://github.com/logos-storage/logos-storage-nim-dht/blob/6c7de036224724b064dcaa6b1d898be1c6d03242/codexdht/private/eth/p2p/discoveryv5/protocol.nim#L330)). Other nodes use it to route their own lookups. -- **Client mode**: the node can query the DHT and publish provider records (telling the network "I have this content at this address"), but it ignores inbound routing requests and does not appear in routing tables. A node that is not directly reachable must stay in client mode: if it were in server mode, other nodes would try to contact it, get timeouts, and that would degrade the DHT for everyone. - -The node starts in client mode. It switches to server mode only after AutoNAT confirms it is reachable (Step 2). - ---- - -## Step 2: AutoNAT - -After collecting and filtering addresses (Step 1), the node needs to check if it is actually reachable from the outside. It does this using AutoNAT: it asks a few connected peers to try to connect back to it. Bootstrap nodes are the first peers available for this, as they are dialed at startup. - -**Note:** AutoNAT tests TCP reachability via the libp2p switch, we infer UDP reachability from it. - -### How it works - -The node asks a few peers: "try to connect to me at this address". Each peer attempts the connection and reports success or failure. The results are collected and a confidence score is calculated. When confidence crosses a threshold, the node is marked `Reachable` or `NotReachable`. - -### Reference implementation - -logos-delivery already has a minimal setup at [`waku/discovery/autonat_service.nim`](https://github.com/logos-messaging/logos-delivery/blob/0b86093247da92060c503544d39e5d0a23922c15/waku/discovery/autonat_service.nim): - -```nim -AutonatService.new( - autonatClient = AutonatClient.new(), - rng = rng, - scheduleInterval = Opt.some(30.seconds), # logos-storage uses 5min - askNewConnectedPeers = true, # triggers a check on first bootstrap connection, avoids waiting 5min at startup - numPeersToAsk = 3, - maxQueueSize = 3, - minConfidence = 0.7, -) -``` - -### Parameters - -| Parameter | logos-delivery | logos-storage | libp2p default | Notes | -| --- | --- | --- | --- | --- | -| `numPeersToAsk` | 3 | 3 | 5 | how many peers are asked per round | -| `maxQueueSize` | 3 | 3 | 10 | how many past results are kept to calculate confidence | -| `minConfidence` | 0.7 | 0.7 | 0.3 | fraction of successful answers needed to confirm a state | -| `scheduleInterval` | 30s | 5min | none | reachability changes rarely in a storage network | -| `askNewConnectedPeers` | false | true | true | triggers a check on first bootstrap connection | - -logos-delivery uses a higher confidence threshold (0.7 vs 0.3) and a smaller history window (3 vs 10): fewer samples but a stricter bar to confirm reachability. - -libp2p default values are available [here](https://github.com/vacp2p/nim-libp2p/blob/e82080f7b1aa61c6d35fa5311b873f41eff4bb52/libp2p/protocols/connectivity/autonat/service.nim#L60-L64). - -## Step 3: UPnP / NAT-PMP (fallback) - -If AutoNAT says the node is not reachable, we try to ask the router to open a port using UPnP or NAT-PMP already implemented. - -### What already exists - -The implementation is already in `nat.nim`: -- [`getExternalIP()`](https://github.com/logos-storage/logos-storage-nim/blob/48f2508b07e51a222070ada72c254927da9c5806/storage/nat.nim#L67): tries UPnP first, then NAT-PMP, returns the external IP -- [`redirectPorts()`](https://github.com/logos-storage/logos-storage-nim/blob/48f2508b07e51a222070ada72c254927da9c5806/storage/nat.nim#L305): creates the port mapping on the router -- [`nattedAddress()`](https://github.com/logos-storage/logos-storage-nim/blob/48f2508b07e51a222070ada72c254927da9c5806/storage/nat.nim#L400): calls both and returns the updated addresses to announce - -### The problem - -[`nattedAddress()` is called unconditionally at startup](https://github.com/logos-storage/logos-storage-nim/blob/48f2508b07e51a222070ada72c254927da9c5806/storage/storage.nim#L79), before AutoNAT has run. It should only be called when AutoNAT returns `NotReachable`. - -### What needs to change - -Move the `nattedAddress()` call out of `start()` and into the AutoNAT `statusAndConfidenceHandler`: - -```nim -of NotReachable: - let (announceAddrs, discoveryAddrs) = nattedAddress( - config.nat, switch.peerInfo.addrs, config.discoveryPort - ) - discovery.updateAnnounceRecord(announceAddrs) - discovery.updateDhtRecord(discoveryAddrs) -``` - -`nattedAddress()` opens a port on the router and starts [`repeatPortMapping`](https://github.com/logos-storage/logos-storage-nim/blob/48f2508b07e51a222070ada72c254927da9c5806/storage/nat.nim#L231) in the background to renew it every [`20 min`](https://github.com/logos-storage/logos-storage-nim/blob/48f2508b07e51a222070ada72c254927da9c5806/storage/nat.nim#L28) (routers forget the mapping on reboot for UPnP, or after [`1 hour`](https://github.com/logos-storage/logos-storage-nim/blob/48f2508b07e51a222070ada72c254927da9c5806/storage/nat.nim#L29) for NAT-PMP). It only needs to be called once, if it fails, the node falls back to relay immediately. - -### States - -The `statusAndConfidenceHandler` needs to track what has already been tried to avoid re-running UPnP on every AutoNAT cycle: - -- `Unknown` - - `NotReachable`: try UPnP - - `Reachable`: switch DHT to server -- `UPnP` - - `NotReachable`: switch DHT to client, start relay - - `Reachable`: switch DHT to server - - background: nat.nim renews the mapping every 20 min and fires `onMappingRestored` on restoration -- `Relay` - - `NotReachable`: do nothing - - if `hasExtIp: true`: start a background task that periodically asks a peer to dial the static IP. On success, fires the same `onMappingRestored` callback - -When applying a state transition: always update announce records first, then change DHT mode, then start or stop the relay. If you set server mode before publishing the new addresses, peers will try to contact you before your records are up to date. - -**Note:** nim-libp2p's AutoNAT [filters out relay addresses](https://github.com/vacp2p/nim-libp2p/blob/e82080f7b1aa61c6d35fa5311b873f41eff4bb52/libp2p/protocols/connectivity/autonat/server.nim#L122-L126) before attempting dial-back. A node behind relay will always get `NotReachable` from AutoNAT. - -## Step 4: Relay and hole punching - -We do not use `HPService` because it starts the relay immediately on `NotReachable`, bypassing the UPnP step. Instead we wire `AutonatService` and `AutoRelayService` directly and control the relay from our own `statusAndConfidenceHandler`. - -A node in relay mode can receive inbound connections via the relay, so other peers can download data from it. The relay server acts as a middleman: the node connects to it, reserves a slot, and gets a relay address of the form `/ip4//tcp//p2p//p2p-circuit/p2p/`. It publishes this address in its DHT provider records so peers looking for its content can find and connect to it. It stays in DHT client mode (see Step 1): it can publish provider records but cannot act as a routing hop. - -Bootstrap nodes serve as the initial relay servers. `AutoRelayService` finds relay candidates among connected peers, so bootstrap nodes are the first ones used. - -### Setup - -```nim -let relayClient = RelayClient.new() -let autoRelayService = AutoRelayService.new(2, relayClient, onReservation, rng) - -let switch = SwitchBuilder - .new() - ... - .withServices(@[Service(autonatService)]) - .build() -``` - -The `onReservation` callback updates DHT addresses when relay reservations change: - -```nim -proc onReservation(addresses: seq[MultiAddress]) {.gcsafe, raises: [].} = - discovery.updateAnnounceRecord(addresses) - discovery.updateDhtRecord(addresses) -``` - -`updateAnnounceRecord` updates the libp2p peer record (multiaddr, used for content routing). `updateDhtRecord` updates the discv5 node record (SPR, used for peer discovery). Both replace the current set of addresses, they do not accumulate. - -### Mapping restored callback - -`repeatPortMapping` runs in the background even in `Relay` state, this is how we detect when the mapping comes back. It needs a callback added so we can stop the relay and update addresses when it fires. The external IP may have changed after a router reboot, so we always update regardless. - -```nim -proc onMappingRestored(addrs: seq[MultiAddress]) = - if natState == Relay: - await autoRelayService.stop(switch) - natState = UPnP - # addrs may differ from previously announced ones (e.g. router reboot changed external IP) - discovery.updateAnnounceRecord(addrs) - discovery.updateDhtRecord(addrs) -``` - -### Hole punching - -When a peer connects through a relay, libp2p automatically tries to establish a direct connection using dcutr (a protocol for hole punching). If it works, that specific connection bypasses the relay: lower latency and less load on the relay server. This does not change the node's reachability: new peers still need to go through the relay first. - -## Step 5: Periodic Re-evaluation - -`AutonatService` re-runs every 5 minutes. On each cycle it asks `numPeersToAsk` peers to dial back. If confidence crosses `minConfidence`, `statusAndConfidenceHandler` fires and our handler updates DHT mode, relay state, and announced addresses. The `onMappingRestored` callback from `nat.nim` can also fire independently between cycles. - -## What needs to be implemented - -- DHT client/server mode in `discovery.nim`: the node currently starts as a DHT server unconditionally -- Move `nattedAddress()` call out of `start()` and into the AutoNAT `statusAndConfidenceHandler` -- Expose a `onRestored` callback in `nat.nim`'s `repeatPortMapping` -- For `hasExtIp: true` (manual port forwarding): add a background task that periodically asks a peer to dial the static IP directly. If it succeeds, fire the same `onMappingRestored` callback. Without this, a node in `Relay` state with a restored manual port forwarding has no way to recover automatically. -- `statusAndConfidenceHandler` with `natState` tracking -- chronos needs to be updated to expose IPv6 address stability flags to distinguish stable SLAAC addresses from temporary ones - -## Open questions - -- [ ] Mobile nodes: UDP is often blocked on cellular networks. discv5 is UDP-only. How do we support mobile participation? From f9ee0ac2abe6aea4026f77e892ea5c3b8f5e7822 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Wed, 15 Apr 2026 09:07:42 +0400 Subject: [PATCH 012/167] Remove local settings --- .claude/settings.local.json | 23 ----------------------- 1 file changed, 23 deletions(-) delete mode 100644 .claude/settings.local.json diff --git a/.claude/settings.local.json b/.claude/settings.local.json deleted file mode 100644 index 4494ff72..00000000 --- a/.claude/settings.local.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "permissions": { - "allow": [ - "Bash(grep -rn \"testnat\" /home/arnaud/Work/logos/logos-storage-nim/*.nims)", - "Bash(grep:*)", - "Bash(find:*)", - "Bash(curl -s http://127.0.0.1:8080/api/storage/v1/debug/info)", - "Bash(ls /home/arnaud/Work/logos/logos-storage-nim/tests/*.cfg /home/arnaud/Work/logos/logos-storage-nim/tests/*.nim)", - "Bash(nph --version)", - "Bash(make print-nph-path:*)", - "Bash(vendor/nimbus-build-system/vendor/Nim/bin/nph:*)", - "Bash(nimble build:*)", - "Bash(nim --version)", - "Bash(nimble --version)", - "Bash(/home/arnaud/.nimble/bin/nim --version)", - "Bash(make -n build-nph)", - "Bash(make build-nph:*)", - "Bash(/home/arnaud/.nimble/bin/nim c:*)", - "Bash(NIMFLAGS=\"--skipParentCfg\" nimble build)", - "Bash(NIMFLAGS=\"--skipParentCfg\" nimble build --verbose)" - ] - } -} From e1742a2452c917634fda01a996276b1b4a671c7d Mon Sep 17 00:00:00 2001 From: Arnaud Date: Wed, 15 Apr 2026 09:17:45 +0400 Subject: [PATCH 013/167] Add bootstrap nodes connect --- storage/storage.nim | 13 +++++++++++-- tests/integration/1_minute/testnat.nim | 19 ++++--------------- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/storage/storage.nim b/storage/storage.nim index 23b4e626..eaf11564 100644 --- a/storage/storage.nim +++ b/storage/storage.nim @@ -39,6 +39,7 @@ import ./namespaces import ./storagetypes import ./logutils import ./nat +import ./utils/natutils logScope: topics = "storage node" @@ -95,7 +96,7 @@ proc start*(s: StorageServer) {.async.} = .deduplicate() let discoveryAddrs = @[getMultiAddrWithIPAndUDPPort(announceIp.get, s.config.discoveryPort)] - s.storageNode.discovery.updateDhtRecord(discoveryAddrs) + s.storageNode.discovery.updateDhtRecord(announceAddrs & discoveryAddrs) s.storageNode.discovery.updateAnnounceRecord(announceAddrs) var hasPublicAddr = false @@ -110,6 +111,14 @@ proc start*(s: StorageServer) {.async.} = await s.storageNode.start() + for spr in s.config.bootstrapNodes: + try: + let addrs = spr.data.addresses.mapIt(it.address) + await s.storageNode.switch.connect(spr.data.peerId, addrs) + except CatchableError as e: + warn "Cannot connect to bootstrap node", error = e.msg + discard + if s.restServer != nil: s.restServer.start() @@ -351,7 +360,7 @@ proc new*( let (announceAddrs, discoveryAddrs) = nattedAddress(config.nat, switch.peerInfo.addrs, config.discoveryPort) discovery.updateAnnounceRecord(announceAddrs) - discovery.updateDhtRecord(discoveryAddrs) + discovery.updateDhtRecord(announceAddrs & discoveryAddrs) ) StorageServer( diff --git a/tests/integration/1_minute/testnat.nim b/tests/integration/1_minute/testnat.nim index d7b9288f..4aa13bce 100644 --- a/tests/integration/1_minute/testnat.nim +++ b/tests/integration/1_minute/testnat.nim @@ -16,8 +16,9 @@ multinodesuite "AutoNAT integration": .withNatMinConfidence(0.5) .withNatScheduleInterval(10.seconds) .withNatMaxQueueSize(1) - .withLogFile() - .withLogLevel("DEBUG").some + # .withLogFile() + # .withLogLevel("DEBUG") + .some ) # Reminder: multinodesuite setup the first node as bootstrap node @@ -25,20 +26,8 @@ multinodesuite "AutoNAT integration": let node1 = clients()[0] let node2 = clients()[1] - # Temporary - # DHT exposes only UDP information - # So we force temporary by connection the 2 node together - # to update the autonat reachability - let info = await node2.client.info() - - check not info.isErr - - await node1.client.connectPeer( - info.get()["id"].getStr(), info.get()["addrs"].getElems().mapIt(it.getStr()) - ) - check eventuallySafe( - (await node1.client.natReachability()).get() == "Reachable", + (await node2.client.natReachability()).get() == "Reachable", timeout = 30_000, pollInterval = 500, ) From 32ffd069a565b7ad62960ef29883e6d41ccbd882 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Thu, 4 Jun 2026 13:22:23 +0400 Subject: [PATCH 014/167] Move to autonatv2 --- storage/rest/api.nim | 6 +++--- storage/storage.nim | 27 ++++++++++++++++----------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/storage/rest/api.nim b/storage/rest/api.nim index af4b009e..6c54cc8b 100644 --- a/storage/rest/api.nim +++ b/storage/rest/api.nim @@ -23,7 +23,7 @@ import pkg/confutils import pkg/libp2p import pkg/libp2p/routing_record -import pkg/libp2p/protocols/connectivity/autonat/service +import pkg/libp2p/protocols/connectivity/autonatv2/service import pkg/codexdht/discv5/spr as spr import ../logutils @@ -561,7 +561,7 @@ proc initNodeApi(node: StorageNodeRef, conf: StorageConf, router: var RestRouter proc initDebugApi( node: StorageNodeRef, conf: StorageConf, - autonat: AutonatService, + autonat: AutonatV2Service, router: var RestRouter, ) = let allowedOrigin = router.allowedOrigin @@ -644,7 +644,7 @@ proc initRestApi*( node: StorageNodeRef, conf: StorageConf, repoStore: RepoStore, - autonat: AutonatService, + autonat: AutonatV2Service, corsAllowedOrigin: ?string, ): RestRouter = var router = RestRouter.init(validate, corsAllowedOrigin) diff --git a/storage/storage.nim b/storage/storage.nim index eaf11564..9ffb8205 100644 --- a/storage/storage.nim +++ b/storage/storage.nim @@ -17,7 +17,7 @@ import pkg/chronos import pkg/taskpools import pkg/presto import pkg/libp2p -import pkg/libp2p/protocols/connectivity/autonat/[service, client] +import pkg/libp2p/protocols/connectivity/autonatv2/[service, client] import pkg/confutils import pkg/confutils/defs import pkg/stew/io2 @@ -53,7 +53,7 @@ type repoStore: RepoStore maintenance: BlockMaintainer taskpool: Taskpool - autonatService*: AutonatService + autonatService*: AutonatV2Service isStarted: bool StoragePrivateKey* = libp2p.PrivateKey # alias @@ -195,14 +195,17 @@ proc new*( ## create StorageServer including setting up datastore, repostore, etc let listenMultiAddr = getMultiAddrWithIpAndTcpPort(config.listenIp, config.listenPort) - let autonatService = AutonatService.new( - autonatClient = AutonatClient.new(), + let autonatClient = AutonatV2Client.new(random.Rng.instance()) + let autonatService = AutonatV2Service.new( rng = random.Rng.instance(), - scheduleInterval = Opt.some(config.natScheduleInterval), - askNewConnectedPeers = true, - numPeersToAsk = config.natNumPeersToAsk, - maxQueueSize = config.natMaxQueueSize, - minConfidence = config.natMinConfidence, + client = autonatClient, + config = AutonatV2ServiceConfig.new( + scheduleInterval = Opt.some(config.natScheduleInterval), + askNewConnectedPeers = true, + numPeersToAsk = config.natNumPeersToAsk, + maxQueueSize = config.natMaxQueueSize, + minConfidence = config.natMinConfidence, + ), ) let switch = SwitchBuilder @@ -217,11 +220,13 @@ proc new*( .withAgentVersion(config.agentString) .withSignedPeerRecord(true) .withTcpTransport({ServerFlags.ReuseAddr, ServerFlags.TcpNoDelay}) - .withAutonat() + .withAutonatV2Server() .withServices(@[Service(autonatService)]) .build() var taskPool: Taskpool + autonatClient.setup(switch) + switch.mount(autonatClient) try: if config.numThreads == ThreadCount(0): @@ -352,7 +357,7 @@ proc new*( switch.mount(network) switch.mount(manifestProto) - autonatService.statusAndConfidenceHandler( + autonatService.setStatusAndConfidenceHandler( proc( networkReachability: NetworkReachability, confidence: Opt[float] ) {.async: (raises: [CancelledError]).} = From 7b8d526f4b3c710cfb851711afe79d8dfa1278cf Mon Sep 17 00:00:00 2001 From: Arnaud Date: Wed, 15 Apr 2026 10:25:36 +0400 Subject: [PATCH 015/167] Add abstraction for reachable nodes --- storage/nat.nim | 6 ++++++ storage/storage.nim | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/storage/nat.nim b/storage/nat.nim index f11d9786..0b83729e 100644 --- a/storage/nat.nim +++ b/storage/nat.nim @@ -401,6 +401,12 @@ proc setupAddress*( of NatStrategy.NatUpnp, NatStrategy.NatPmp: return setupNat(natConfig.nat, tcpPort, udpPort, clientId) +proc findReachableNodes*(bootstrapNodes: seq[SignedPeerRecord]): seq[SignedPeerRecord] = + ## Returns the list of nodes known to be directly reachable. + ## Currently returns bootstrap nodes. In the future, any network participant + ## confirmed reachable by AutoNAT could be included. + bootstrapNodes + proc nattedAddress*( natConfig: NatConfig, addrs: seq[MultiAddress], udpPort: Port ): tuple[libp2p, discovery: seq[MultiAddress]] = diff --git a/storage/storage.nim b/storage/storage.nim index 9ffb8205..906ae51c 100644 --- a/storage/storage.nim +++ b/storage/storage.nim @@ -111,7 +111,7 @@ proc start*(s: StorageServer) {.async.} = await s.storageNode.start() - for spr in s.config.bootstrapNodes: + for spr in findReachableNodes(s.config.bootstrapNodes): try: let addrs = spr.data.addresses.mapIt(it.address) await s.storageNode.switch.connect(spr.data.peerId, addrs) From ac01e7cc2adb0559bee966ca5d712098d61feb24 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Tue, 21 Apr 2026 17:17:25 +0400 Subject: [PATCH 016/167] Introduce first tests for the async machine --- storage/nat.nim | 40 +++++++++++++++++++++++++++++++++++++++ storage/storage.nim | 10 +++++----- tests/storage/testnat.nim | 27 ++++++++++++++++++++++++++ 3 files changed, 72 insertions(+), 5 deletions(-) diff --git a/storage/nat.nim b/storage/nat.nim index 0b83729e..ad06fba3 100644 --- a/storage/nat.nim +++ b/storage/nat.nim @@ -17,10 +17,12 @@ import import pkg/chronos import pkg/chronicles import pkg/libp2p +import pkg/libp2p/protocols/connectivity/autonatv2/service import ./utils import ./utils/natutils import ./utils/addrutils +import ./discovery const UPNP_TIMEOUT = 200 # ms @@ -61,6 +63,16 @@ type PrefSrcStatus = enum BindAddressIsPublic BindAddressIsPrivate +type NatMapper* = ref object of RootObj + +method mapNatAddresses*( + m: NatMapper, addrs: seq[MultiAddress], discoveryPort: Port +): tuple[libp2p, discovery: seq[MultiAddress]] {.base, gcsafe, raises: [].} = + raiseAssert "mapNatAddresses not implemented" + +type DefaultNatMapper* = ref object of NatMapper + natConfig*: NatConfig + ## Also does threadvar initialisation. ## Must be called before redirectPorts() in each thread. proc getExternalIP*(natStrategy: NatStrategy, quiet = false): Option[IpAddress] = @@ -437,3 +449,31 @@ proc nattedAddress*( # Invalid multiaddress format - return as is it (newAddrs, discoveryAddrs) + +method mapNatAddresses*( + m: DefaultNatMapper, addrs: seq[MultiAddress], discoveryPort: Port +): tuple[libp2p, discovery: seq[MultiAddress]] {.gcsafe, raises: [].} = + nattedAddress(m.natConfig, addrs, discoveryPort) + +proc handleNatStatus*( + networkReachability: NetworkReachability, + confidence: Opt[float], + mapper: NatMapper, + listenAddrs: seq[MultiAddress], + discoveryPort: Port, + discovery: Discovery, +) {.async: (raises: [CancelledError]).} = + debug "AutoNAT status", reachability = networkReachability, confidence + + case networkReachability + of Reachable: + # TODO: switch DHT to server mode, stop relay if running + discard + of NotReachable: + let (announceAddrs, discoveryAddrs) = + mapper.mapNatAddresses(listenAddrs, discoveryPort) + discovery.updateAnnounceRecord(announceAddrs) + discovery.updateDhtRecord(announceAddrs & discoveryAddrs) + of Unknown: + # Nothing to do here, not enough confidence score result + discard diff --git a/storage/storage.nim b/storage/storage.nim index 906ae51c..a480f832 100644 --- a/storage/storage.nim +++ b/storage/storage.nim @@ -357,15 +357,15 @@ proc new*( switch.mount(network) switch.mount(manifestProto) + let natMapper = DefaultNatMapper(natConfig: config.nat) autonatService.setStatusAndConfidenceHandler( proc( networkReachability: NetworkReachability, confidence: Opt[float] ) {.async: (raises: [CancelledError]).} = - if networkReachability == NotReachable: - let (announceAddrs, discoveryAddrs) = - nattedAddress(config.nat, switch.peerInfo.addrs, config.discoveryPort) - discovery.updateAnnounceRecord(announceAddrs) - discovery.updateDhtRecord(announceAddrs & discoveryAddrs) + await handleNatStatus( + networkReachability, confidence, natMapper, switch.peerInfo.addrs, + config.discoveryPort, discovery, + ) ) StorageServer( diff --git a/tests/storage/testnat.nim b/tests/storage/testnat.nim index 71155540..0acb0725 100644 --- a/tests/storage/testnat.nim +++ b/tests/storage/testnat.nim @@ -1,12 +1,24 @@ import std/[unittest, net] import pkg/chronos +import pkg/libp2p import pkg/libp2p/[multiaddress, multihash, multicodec] +import pkg/libp2p/protocols/connectivity/autonatv2/service import pkg/results import ../../storage/nat +import ../../storage/discovery +import ../../storage/rng import ../../storage/utils import ../../storage/utils/natutils +type MockNatMapper = ref object of NatMapper + mapped: tuple[libp2p, discovery: seq[MultiAddress]] + +method mapNatAddresses*( + m: MockNatMapper, addrs: seq[MultiAddress], discoveryPort: Port +): tuple[libp2p, discovery: seq[MultiAddress]] {.raises: [].} = + m.mapped + suite "NAT Address Tests": test "nattedAddress with local addresses": # Setup test data @@ -65,3 +77,18 @@ suite "setupAddress": check ip == none(IpAddress) check tcpPort == some(Port(5000)) check udpPort == some(Port(5001)) + +suite "handleNatStatus": + let key = PrivateKey.random(Rng.instance[]).get() + + test "NotReachable updates announce addresses": + let disc = Discovery.new(key, announceAddrs = @[]) + let announceAddr = MultiAddress.init("/ip4/1.2.3.4/tcp/8080").expect("valid") + let discAddr = MultiAddress.init("/ip4/1.2.3.4/udp/8090").expect("valid") + let mapper = MockNatMapper(mapped: (@[announceAddr], @[discAddr])) + + waitFor handleNatStatus( + NotReachable, Opt.none(float), mapper, @[], Port(8090), disc + ) + + check disc.announceAddrs == @[announceAddr] From e1aa7958c1cd56279e783e2f4bc4e3c68496f0fa Mon Sep 17 00:00:00 2001 From: Arnaud Date: Wed, 22 Apr 2026 12:19:26 +0400 Subject: [PATCH 017/167] Introduce max relay config --- storage/conf.nim | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/storage/conf.nim b/storage/conf.nim index 300d7c0a..7499fa5e 100644 --- a/storage/conf.nim +++ b/storage/conf.nim @@ -336,6 +336,12 @@ type name: "nat-min-confidence" .}: float + natMaxRelays* {. + desc: "Maximum number of relay servers to reserve slots on simultaneously", + defaultValue: 2, + name: "nat-max-relays" + .}: int + func defaultAddress*(conf: StorageConf): IpAddress = result = static parseIpAddress("127.0.0.1") From 50c708ade1fb0da3d80ab5b1cbc3db6c8c0cd706 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Wed, 22 Apr 2026 12:21:43 +0400 Subject: [PATCH 018/167] Add relay integration and tests --- storage/nat.nim | 80 +++++++++++++++++++++------ storage/storage.nim | 24 +++++++-- tests/storage/testnat.nim | 110 +++++++++++++++++++++++++++++++++----- 3 files changed, 183 insertions(+), 31 deletions(-) diff --git a/storage/nat.nim b/storage/nat.nim index ad06fba3..eccb193a 100644 --- a/storage/nat.nim +++ b/storage/nat.nim @@ -18,6 +18,7 @@ import pkg/chronos import pkg/chronicles import pkg/libp2p import pkg/libp2p/protocols/connectivity/autonatv2/service +import pkg/libp2p/services/autorelayservice import ./utils import ./utils/natutils @@ -66,12 +67,18 @@ type PrefSrcStatus = enum type NatMapper* = ref object of RootObj method mapNatAddresses*( - m: NatMapper, addrs: seq[MultiAddress], discoveryPort: Port + m: NatMapper, addrs: seq[MultiAddress] ): tuple[libp2p, discovery: seq[MultiAddress]] {.base, gcsafe, raises: [].} = raiseAssert "mapNatAddresses not implemented" +method getReachableAddresses*( + m: NatMapper, addrs: seq[MultiAddress] +): tuple[libp2p, discovery: seq[MultiAddress]] {.base, gcsafe, raises: [].} = + raiseAssert "getReachableAddresses not implemented" + type DefaultNatMapper* = ref object of NatMapper natConfig*: NatConfig + discoveryPort*: Port ## Also does threadvar initialisation. ## Must be called before redirectPorts() in each thread. @@ -451,29 +458,70 @@ proc nattedAddress*( (newAddrs, discoveryAddrs) method mapNatAddresses*( - m: DefaultNatMapper, addrs: seq[MultiAddress], discoveryPort: Port + m: DefaultNatMapper, addrs: seq[MultiAddress] ): tuple[libp2p, discovery: seq[MultiAddress]] {.gcsafe, raises: [].} = - nattedAddress(m.natConfig, addrs, discoveryPort) + nattedAddress(m.natConfig, addrs, m.discoveryPort) + +method getReachableAddresses*( + m: DefaultNatMapper, addrs: seq[MultiAddress] +): tuple[libp2p, discovery: seq[MultiAddress]] {.gcsafe, raises: [].} = + let ip = + if m.natConfig.hasExtIp: + some(m.natConfig.extIp) + else: + let (routeIp, _) = getRoutePrefSrc(static parseIpAddress("0.0.0.0")) + routeIp + if ip.isNone: + return (@[], @[]) + let announceAddrs = + addrs.mapIt(it.remapAddr(ip = ip, port = none(Port))).deduplicate() + (announceAddrs, @[getMultiAddrWithIPAndUDPPort(ip.get, m.discoveryPort)]) + +proc hasPublicIp*(addrs: seq[MultiAddress]): bool = + for addr in addrs: + let (ip, _) = getAddressAndPort(addr) + if ip.isSome and isGlobalUnicast(ip.get): + return true proc handleNatStatus*( networkReachability: NetworkReachability, - confidence: Opt[float], mapper: NatMapper, - listenAddrs: seq[MultiAddress], - discoveryPort: Port, discovery: Discovery, + switch: Switch, + autoRelayService: AutoRelayService, ) {.async: (raises: [CancelledError]).} = - debug "AutoNAT status", reachability = networkReachability, confidence - case networkReachability - of Reachable: - # TODO: switch DHT to server mode, stop relay if running - discard - of NotReachable: - let (announceAddrs, discoveryAddrs) = - mapper.mapNatAddresses(listenAddrs, discoveryPort) - discovery.updateAnnounceRecord(announceAddrs) - discovery.updateDhtRecord(announceAddrs & discoveryAddrs) of Unknown: # Nothing to do here, not enough confidence score result discard + of Reachable: + # For UPnP, it the mapping was a success, + # the autorelay service has been stopped + # and the address was already announced + if autoRelayService.isRunning: + if not await autoRelayService.stop(switch): + debug "AutoRelayService stop method returned false" + + let (announceAddrs, discoveryAddrs) = + mapper.getReachableAddresses(switch.peerInfo.addrs) + discovery.updateAnnounceRecord(announceAddrs) + discovery.updateDhtRecord(announceAddrs & discoveryAddrs) + # TODO: switch DHT to server mode + of NotReachable: + let (announceAddrs, discoveryAddrs) = mapper.mapNatAddresses(switch.peerInfo.addrs) + + # With a UPnP / NatPmP successful mapping, + # we suppose that having a public IP make it Reachable. + # If not, the state will be updated in the next Autonat iteration. + # TODO: Do we need to manually call dialMe to make sure we are Reachable ? + if hasPublicIp(announceAddrs): + discovery.updateAnnounceRecord(announceAddrs) + discovery.updateDhtRecord(announceAddrs & discoveryAddrs) + + if autoRelayService.isRunning: + if not await autoRelayService.stop(switch): + debug "AutoRelayService stop method returned false" + else: + if not autoRelayService.isRunning: + if not await autoRelayService.setup(switch): + debug "AutoRelayService setup method returned false" diff --git a/storage/storage.nim b/storage/storage.nim index a480f832..d717ac1a 100644 --- a/storage/storage.nim +++ b/storage/storage.nim @@ -18,6 +18,8 @@ import pkg/taskpools import pkg/presto import pkg/libp2p import pkg/libp2p/protocols/connectivity/autonatv2/[service, client] +import pkg/libp2p/protocols/connectivity/relay/client as relayClientModule +import pkg/libp2p/services/autorelayservice import pkg/confutils import pkg/confutils/defs import pkg/stew/io2 @@ -54,6 +56,7 @@ type maintenance: BlockMaintainer taskpool: Taskpool autonatService*: AutonatV2Service + autoRelayService: AutoRelayService isStarted: bool StoragePrivateKey* = libp2p.PrivateKey # alias @@ -195,6 +198,8 @@ proc new*( ## create StorageServer including setting up datastore, repostore, etc let listenMultiAddr = getMultiAddrWithIpAndTcpPort(config.listenIp, config.listenPort) + let relayClient = relayClientModule.RelayClient.new() + let autonatClient = AutonatV2Client.new(random.Rng.instance()) let autonatService = AutonatV2Service.new( rng = random.Rng.instance(), @@ -221,6 +226,7 @@ proc new*( .withSignedPeerRecord(true) .withTcpTransport({ServerFlags.ReuseAddr, ServerFlags.TcpNoDelay}) .withAutonatV2Server() + .withCircuitRelay(relayClient) .withServices(@[Service(autonatService)]) .build() @@ -340,6 +346,16 @@ proc new*( taskPool = taskPool, ) + autoRelayService = AutoRelayService.new( + maxNumRelays = config.natMaxRelays, + client = relayClient, + onReservation = proc(addresses: seq[MultiAddress]) {.gcsafe, raises: [].} = + debug "Relay reservation updated", addresses + discovery.updateAnnounceRecord(addresses) + discovery.updateDhtRecord(addresses), + rng = random.Rng.instance(), + ) + var restServer: RestServerRef = nil if config.apiBindAddress.isSome: @@ -357,14 +373,15 @@ proc new*( switch.mount(network) switch.mount(manifestProto) - let natMapper = DefaultNatMapper(natConfig: config.nat) + let natMapper = + DefaultNatMapper(natConfig: config.nat, discoveryPort: config.discoveryPort) autonatService.setStatusAndConfidenceHandler( proc( networkReachability: NetworkReachability, confidence: Opt[float] ) {.async: (raises: [CancelledError]).} = + debug "AutoNAT status", reachability = networkReachability, confidence await handleNatStatus( - networkReachability, confidence, natMapper, switch.peerInfo.addrs, - config.discoveryPort, discovery, + networkReachability, natMapper, discovery, switch, autoRelayService ) ) @@ -377,4 +394,5 @@ proc new*( taskPool: taskPool, logFile: logFile, autonatService: autonatService, + autoRelayService: autoRelayService, ) diff --git a/tests/storage/testnat.nim b/tests/storage/testnat.nim index 0acb0725..3f772ee2 100644 --- a/tests/storage/testnat.nim +++ b/tests/storage/testnat.nim @@ -1,10 +1,15 @@ -import std/[unittest, net] +import std/net import pkg/chronos -import pkg/libp2p import pkg/libp2p/[multiaddress, multihash, multicodec] -import pkg/libp2p/protocols/connectivity/autonatv2/service +import pkg/libp2p/protocols/connectivity/autonatv2/service except setup +import pkg/libp2p/protocols/connectivity/autonatv2/types +import pkg/libp2p/protocols/connectivity/relay/client as relayClientModule +import pkg/libp2p/services/autorelayservice except setup + import pkg/results +import ./helpers +import ../asynctest import ../../storage/nat import ../../storage/discovery import ../../storage/rng @@ -15,7 +20,12 @@ type MockNatMapper = ref object of NatMapper mapped: tuple[libp2p, discovery: seq[MultiAddress]] method mapNatAddresses*( - m: MockNatMapper, addrs: seq[MultiAddress], discoveryPort: Port + m: MockNatMapper, addrs: seq[MultiAddress] +): tuple[libp2p, discovery: seq[MultiAddress]] {.raises: [].} = + m.mapped + +method getReachableAddresses*( + m: MockNatMapper, addrs: seq[MultiAddress] ): tuple[libp2p, discovery: seq[MultiAddress]] {.raises: [].} = m.mapped @@ -44,7 +54,6 @@ suite "NAT Address Tests": MultiAddress.init("/ip4/8.8.8.8/tcp/5000").expect("valid multiaddr"), MultiAddress.init("/ip4/8.8.8.8/tcp/5000").expect("valid multiaddr"), ] - #ipv6Addr = MultiAddress.init("/ip6/::1/tcp/5000").expect("valid multiaddr") addrs = @[localAddr, anyAddr, publicAddr] @@ -78,17 +87,94 @@ suite "setupAddress": check tcpPort == some(Port(5000)) check udpPort == some(Port(5001)) -suite "handleNatStatus": - let key = PrivateKey.random(Rng.instance[]).get() +suite "getReachableAddresses": + test "returns remapped addresses when extIp is configured": + let + natConfig = NatConfig(hasExtIp: true, extIp: parseIpAddress("1.2.3.4")) + mapper = DefaultNatMapper(natConfig: natConfig, discoveryPort: Port(8090)) + listenAddr = MultiAddress.init("/ip4/0.0.0.0/tcp/5000").expect("valid") - test "NotReachable updates announce addresses": - let disc = Discovery.new(key, announceAddrs = @[]) + let (libp2pAddrs, discAddrs) = mapper.getReachableAddresses(@[listenAddr]) + + check libp2pAddrs == @[MultiAddress.init("/ip4/1.2.3.4/tcp/5000").expect("valid")] + check discAddrs == @[MultiAddress.init("/ip4/1.2.3.4/udp/8090").expect("valid")] + +suite "hasPublicIp": + test "hasPublicIp returns true when the address is public": + let ma = MultiAddress.init("/ip4/8.8.8.8/tcp/8080").expect("valid") + check hasPublicIp(@[ma]) + + test "hasPublicIp returns false when the address is private": + let ma = MultiAddress.init("/ip4/192.168.1.1/tcp/8080").expect("valid") + check not hasPublicIp(@[ma]) + + test "hasPublicIp returns false when the address is empty": + check not hasPublicIp(@[]) + +asyncchecksuite "handleNatStatus": + var sw: Switch + var key: PrivateKey + var disc: Discovery + let autoRelay = + AutoRelayService.new(1, relayClientModule.RelayClient.new(), nil, Rng.instance()) + + setup: + key = PrivateKey.random(Rng.instance[]).get() + disc = Discovery.new(key, announceAddrs = @[]) + sw = newStandardSwitch() + await sw.start() + + teardown: + await sw.stop() + + if autoRelay.isRunning: + discard await autoRelay.stop(sw) + + test "handleNatStatus announces address when the node is not Reachable and the UPnP succeed with public ip": let announceAddr = MultiAddress.init("/ip4/1.2.3.4/tcp/8080").expect("valid") let discAddr = MultiAddress.init("/ip4/1.2.3.4/udp/8090").expect("valid") let mapper = MockNatMapper(mapped: (@[announceAddr], @[discAddr])) - waitFor handleNatStatus( - NotReachable, Opt.none(float), mapper, @[], Port(8090), disc - ) + await handleNatStatus(NotReachable, mapper, disc, sw, autoRelay) check disc.announceAddrs == @[announceAddr] + check not autoRelay.isRunning + + # test "handleNatStatus does not announce address when the node is not Reachable and the UPnP succeed with private ip": + # let privateAddr = MultiAddress.init("/ip4/192.168.1.1/tcp/8080").expect("valid") + # let mapper = MockNatMapper(mapped: (@[privateAddr], @[])) + + # await handleNatStatus( + # NotReachable, mapper, disc, sw, autoRelay + # ) + + # check disc.announceAddrs == @[] + # check not autoRelay.isRunning + + test "handleNatStatus starts autoRelay when node is not Reachable and UPnP failed": + let mapper = MockNatMapper(mapped: (@[], @[])) + + await handleNatStatus(NotReachable, mapper, disc, sw, autoRelay) + + check autoRelay.isRunning + # The addresses will be announced in the onReservation callback + # after a node accepted a Relay reservation. + + test "handleNatStatus does not announce address when node is Reachable and relay is not running": + let mapper = MockNatMapper(mapped: (@[], @[])) + + await handleNatStatus(Reachable, mapper, disc, sw, autoRelay) + + check disc.announceAddrs == newSeq[MultiAddress]() + check not autoRelay.isRunning + + test "handleNatStatus stops relay and announces address when node is Reachable and relay is running": + let announceAddr = MultiAddress.init("/ip4/1.2.3.4/tcp/8080").expect("valid") + let discAddr = MultiAddress.init("/ip4/1.2.3.4/udp/8090").expect("valid") + let mapper = MockNatMapper(mapped: (@[announceAddr], @[discAddr])) + + discard await autorelayservice.setup(autoRelay, sw) + await handleNatStatus(Reachable, mapper, disc, sw, autoRelay) + + check not autoRelay.isRunning + check disc.announceAddrs == @[announceAddr] From 7838baa27d9078d85d162702120aeb5abef05286 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Wed, 22 Apr 2026 19:53:39 +0400 Subject: [PATCH 019/167] Add relay config --- storage/conf.nim | 6 ++++++ storage/storage.nim | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/storage/conf.nim b/storage/conf.nim index 7499fa5e..c21b72b1 100644 --- a/storage/conf.nim +++ b/storage/conf.nim @@ -342,6 +342,12 @@ type name: "nat-max-relays" .}: int + relay* {. + desc: "Enable circuit relay server (hop) - use on publicly reachable nodes only", + defaultValue: false, + name: "relay" + .}: bool + func defaultAddress*(conf: StorageConf): IpAddress = result = static parseIpAddress("127.0.0.1") diff --git a/storage/storage.nim b/storage/storage.nim index d717ac1a..702522f7 100644 --- a/storage/storage.nim +++ b/storage/storage.nim @@ -198,7 +198,7 @@ proc new*( ## create StorageServer including setting up datastore, repostore, etc let listenMultiAddr = getMultiAddrWithIpAndTcpPort(config.listenIp, config.listenPort) - let relayClient = relayClientModule.RelayClient.new() + let relayClient = relayClientModule.RelayClient.new(canHop = config.relay) let autonatClient = AutonatV2Client.new(random.Rng.instance()) let autonatService = AutonatV2Service.new( From 006df11163859cfa3be7340df4a1c8ec3fbdd8d6 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Tue, 5 May 2026 10:48:55 +0400 Subject: [PATCH 020/167] Remove nat none stategy --- storage/conf.nim | 4 +--- storage/nat.nim | 14 +------------- storage/utils/natutils.nim | 1 - tests/integration/multinodes.nim | 1 - tests/storage/helpers/nodeutils.nim | 2 +- tests/storage/testnat.nim | 23 ----------------------- 6 files changed, 3 insertions(+), 42 deletions(-) diff --git a/storage/conf.nim b/storage/conf.nim index c21b72b1..4aa178b7 100644 --- a/storage/conf.nim +++ b/storage/conf.nim @@ -158,7 +158,7 @@ type nat* {. desc: "Specify method to use for determining public address. " & - "Must be one of: any, none, upnp, pmp, extip:. " & + "Must be one of: any, upnp, pmp, extip:. " & "If connecting to peers on a local network only, use 'none'.", defaultValue: defaultNatConfig(), defaultValueDesc: "any", @@ -420,8 +420,6 @@ func parse*(T: type NatConfig, p: string): Result[NatConfig, string] = case p.toLowerAscii of "any": return ok(NatConfig(hasExtIp: false, nat: NatStrategy.NatAny)) - of "none": - return ok(NatConfig(hasExtIp: false, nat: NatStrategy.NatNone)) of "upnp": return ok(NatConfig(hasExtIp: false, nat: NatStrategy.NatUpnp)) of "pmp": diff --git a/storage/nat.nim b/storage/nat.nim index eccb193a..84ecef1d 100644 --- a/storage/nat.nim +++ b/storage/nat.nim @@ -48,7 +48,7 @@ type NatConfig* = object var upnp {.threadvar.}: Miniupnp npmp {.threadvar.}: NatPmp - strategy = NatStrategy.NatNone + strategy = NatStrategy.NatAny natClosed: Atomic[bool] extIp: Option[IpAddress] activeMappings: seq[PortMappings] @@ -405,18 +405,6 @@ proc setupAddress*( return (prefSrcIp, some(tcpPort), some(udpPort)) of PrefSrcIsPrivate, BindAddressIsPrivate: return setupNat(natConfig.nat, tcpPort, udpPort, clientId) - of NatStrategy.NatNone: - let (prefSrcIp, prefSrcStatus) = getRoutePrefSrc(bindIp) - - case prefSrcStatus - of NoRoutingInfo, PrefSrcIsPublic, BindAddressIsPublic: - return (prefSrcIp, some(tcpPort), some(udpPort)) - of PrefSrcIsPrivate: - error "No public IP address found. Should not use --nat:none option" - return (none(IpAddress), some(tcpPort), some(udpPort)) - of BindAddressIsPrivate: - error "Bind IP is not a public IP address. Should not use --nat:none option" - return (none(IpAddress), some(tcpPort), some(udpPort)) of NatStrategy.NatUpnp, NatStrategy.NatPmp: return setupNat(natConfig.nat, tcpPort, udpPort, clientId) diff --git a/storage/utils/natutils.nim b/storage/utils/natutils.nim index 04447d26..2a129793 100644 --- a/storage/utils/natutils.nim +++ b/storage/utils/natutils.nim @@ -8,7 +8,6 @@ type NatStrategy* = enum NatAny NatUpnp NatPmp - NatNone func isGlobalUnicast*(address: TransportAddress): bool = if address.isGlobal() and address.isUnicast(): true else: false diff --git a/tests/integration/multinodes.nim b/tests/integration/multinodes.nim index 9d4153bd..034ee657 100644 --- a/tests/integration/multinodes.nim +++ b/tests/integration/multinodes.nim @@ -131,7 +131,6 @@ template multinodesuite*(suiteName: string, body: untyped) = config.addCliOption("--bootstrap-node", bootstrapNode) config.addCliOption("--data-dir", datadir) - config.addCliOption("--nat", "none") except StorageConfigError as e: raiseMultiNodeSuiteError "invalid cli option, error: " & e.msg diff --git a/tests/storage/helpers/nodeutils.nim b/tests/storage/helpers/nodeutils.nim index fdaf6162..2d9243fd 100644 --- a/tests/storage/helpers/nodeutils.nim +++ b/tests/storage/helpers/nodeutils.nim @@ -227,7 +227,7 @@ proc generateNodes*( if config.enableBootstrap: waitFor switch.peerInfo.update() let (announceAddrs, discoveryAddrs) = nattedAddress( - NatConfig(hasExtIp: false, nat: NatNone), + nat.NatConfig(hasExtIp: false, nat: NatAny), switch.peerInfo.addrs, bindPort.Port, ) diff --git a/tests/storage/testnat.nim b/tests/storage/testnat.nim index 3f772ee2..ee9ed27e 100644 --- a/tests/storage/testnat.nim +++ b/tests/storage/testnat.nim @@ -64,29 +64,6 @@ suite "NAT Address Tests": check(discoveryAddrs == expectedDiscoveryAddrs) check(libp2pAddrs == expectedlibp2pAddrs) -suite "setupAddress": - test "public bind IP with NatNone returns bind IP": - let - bindIp = parseIpAddress("8.8.8.8") - natConfig = NatConfig(hasExtIp: false, nat: NatStrategy.NatNone) - (ip, tcpPort, udpPort) = - setupAddress(natConfig, bindIp, Port(5000), Port(5001), "test") - - check ip == some(bindIp) - check tcpPort == some(Port(5000)) - check udpPort == some(Port(5001)) - - test "private bind IP with NatNone returns no IP": - let - bindIp = parseIpAddress("192.168.1.1") - natConfig = NatConfig(hasExtIp: false, nat: NatStrategy.NatNone) - (ip, tcpPort, udpPort) = - setupAddress(natConfig, bindIp, Port(5000), Port(5001), "test") - - check ip == none(IpAddress) - check tcpPort == some(Port(5000)) - check udpPort == some(Port(5001)) - suite "getReachableAddresses": test "returns remapped addresses when extIp is configured": let From 49fb1daf6d02dc0908c7de5313f24b0126d2a678 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Tue, 5 May 2026 10:55:11 +0400 Subject: [PATCH 021/167] Rename nat any to nat auto --- storage/conf.nim | 11 +++++------ storage/nat.nim | 8 ++++---- storage/utils/natutils.nim | 2 +- tests/storage/helpers/nodeutils.nim | 2 +- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/storage/conf.nim b/storage/conf.nim index 4aa178b7..abc5a219 100644 --- a/storage/conf.nim +++ b/storage/conf.nim @@ -158,10 +158,9 @@ type nat* {. desc: "Specify method to use for determining public address. " & - "Must be one of: any, upnp, pmp, extip:. " & - "If connecting to peers on a local network only, use 'none'.", + "Must be one of: auto, upnp, pmp, extip:.", defaultValue: defaultNatConfig(), - defaultValueDesc: "any", + defaultValueDesc: "auto", name: "nat" .}: NatConfig @@ -352,7 +351,7 @@ func defaultAddress*(conf: StorageConf): IpAddress = result = static parseIpAddress("127.0.0.1") func defaultNatConfig*(): NatConfig = - result = NatConfig(hasExtIp: false, nat: NatStrategy.NatAny) + result = NatConfig(hasExtIp: false, nat: NatStrategy.NatAuto) proc getStorageVersion(): string = let tag = strip(staticExec("git describe --tags --abbrev=0")) @@ -418,8 +417,8 @@ proc parseCmdArg*(T: type SignedPeerRecord, uri: string): T = func parse*(T: type NatConfig, p: string): Result[NatConfig, string] = case p.toLowerAscii - of "any": - return ok(NatConfig(hasExtIp: false, nat: NatStrategy.NatAny)) + of "auto": + return ok(NatConfig(hasExtIp: false, nat: NatStrategy.NatAuto)) of "upnp": return ok(NatConfig(hasExtIp: false, nat: NatStrategy.NatUpnp)) of "pmp": diff --git a/storage/nat.nim b/storage/nat.nim index 84ecef1d..ee9ce622 100644 --- a/storage/nat.nim +++ b/storage/nat.nim @@ -48,7 +48,7 @@ type NatConfig* = object var upnp {.threadvar.}: Miniupnp npmp {.threadvar.}: NatPmp - strategy = NatStrategy.NatAny + strategy = NatStrategy.NatAuto natClosed: Atomic[bool] extIp: Option[IpAddress] activeMappings: seq[PortMappings] @@ -85,7 +85,7 @@ type DefaultNatMapper* = ref object of NatMapper proc getExternalIP*(natStrategy: NatStrategy, quiet = false): Option[IpAddress] = var externalIP: IpAddress - if natStrategy == NatStrategy.NatAny or natStrategy == NatStrategy.NatUpnp: + if natStrategy == NatStrategy.NatAuto or natStrategy == NatStrategy.NatUpnp: if upnp == nil: upnp = newMiniupnp() @@ -127,7 +127,7 @@ proc getExternalIP*(natStrategy: NatStrategy, quiet = false): Option[IpAddress] error "parseIpAddress() exception", err = e.msg return - if natStrategy == NatStrategy.NatAny or natStrategy == NatStrategy.NatPmp: + if natStrategy == NatStrategy.NatAuto or natStrategy == NatStrategy.NatPmp: if npmp == nil: npmp = newNatPmp() let nres = npmp.init() @@ -397,7 +397,7 @@ proc setupAddress*( return (some(natConfig.extIp), some(tcpPort), some(udpPort)) case natConfig.nat - of NatStrategy.NatAny: + of NatStrategy.NatAuto: let (prefSrcIp, prefSrcStatus) = getRoutePrefSrc(bindIp) case prefSrcStatus diff --git a/storage/utils/natutils.nim b/storage/utils/natutils.nim index 2a129793..04cc57f9 100644 --- a/storage/utils/natutils.nim +++ b/storage/utils/natutils.nim @@ -5,7 +5,7 @@ import std/[net, tables, hashes, options], pkg/results, chronos, chronicles import pkg/libp2p type NatStrategy* = enum - NatAny + NatAuto NatUpnp NatPmp diff --git a/tests/storage/helpers/nodeutils.nim b/tests/storage/helpers/nodeutils.nim index 2d9243fd..11aa246e 100644 --- a/tests/storage/helpers/nodeutils.nim +++ b/tests/storage/helpers/nodeutils.nim @@ -227,7 +227,7 @@ proc generateNodes*( if config.enableBootstrap: waitFor switch.peerInfo.update() let (announceAddrs, discoveryAddrs) = nattedAddress( - nat.NatConfig(hasExtIp: false, nat: NatAny), + nat.NatConfig(hasExtIp: false, nat: NatAuto), switch.peerInfo.addrs, bindPort.Port, ) From da91a720b8dc8a8d5487255613768832926c76bd Mon Sep 17 00:00:00 2001 From: Arnaud Date: Tue, 5 May 2026 14:58:26 +0400 Subject: [PATCH 022/167] use AutoNat dialback IP instead of router public IP --- storage/nat.nim | 281 +++++++--------------------- storage/storage.nim | 10 +- storage/utils/addrutils.nim | 9 +- tests/storage/helpers/nodeutils.nim | 9 +- tests/storage/testnat.nim | 132 +++++-------- 5 files changed, 133 insertions(+), 308 deletions(-) diff --git a/storage/nat.nim b/storage/nat.nim index ee9ce622..863b1e68 100644 --- a/storage/nat.nim +++ b/storage/nat.nim @@ -9,15 +9,14 @@ {.push raises: [].} import - std/[options, os, times, net, atomics, exitprocs], + std/[options, os, times, atomics, exitprocs], nat_traversal/[miniupnpc, natpmp], - json_serialization/std/net, results import pkg/chronos import pkg/chronicles import pkg/libp2p -import pkg/libp2p/protocols/connectivity/autonatv2/service +import pkg/libp2p/protocols/connectivity/autonat/types import pkg/libp2p/services/autorelayservice import ./utils @@ -50,41 +49,25 @@ var npmp {.threadvar.}: NatPmp strategy = NatStrategy.NatAuto natClosed: Atomic[bool] - extIp: Option[IpAddress] activeMappings: seq[PortMappings] natThreads: seq[Thread[PortMappingArgs]] = @[] logScope: topics = "nat" -type PrefSrcStatus = enum - NoRoutingInfo - PrefSrcIsPublic - PrefSrcIsPrivate - BindAddressIsPublic - BindAddressIsPrivate - type NatMapper* = ref object of RootObj -method mapNatAddresses*( - m: NatMapper, addrs: seq[MultiAddress] -): tuple[libp2p, discovery: seq[MultiAddress]] {.base, gcsafe, raises: [].} = - raiseAssert "mapNatAddresses not implemented" - -method getReachableAddresses*( - m: NatMapper, addrs: seq[MultiAddress] -): tuple[libp2p, discovery: seq[MultiAddress]] {.base, gcsafe, raises: [].} = - raiseAssert "getReachableAddresses not implemented" +method mapNatPorts*(m: NatMapper): Option[(Port, Port)] {.base, gcsafe, raises: [].} = + raiseAssert "mapNatPorts not implemented" type DefaultNatMapper* = ref object of NatMapper natConfig*: NatConfig + tcpPort*: Port discoveryPort*: Port -## Also does threadvar initialisation. +## Initialises the UPnP or NAT-PMP threadvar and sets the `strategy` threadvar. ## Must be called before redirectPorts() in each thread. -proc getExternalIP*(natStrategy: NatStrategy, quiet = false): Option[IpAddress] = - var externalIP: IpAddress - +proc initNatDevice(natStrategy: NatStrategy, quiet = false): bool = if natStrategy == NatStrategy.NatAuto or natStrategy == NatStrategy.NatUpnp: if upnp == nil: upnp = newMiniupnp() @@ -114,18 +97,8 @@ proc getExternalIP*(natStrategy: NatStrategy, quiet = false): Option[IpAddress] if not quiet: debug "UPnP", msg if canContinue: - let ires = upnp.externalIPAddress() - if ires.isErr: - debug "UPnP", msg = ires.error - else: - # if we got this far, UPnP is working and we don't need to try NAT-PMP - try: - externalIP = parseIpAddress(ires.value) - strategy = NatStrategy.NatUpnp - return some(externalIP) - except ValueError as e: - error "parseIpAddress() exception", err = e.msg - return + strategy = NatStrategy.NatUpnp + return true if natStrategy == NatStrategy.NatAuto or natStrategy == NatStrategy.NatPmp: if npmp == nil: @@ -134,61 +107,10 @@ proc getExternalIP*(natStrategy: NatStrategy, quiet = false): Option[IpAddress] if nres.isErr: debug "NAT-PMP", msg = nres.error else: - let nires = npmp.externalIPAddress() - if nires.isErr: - debug "NAT-PMP", msg = nires.error - else: - try: - externalIP = parseIpAddress($(nires.value)) - strategy = NatStrategy.NatPmp - return some(externalIP) - except ValueError as e: - error "parseIpAddress() exception", err = e.msg - return + strategy = NatStrategy.NatPmp + return true -# This queries the routing table to get the "preferred source" attribute and -# checks if it's a public IP. If so, then it's our public IP. -# -# Further more, we check if the bind address (user provided, or a "0.0.0.0" -# default) is a public IP. That's a long shot, because code paths involving a -# user-provided bind address are not supposed to get here. -proc getRoutePrefSrc(bindIp: IpAddress): (Option[IpAddress], PrefSrcStatus) = - let bindAddress = initTAddress(bindIp, Port(0)) - - if bindAddress.isAnyLocal(): - let ip = - if bindIp.family == IpAddressFamily.IPv6: - getRouteIpv6() - else: - getRouteIpv4() - - if ip.isErr(): - # No route was found, log error and continue without IP. - error "No routable IP address found, check your network connection", - error = ip.error - return (none(IpAddress), NoRoutingInfo) - elif ip.get().isGlobalUnicast(): - return (some(ip.get()), PrefSrcIsPublic) - else: - return (none(IpAddress), PrefSrcIsPrivate) - elif bindAddress.isGlobalUnicast(): - return (some(bindIp), BindAddressIsPublic) - else: - return (none(IpAddress), BindAddressIsPrivate) - -# Try to detect a public IP assigned to this host, before trying NAT traversal. -proc getPublicRoutePrefSrcOrExternalIP*( - natStrategy: NatStrategy, bindIp: IpAddress, quiet = true -): Option[IpAddress] = - let (prefSrcIp, prefSrcStatus) = getRoutePrefSrc(bindIp) - - case prefSrcStatus - of NoRoutingInfo, PrefSrcIsPublic, BindAddressIsPublic: - return prefSrcIp - of PrefSrcIsPrivate, BindAddressIsPrivate: - let extIp = getExternalIP(natStrategy, quiet) - if extIp.isSome: - return some(extIp.get) + return false proc doPortMapping( strategy: NatStrategy, tcpPort, udpPort: Port, description: string @@ -262,10 +184,8 @@ proc repeatPortMapping(args: PortMappingArgs) {.thread, raises: [ValueError].} = # We can't use copies of Miniupnp and NatPmp objects in this thread, because they share # C pointers with other instances that have already been garbage collected, so - # we use threadvars instead and initialise them again with getExternalIP(), - # even though we don't need the external IP's value. - let ipres = getExternalIP(strategy, quiet = true) - if ipres.isSome: + # we use threadvars instead and initialise them again here. + if initNatDevice(strategy, quiet = true): while natClosed.load() == false: let currTime = now() if currTime >= (lastUpdate + interval): @@ -292,8 +212,7 @@ proc stopNatThreads() {.noconv.} = # In Windows, a new thread is created for the signal handler, so we need to # initialise our threadvars again. - let ipres = getExternalIP(strategy, quiet = true) - if ipres.isSome: + if initNatDevice(strategy, quiet = true): if strategy == NatStrategy.NatUpnp: for entry in activeMappings: for t in [ @@ -358,55 +277,23 @@ proc redirectPorts*( proc setupNat*( natStrategy: NatStrategy, tcpPort, udpPort: Port, clientId: string -): tuple[ip: Option[IpAddress], tcpPort, udpPort: Option[Port]] = - ## Setup NAT port mapping and get external IP address. - ## If any of this fails, we don't return any IP address but do return the - ## original ports as best effort. +): Option[(Port, Port)] = + ## Setup NAT port mapping. + ## Returns the external (tcpPort, udpPort) if port mapping succeeded, none otherwise. ## TODO: Allow for tcp or udp port mapping to be optional. - if extIp.isNone: - extIp = getExternalIP(natStrategy) - if extIp.isSome: - let ip = extIp.get - let extPorts = ( - {.gcsafe.}: - redirectPorts( - strategy, tcpPort = tcpPort, udpPort = udpPort, description = clientId - ) - ) - if extPorts.isSome: - let (extTcpPort, extUdpPort) = extPorts.get() - (ip: some(ip), tcpPort: some(extTcpPort), udpPort: some(extUdpPort)) - else: - warn "UPnP/NAT-PMP available but port forwarding failed" - (ip: none(IpAddress), tcpPort: some(tcpPort), udpPort: some(udpPort)) - else: + if not initNatDevice(natStrategy): warn "UPnP/NAT-PMP not available" - (ip: none(IpAddress), tcpPort: some(tcpPort), udpPort: some(udpPort)) + return none((Port, Port)) -proc setupAddress*( - natConfig: NatConfig, bindIp: IpAddress, tcpPort, udpPort: Port, clientId: string -): tuple[ip: Option[IpAddress], tcpPort, udpPort: Option[Port]] {.gcsafe.} = - ## Set-up of the external address via any of the ways as configured in - ## `NatConfig`. In case all fails an error is logged and the bind ports are - ## selected also as external ports, as best effort and in hope that the - ## external IP can be figured out by other means at a later stage. - ## TODO: Allow for tcp or udp bind ports to be optional. - - if natConfig.hasExtIp: - # any required port redirection must be done by hand - return (some(natConfig.extIp), some(tcpPort), some(udpPort)) - - case natConfig.nat - of NatStrategy.NatAuto: - let (prefSrcIp, prefSrcStatus) = getRoutePrefSrc(bindIp) - - case prefSrcStatus - of NoRoutingInfo, PrefSrcIsPublic, BindAddressIsPublic: - return (prefSrcIp, some(tcpPort), some(udpPort)) - of PrefSrcIsPrivate, BindAddressIsPrivate: - return setupNat(natConfig.nat, tcpPort, udpPort, clientId) - of NatStrategy.NatUpnp, NatStrategy.NatPmp: - return setupNat(natConfig.nat, tcpPort, udpPort, clientId) + let extPorts = ( + {.gcsafe.}: + redirectPorts( + strategy, tcpPort = tcpPort, udpPort = udpPort, description = clientId + ) + ) + if extPorts.isNone: + warn "UPnP/NAT-PMP available but port forwarding failed" + extPorts proc findReachableNodes*(bootstrapNodes: seq[SignedPeerRecord]): seq[SignedPeerRecord] = ## Returns the list of nodes known to be directly reachable. @@ -414,56 +301,13 @@ proc findReachableNodes*(bootstrapNodes: seq[SignedPeerRecord]): seq[SignedPeerR ## confirmed reachable by AutoNAT could be included. bootstrapNodes -proc nattedAddress*( - natConfig: NatConfig, addrs: seq[MultiAddress], udpPort: Port -): tuple[libp2p, discovery: seq[MultiAddress]] = - ## Takes a NAT configuration, sequence of multiaddresses and UDP port and returns: - ## - Modified multiaddresses with NAT-mapped addresses for libp2p - ## - Discovery addresses with NAT-mapped UDP ports +proc nattedPorts*(natConfig: NatConfig, tcpPort, udpPort: Port): Option[(Port, Port)] = + if natConfig.hasExtIp: + return none((Port, Port)) # manual setup, no port mapping needed + setupNat(natConfig.nat, tcpPort, udpPort, "storage") - var discoveryAddrs = newSeq[MultiAddress](0) - let newAddrs = addrs.mapIt: - block: - # Extract IP address and port from the multiaddress - let (ipPart, port) = getAddressAndPort(it) - if ipPart.isSome and port.isSome: - # Try to setup NAT mapping for the address - let (newIP, tcp, udp) = - setupAddress(natConfig, ipPart.get, port.get, udpPort, "storage") - if newIP.isSome: - # NAT mapping successful - add discovery address with mapped UDP port - discoveryAddrs.add(getMultiAddrWithIPAndUDPPort(newIP.get, udp.get)) - # Remap original address with NAT IP and TCP port - it.remapAddr(ip = newIP, port = tcp) - else: - # NAT mapping failed - use original address - echo "Failed to get external IP, using original address", it - discoveryAddrs.add(getMultiAddrWithIPAndUDPPort(ipPart.get, udpPort)) - it - else: - # Invalid multiaddress format - return as is - it - (newAddrs, discoveryAddrs) - -method mapNatAddresses*( - m: DefaultNatMapper, addrs: seq[MultiAddress] -): tuple[libp2p, discovery: seq[MultiAddress]] {.gcsafe, raises: [].} = - nattedAddress(m.natConfig, addrs, m.discoveryPort) - -method getReachableAddresses*( - m: DefaultNatMapper, addrs: seq[MultiAddress] -): tuple[libp2p, discovery: seq[MultiAddress]] {.gcsafe, raises: [].} = - let ip = - if m.natConfig.hasExtIp: - some(m.natConfig.extIp) - else: - let (routeIp, _) = getRoutePrefSrc(static parseIpAddress("0.0.0.0")) - routeIp - if ip.isNone: - return (@[], @[]) - let announceAddrs = - addrs.mapIt(it.remapAddr(ip = ip, port = none(Port))).deduplicate() - (announceAddrs, @[getMultiAddrWithIPAndUDPPort(ip.get, m.discoveryPort)]) +method mapNatPorts*(m: DefaultNatMapper): Option[(Port, Port)] {.gcsafe, raises: [].} = + nattedPorts(m.natConfig, m.tcpPort, m.discoveryPort) proc hasPublicIp*(addrs: seq[MultiAddress]): bool = for addr in addrs: @@ -473,6 +317,8 @@ proc hasPublicIp*(addrs: seq[MultiAddress]): bool = proc handleNatStatus*( networkReachability: NetworkReachability, + dialBackAddr: Opt[MultiAddress], + discoveryPort: Port, mapper: NatMapper, discovery: Discovery, switch: Switch, @@ -483,33 +329,42 @@ proc handleNatStatus*( # Nothing to do here, not enough confidence score result discard of Reachable: - # For UPnP, it the mapping was a success, - # the autorelay service has been stopped - # and the address was already announced + if dialBackAddr.isNone: + warn "Got empty dialback address in AutoNat when node is Reachable" + return + if autoRelayService.isRunning: if not await autoRelayService.stop(switch): debug "AutoRelayService stop method returned false" - let (announceAddrs, discoveryAddrs) = - mapper.getReachableAddresses(switch.peerInfo.addrs) - discovery.updateAnnounceRecord(announceAddrs) - discovery.updateDhtRecord(announceAddrs & discoveryAddrs) + let discAddr = + dialBackAddr.get.remapAddr(protocol = some("udp"), port = some(discoveryPort)) + discovery.updateAnnounceRecord(@[dialBackAddr.get]) + discovery.updateDhtRecord(@[dialBackAddr.get, discAddr]) # TODO: switch DHT to server mode of NotReachable: - let (announceAddrs, discoveryAddrs) = mapper.mapNatAddresses(switch.peerInfo.addrs) + var hasPortMapping = false - # With a UPnP / NatPmP successful mapping, - # we suppose that having a public IP make it Reachable. - # If not, the state will be updated in the next Autonat iteration. - # TODO: Do we need to manually call dialMe to make sure we are Reachable ? - if hasPublicIp(announceAddrs): - discovery.updateAnnounceRecord(announceAddrs) - discovery.updateDhtRecord(announceAddrs & discoveryAddrs) + if dialBackAddr.isSome: + let maybePorts = mapper.mapNatPorts() - if autoRelayService.isRunning: - if not await autoRelayService.stop(switch): - debug "AutoRelayService stop method returned false" - else: - if not autoRelayService.isRunning: - if not await autoRelayService.setup(switch): - debug "AutoRelayService setup method returned false" + if maybePorts.isSome: + let (tcpPort, udpPort) = maybePorts.get() + let announceAddr = dialBackAddr.get.remapAddr(port = some(tcpPort)) + let discAddr = + dialBackAddr.get.remapAddr(protocol = some("udp"), port = some(udpPort)) + + # TODO: Try a dial me to make sure we are reachable + + if autoRelayService.isRunning: + if not await autoRelayService.stop(switch): + debug "AutoRelayService stop method returned false" + + discovery.updateAnnounceRecord(@[announceAddr]) + discovery.updateDhtRecord(@[announceAddr, discAddr]) + + hasPortMapping = true + + if not hasPortMapping and not autoRelayService.isRunning: + if not await autoRelayService.setup(switch): + debug "AutoRelayService setup method returned false" diff --git a/storage/storage.nim b/storage/storage.nim index 702522f7..86d2407a 100644 --- a/storage/storage.nim +++ b/storage/storage.nim @@ -373,15 +373,19 @@ proc new*( switch.mount(network) switch.mount(manifestProto) - let natMapper = - DefaultNatMapper(natConfig: config.nat, discoveryPort: config.discoveryPort) + let natMapper = DefaultNatMapper( + natConfig: config.nat, + tcpPort: config.listenPort, + discoveryPort: config.discoveryPort, + ) autonatService.setStatusAndConfidenceHandler( proc( networkReachability: NetworkReachability, confidence: Opt[float] ) {.async: (raises: [CancelledError]).} = debug "AutoNAT status", reachability = networkReachability, confidence await handleNatStatus( - networkReachability, natMapper, discovery, switch, autoRelayService + networkReachability, addrs, config.discoveryPort, natMapper, discovery, switch, + autoRelayService, ) ) diff --git a/storage/utils/addrutils.nim b/storage/utils/addrutils.nim index 31570e06..09891cfa 100644 --- a/storage/utils/addrutils.nim +++ b/storage/utils/addrutils.nim @@ -20,8 +20,9 @@ func remapAddr*( address: MultiAddress, ip: Option[IpAddress] = IpAddress.none, port: Option[Port] = Port.none, + protocol: Option[string] = string.none, ): MultiAddress = - ## Remap addresses to new IP and/or Port + ## Remap addresses to new IP, port, and/or transport protocol (e.g. "tcp" → "udp") ## var parts = ($address).split("/") @@ -32,6 +33,12 @@ func remapAddr*( else: parts[2] + parts[3] = + if protocol.isSome: + protocol.get + else: + parts[3] + parts[4] = if port.isSome: $port.get diff --git a/tests/storage/helpers/nodeutils.nim b/tests/storage/helpers/nodeutils.nim index 11aa246e..f084b423 100644 --- a/tests/storage/helpers/nodeutils.nim +++ b/tests/storage/helpers/nodeutils.nim @@ -226,13 +226,8 @@ proc generateNodes*( if config.enableBootstrap: waitFor switch.peerInfo.update() - let (announceAddrs, discoveryAddrs) = nattedAddress( - nat.NatConfig(hasExtIp: false, nat: NatAuto), - switch.peerInfo.addrs, - bindPort.Port, - ) - blockDiscovery.updateAnnounceRecord(announceAddrs) - blockDiscovery.updateDhtRecord(discoveryAddrs) + blockDiscovery.updateAnnounceRecord(switch.peerInfo.addrs) + blockDiscovery.updateDhtRecord(switch.peerInfo.addrs) if blockDiscovery.dhtRecord.isSome: bootstrapNodes.add !blockDiscovery.dhtRecord diff --git a/tests/storage/testnat.nim b/tests/storage/testnat.nim index ee9ed27e..70b80572 100644 --- a/tests/storage/testnat.nim +++ b/tests/storage/testnat.nim @@ -15,66 +15,34 @@ import ../../storage/discovery import ../../storage/rng import ../../storage/utils import ../../storage/utils/natutils +import ../../storage/utils/addrutils type MockNatMapper = ref object of NatMapper - mapped: tuple[libp2p, discovery: seq[MultiAddress]] + mappedPorts: Option[(Port, Port)] -method mapNatAddresses*( - m: MockNatMapper, addrs: seq[MultiAddress] -): tuple[libp2p, discovery: seq[MultiAddress]] {.raises: [].} = - m.mapped +method mapNatPorts*(m: MockNatMapper): Option[(Port, Port)] {.raises: [].} = + m.mappedPorts -method getReachableAddresses*( - m: MockNatMapper, addrs: seq[MultiAddress] -): tuple[libp2p, discovery: seq[MultiAddress]] {.raises: [].} = - m.mapped +suite "remapAddr": + test "replaces protocol tcp with udp": + let ma = MultiAddress.init("/ip4/1.2.3.4/tcp/5000").expect("valid") + let remapped = ma.remapAddr(protocol = some("udp"), port = some(Port(9000))) + check remapped == MultiAddress.init("/ip4/1.2.3.4/udp/9000").expect("valid") -suite "NAT Address Tests": - test "nattedAddress with local addresses": - # Setup test data - let - udpPort = Port(1234) - natConfig = NatConfig(hasExtIp: true, extIp: parseIpAddress("8.8.8.8")) + test "replaces only port, keeping protocol": + let ma = MultiAddress.init("/ip4/1.2.3.4/tcp/5000").expect("valid") + let remapped = ma.remapAddr(port = some(Port(9000))) + check remapped == MultiAddress.init("/ip4/1.2.3.4/tcp/9000").expect("valid") - # Create test addresses - localAddr = MultiAddress.init("/ip4/127.0.0.1/tcp/5000").expect("valid multiaddr") - anyAddr = MultiAddress.init("/ip4/0.0.0.0/tcp/5000").expect("valid multiaddr") - publicAddr = - MultiAddress.init("/ip4/192.168.1.1/tcp/5000").expect("valid multiaddr") + test "replaces only ip, keeping protocol and port": + let ma = MultiAddress.init("/ip4/1.2.3.4/tcp/5000").expect("valid") + let remapped = ma.remapAddr(ip = some(parseIpAddress("8.8.8.8"))) + check remapped == MultiAddress.init("/ip4/8.8.8.8/tcp/5000").expect("valid") - # Expected results - let - expectedDiscoveryAddrs = @[ - MultiAddress.init("/ip4/8.8.8.8/udp/1234").expect("valid multiaddr"), - MultiAddress.init("/ip4/8.8.8.8/udp/1234").expect("valid multiaddr"), - MultiAddress.init("/ip4/8.8.8.8/udp/1234").expect("valid multiaddr"), - ] - expectedlibp2pAddrs = @[ - MultiAddress.init("/ip4/8.8.8.8/tcp/5000").expect("valid multiaddr"), - MultiAddress.init("/ip4/8.8.8.8/tcp/5000").expect("valid multiaddr"), - MultiAddress.init("/ip4/8.8.8.8/tcp/5000").expect("valid multiaddr"), - ] - #ipv6Addr = MultiAddress.init("/ip6/::1/tcp/5000").expect("valid multiaddr") - addrs = @[localAddr, anyAddr, publicAddr] - - # Test address remapping - let (libp2pAddrs, discoveryAddrs) = nattedAddress(natConfig, addrs, udpPort) - - # Verify results - check(discoveryAddrs == expectedDiscoveryAddrs) - check(libp2pAddrs == expectedlibp2pAddrs) - -suite "getReachableAddresses": - test "returns remapped addresses when extIp is configured": - let - natConfig = NatConfig(hasExtIp: true, extIp: parseIpAddress("1.2.3.4")) - mapper = DefaultNatMapper(natConfig: natConfig, discoveryPort: Port(8090)) - listenAddr = MultiAddress.init("/ip4/0.0.0.0/tcp/5000").expect("valid") - - let (libp2pAddrs, discAddrs) = mapper.getReachableAddresses(@[listenAddr]) - - check libp2pAddrs == @[MultiAddress.init("/ip4/1.2.3.4/tcp/5000").expect("valid")] - check discAddrs == @[MultiAddress.init("/ip4/1.2.3.4/udp/8090").expect("valid")] +suite "nattedPorts": + test "returns none when extIp is configured (manual setup)": + let natConfig = NatConfig(hasExtIp: true, extIp: parseIpAddress("8.8.8.8")) + check nattedPorts(natConfig, Port(5000), Port(1234)).isNone suite "hasPublicIp": test "hasPublicIp returns true when the address is public": @@ -107,51 +75,47 @@ asyncchecksuite "handleNatStatus": if autoRelay.isRunning: discard await autoRelay.stop(sw) - test "handleNatStatus announces address when the node is not Reachable and the UPnP succeed with public ip": - let announceAddr = MultiAddress.init("/ip4/1.2.3.4/tcp/8080").expect("valid") - let discAddr = MultiAddress.init("/ip4/1.2.3.4/udp/8090").expect("valid") - let mapper = MockNatMapper(mapped: (@[announceAddr], @[discAddr])) + let discoveryPort = Port(8090) - await handleNatStatus(NotReachable, mapper, disc, sw, autoRelay) + test "handleNatStatus announces mapped address when NotReachable and UPnP succeeds": + let dialBack = MultiAddress.init("/ip4/1.2.3.4/tcp/8080").expect("valid") + let mapper = MockNatMapper(mappedPorts: some((Port(9000), Port(9001)))) - check disc.announceAddrs == @[announceAddr] + await handleNatStatus( + NotReachable, Opt.some(dialBack), discoveryPort, mapper, disc, sw, autoRelay + ) + + check disc.announceAddrs == + @[MultiAddress.init("/ip4/1.2.3.4/tcp/9000").expect("valid")] check not autoRelay.isRunning - # test "handleNatStatus does not announce address when the node is not Reachable and the UPnP succeed with private ip": - # let privateAddr = MultiAddress.init("/ip4/192.168.1.1/tcp/8080").expect("valid") - # let mapper = MockNatMapper(mapped: (@[privateAddr], @[])) + test "handleNatStatus starts autoRelay when NotReachable and UPnP failed": + let mapper = MockNatMapper(mappedPorts: none((Port, Port))) - # await handleNatStatus( - # NotReachable, mapper, disc, sw, autoRelay - # ) - - # check disc.announceAddrs == @[] - # check not autoRelay.isRunning - - test "handleNatStatus starts autoRelay when node is not Reachable and UPnP failed": - let mapper = MockNatMapper(mapped: (@[], @[])) - - await handleNatStatus(NotReachable, mapper, disc, sw, autoRelay) + await handleNatStatus( + NotReachable, Opt.none(MultiAddress), discoveryPort, mapper, disc, sw, autoRelay + ) check autoRelay.isRunning - # The addresses will be announced in the onReservation callback - # after a node accepted a Relay reservation. - test "handleNatStatus does not announce address when node is Reachable and relay is not running": - let mapper = MockNatMapper(mapped: (@[], @[])) + test "handleNatStatus does not announce address when Reachable and no dialBackAddr": + let mapper = MockNatMapper(mappedPorts: none((Port, Port))) - await handleNatStatus(Reachable, mapper, disc, sw, autoRelay) + await handleNatStatus( + Reachable, Opt.none(MultiAddress), discoveryPort, mapper, disc, sw, autoRelay + ) check disc.announceAddrs == newSeq[MultiAddress]() check not autoRelay.isRunning - test "handleNatStatus stops relay and announces address when node is Reachable and relay is running": - let announceAddr = MultiAddress.init("/ip4/1.2.3.4/tcp/8080").expect("valid") - let discAddr = MultiAddress.init("/ip4/1.2.3.4/udp/8090").expect("valid") - let mapper = MockNatMapper(mapped: (@[announceAddr], @[discAddr])) + test "handleNatStatus stops relay and announces dialBackAddr when Reachable": + let dialBack = MultiAddress.init("/ip4/1.2.3.4/tcp/8080").expect("valid") + let mapper = MockNatMapper(mappedPorts: none((Port, Port))) discard await autorelayservice.setup(autoRelay, sw) - await handleNatStatus(Reachable, mapper, disc, sw, autoRelay) + await handleNatStatus( + Reachable, Opt.some(dialBack), discoveryPort, mapper, disc, sw, autoRelay + ) check not autoRelay.isRunning - check disc.announceAddrs == @[announceAddr] + check disc.announceAddrs == @[dialBack] From affe4e2c006493d85755d56cac9cfb6fc007cd06 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Thu, 4 Jun 2026 14:34:53 +0400 Subject: [PATCH 023/167] Update imports --- storage/nat.nim | 29 +++++------ storage/storage.nim | 34 ++++--------- storage/utils/addrutils.nim | 35 ------------- storage/utils/natutils.nim | 50 +++++++------------ tests/integration/1_minute/testnat.nim | 1 - tests/integration/storageconfig.nim | 8 +++ tests/storage/testnatutils.nim | 68 +------------------------- 7 files changed, 51 insertions(+), 174 deletions(-) diff --git a/storage/nat.nim b/storage/nat.nim index 863b1e68..9e4e591b 100644 --- a/storage/nat.nim +++ b/storage/nat.nim @@ -16,7 +16,7 @@ import import pkg/chronos import pkg/chronicles import pkg/libp2p -import pkg/libp2p/protocols/connectivity/autonat/types +import pkg/libp2p/protocols/connectivity/autonatv2/service import pkg/libp2p/services/autorelayservice import ./utils @@ -58,7 +58,7 @@ logScope: type NatMapper* = ref object of RootObj method mapNatPorts*(m: NatMapper): Option[(Port, Port)] {.base, gcsafe, raises: [].} = - raiseAssert "mapNatPorts not implemented" + none((Port, Port)) type DefaultNatMapper* = ref object of NatMapper natConfig*: NatConfig @@ -303,18 +303,13 @@ proc findReachableNodes*(bootstrapNodes: seq[SignedPeerRecord]): seq[SignedPeerR proc nattedPorts*(natConfig: NatConfig, tcpPort, udpPort: Port): Option[(Port, Port)] = if natConfig.hasExtIp: - return none((Port, Port)) # manual setup, no port mapping needed - setupNat(natConfig.nat, tcpPort, udpPort, "storage") + return none((Port, Port)) + let clientId = "storage" + return setupNat(natConfig.nat, tcpPort, udpPort, clientId) method mapNatPorts*(m: DefaultNatMapper): Option[(Port, Port)] {.gcsafe, raises: [].} = nattedPorts(m.natConfig, m.tcpPort, m.discoveryPort) -proc hasPublicIp*(addrs: seq[MultiAddress]): bool = - for addr in addrs: - let (ip, _) = getAddressAndPort(addr) - if ip.isSome and isGlobalUnicast(ip.get): - return true - proc handleNatStatus*( networkReachability: NetworkReachability, dialBackAddr: Opt[MultiAddress], @@ -340,18 +335,20 @@ proc handleNatStatus*( let discAddr = dialBackAddr.get.remapAddr(protocol = some("udp"), port = some(discoveryPort)) discovery.updateAnnounceRecord(@[dialBackAddr.get]) - discovery.updateDhtRecord(@[dialBackAddr.get, discAddr]) + discovery.updateDhtRecord(@[discAddr]) # TODO: switch DHT to server mode of NotReachable: var hasPortMapping = false - if dialBackAddr.isSome: + if dialBackAddr.isNone: + warn "Got empty dialback address in AutoNat when node is Reachable" + else: let maybePorts = mapper.mapNatPorts() if maybePorts.isSome: let (tcpPort, udpPort) = maybePorts.get() - let announceAddr = dialBackAddr.get.remapAddr(port = some(tcpPort)) - let discAddr = + let announceAddress = dialBackAddr.get.remapAddr(port = some(tcpPort)) + let discoveryAddrs = dialBackAddr.get.remapAddr(protocol = some("udp"), port = some(udpPort)) # TODO: Try a dial me to make sure we are reachable @@ -360,8 +357,8 @@ proc handleNatStatus*( if not await autoRelayService.stop(switch): debug "AutoRelayService stop method returned false" - discovery.updateAnnounceRecord(@[announceAddr]) - discovery.updateDhtRecord(@[announceAddr, discAddr]) + discovery.updateAnnounceRecord(@[announceAddress]) + discovery.updateDhtRecord(@[discoveryAddrs]) hasPortMapping = true diff --git a/storage/storage.nim b/storage/storage.nim index 86d2407a..2ad6dcc1 100644 --- a/storage/storage.nim +++ b/storage/storage.nim @@ -41,7 +41,6 @@ import ./namespaces import ./storagetypes import ./logutils import ./nat -import ./utils/natutils logScope: topics = "storage node" @@ -88,29 +87,16 @@ proc start*(s: StorageServer) {.async.} = else: getBestLocalAddress(s.config.listenIp) - if announceIp.isNone: - # We should have an IP, even at private IP - raise newException(StorageError, "Unable to determine an IP address to announce") - - # Remap switch addresses to the resolved IP (replaces 0.0.0.0 or :: with the actual address), - # keeping unique entries only. - let announceAddrs = s.storageNode.switch.peerInfo.addrs - .mapIt(it.remapAddr(ip = announceIp, port = none(Port))) - .deduplicate() - let discoveryAddrs = - @[getMultiAddrWithIPAndUDPPort(announceIp.get, s.config.discoveryPort)] - s.storageNode.discovery.updateDhtRecord(announceAddrs & discoveryAddrs) - s.storageNode.discovery.updateAnnounceRecord(announceAddrs) - - var hasPublicAddr = false - for announceAddr in announceAddrs: - let (maybeIp, _) = getAddressAndPort(announceAddr) - if maybeIp.isSome and maybeIp.get.isGlobalUnicast(): - hasPublicAddr = true - break - - if not hasPublicAddr: - warn "Unable to determine a public IP address. This node will only be reachable on a private network." + if announceIp.isSome: + let ip = announceIp.get + let announceAddrs = s.storageNode.switch.peerInfo.addrs + .mapIt(it.remapAddr(ip = some(ip), port = none(Port))) + .deduplicate() + let discAddr = getMultiAddrWithIPAndUDPPort(ip, s.config.discoveryPort) + s.storageNode.discovery.updateAnnounceRecord(announceAddrs) + s.storageNode.discovery.updateDhtRecord(announceAddrs & @[discAddr]) + else: + warn "Unable to determine a local IP address to announce" await s.storageNode.start() diff --git a/storage/utils/addrutils.nim b/storage/utils/addrutils.nim index 09891cfa..72d80ad3 100644 --- a/storage/utils/addrutils.nim +++ b/storage/utils/addrutils.nim @@ -74,38 +74,3 @@ proc getMultiAddrWithIpAndTcpPort*(ip: IpAddress, port: Port): MultiAddress = return MultiAddress.init(ipFamily & $ip & "/tcp/" & $port).expect( "Failed to construct multiaddress with IP and TCP port" ) - -proc getAddressAndPort*( - ma: MultiAddress -): tuple[ip: Option[IpAddress], port: Option[Port]] = - try: - # Try IPv4 first - let ipv4Result = ma[multiCodec("ip4")] - let ip = - if ipv4Result.isOk: - let ipBytes = ipv4Result.get().protoArgument().expect("Invalid IPv4 format") - let ipArray = [ipBytes[0], ipBytes[1], ipBytes[2], ipBytes[3]] - some(IpAddress(family: IPv4, address_v4: ipArray)) - else: - # Try IPv6 if IPv4 not found - let ipv6Result = ma[multiCodec("ip6")] - if ipv6Result.isOk: - let ipBytes = ipv6Result.get().protoArgument().expect("Invalid IPv6 format") - var ipArray: array[16, byte] - for i in 0 .. 15: - ipArray[i] = ipBytes[i] - some(IpAddress(family: IPv6, address_v6: ipArray)) - else: - none(IpAddress) - - # Get TCP Port - let portResult = ma[multiCodec("tcp")] - let port = - if portResult.isOk: - let portBytes = portResult.get().protoArgument().expect("Invalid port format") - some(Port(fromBytesBE(uint16, portBytes))) - else: - none(Port) - (ip: ip, port: port) - except Exception: - (ip: none(IpAddress), port: none(Port)) diff --git a/storage/utils/natutils.nim b/storage/utils/natutils.nim index 04cc57f9..b808d915 100644 --- a/storage/utils/natutils.nim +++ b/storage/utils/natutils.nim @@ -1,6 +1,6 @@ {.push raises: [].} -import std/[net, tables, hashes, options], pkg/results, chronos, chronicles +import std/[net, options], pkg/results, chronos, chronicles import pkg/libp2p @@ -9,54 +9,42 @@ type NatStrategy* = enum NatUpnp NatPmp -func isGlobalUnicast*(address: TransportAddress): bool = - if address.isGlobal() and address.isUnicast(): true else: false +proc getRouteIpv4*(): Result[IpAddress, cstring] = + let + publicAddress = TransportAddress( + family: AddressFamily.IPv4, address_v4: [1'u8, 1, 1, 1], port: Port(0) + ) + route = getBestRoute(publicAddress) -func isGlobalUnicast*(address: IpAddress): bool = - let a = initTAddress(address, Port(0)) - a.isGlobalUnicast() - -proc getRoute(publicAddress: TransportAddress): Result[IpAddress, cstring] = - let route = getBestRoute(publicAddress) - - if route.source.family == AddressFamily.None or route.source.isUnspecified(): - err("No best route found") + if route.source.isUnspecified(): + err("No best ipv4 route found") else: let ip = try: route.source.address() except ValueError as e: - # This should not occur really. error "Address conversion error", exception = e.name, msg = e.msg return err("Invalid IP address") ok(ip) -proc getRouteIpv4*(): Result[IpAddress, cstring] = - # Avoiding Exception with initTAddress and can't make it work with static. - # Note: `publicAddress` is only used an "example" IP to find the best route, - # no data is send over the network to this IP! - let publicAddress = TransportAddress( - family: AddressFamily.IPv4, address_v4: [1'u8, 1, 1, 1], port: Port(0) - ) - - return getRoute(publicAddress) - proc getRouteIpv6*(): Result[IpAddress, cstring] = - # Note: `googleDnsIpv6` is only used as an "example" IP to find the best route, - # no data is sent over the network to this IP! const googleDnsIpv6 = TransportAddress( family: AddressFamily.IPv6, - # 2001:4860:4860::8888 address_v6: [32'u8, 1, 72, 96, 72, 96, 0, 0, 0, 0, 0, 0, 0, 0, 136, 136], port: Port(0), ) + let route = getBestRoute(googleDnsIpv6) + if route.source.isUnspecified(): + return err("No best ipv6 route found") + try: + ok(route.source.address()) + except ValueError as e: + error "Address conversion error", exception = e.name, msg = e.msg + err("Invalid IP address") - return getRoute(googleDnsIpv6) - -# If bindIp is a anyLocal address (0.0.0.0 or ::), -# the function will find the best ip address. -# Otherwise, it will just return the ip as it is. proc getBestLocalAddress*(bindIp: IpAddress): Option[IpAddress] = + ## If bindIp is anyLocal (0.0.0.0 or ::), finds the best local IP via routing table. + ## Otherwise returns bindIp as-is. let bindAddress = initTAddress(bindIp, Port(0)) if bindAddress.isAnyLocal(): let ip = diff --git a/tests/integration/1_minute/testnat.nim b/tests/integration/1_minute/testnat.nim index 4aa13bce..50c0467d 100644 --- a/tests/integration/1_minute/testnat.nim +++ b/tests/integration/1_minute/testnat.nim @@ -1,6 +1,5 @@ import std/json import std/options -import std/sequtils import pkg/chronos import pkg/questionable/results diff --git a/tests/integration/storageconfig.nim b/tests/integration/storageconfig.nim index 240d44a2..691daf34 100644 --- a/tests/integration/storageconfig.nim +++ b/tests/integration/storageconfig.nim @@ -321,3 +321,11 @@ proc withNatScheduleInterval*( for config in startConfig.configs.mitems: config.addCliOption("--nat-schedule-interval", $scheduleInterval) return startConfig + +proc withExtIp*( + self: StorageConfigs, ip = "127.0.0.1" +): StorageConfigs {.raises: [StorageConfigError].} = + var startConfig = self + for config in startConfig.configs.mitems: + config.addCliOption("--nat", "extip:" & ip) + return startConfig diff --git a/tests/storage/testnatutils.nim b/tests/storage/testnatutils.nim index fcc48561..08a6621e 100644 --- a/tests/storage/testnatutils.nim +++ b/tests/storage/testnatutils.nim @@ -1,67 +1 @@ -import std/[unittest, net, options] -import pkg/chronos -import ../../storage/utils/natutils - -suite "isGlobalUnicast": - test "localhost IPv4 is not global unicast": - check not isGlobalUnicast(parseIpAddress("127.0.0.1")) - - test "unspecified IPv4 is not global unicast": - check not isGlobalUnicast(parseIpAddress("0.0.0.0")) - - test "link-local IPv4 is not global unicast": - check not isGlobalUnicast(parseIpAddress("169.254.1.1")) - - test "private IPv4 is not global unicast": - check not isGlobalUnicast(parseIpAddress("10.0.0.1")) - - test "public IPv4 is global unicast": - check isGlobalUnicast(parseIpAddress("8.8.8.8")) - - test "localhost IPv6 is not global unicast": - check not isGlobalUnicast(parseIpAddress("::1")) - - test "unspecified IPv6 is not global unicast": - check not isGlobalUnicast(parseIpAddress("::")) - - test "link-local IPv6 is not global unicast": - check not isGlobalUnicast(parseIpAddress("fe80::1")) - - test "private IPv6 is not global unicast": - check not isGlobalUnicast(parseIpAddress("fc00::1")) - - test "public IPv6 is global unicast": - check isGlobalUnicast(parseIpAddress("2606:4700::1")) - -suite "getRoute": - test "getRouteIpv4 returns a valid IPv4": - let res = getRouteIpv4() - - check res.isOk - check res.get().family == IpAddressFamily.IPv4 - - test "getRouteIpv6 returns a valid IPv6": - let res = getRouteIpv6() - # If the machine does not have a global route because - # it is not configured for IPv6, the test will fail - # because it didn't find the best route. In that case, - # we can just skip the test, because it is not a problem - # with the test itself but the machine configuration. - if res.isErr: - check res.error == "No best route found" - else: - check res.get().family == IpAddressFamily.IPv6 - -suite "getBestLocalAddress": - test "specific IPv4 is returned as it is": - let ip = parseIpAddress("192.168.1.1") - check getBestLocalAddress(ip) == some(ip) - - test "specific IPv6 is returned as it is": - let ip = parseIpAddress("2606:4700::1") - check getBestLocalAddress(ip) == some(ip) - - test "0.0.0.0 resolves to a local IPv4": - let res = getBestLocalAddress(parseIpAddress("0.0.0.0")) - check res.isSome - check res.get().family == IpAddressFamily.IPv4 +discard From d5b032ea7594e46aa0af69e4ca8e0fcb99c3c937 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Wed, 6 May 2026 10:12:39 +0400 Subject: [PATCH 024/167] Update address announcements update --- storage/discovery.nim | 39 +++++++++++++++------ storage/nat.nim | 11 ++---- storage/storage.nim | 35 ++++++++++--------- storage/utils/addrutils.nim | 1 - storage/utils/natutils.nim | 53 ----------------------------- tests/integration/multinodes.nim | 1 + tests/integration/storageconfig.nim | 9 +++++ tests/storage/helpers/nodeutils.nim | 5 +-- tests/storage/testnat.nim | 13 ------- 9 files changed, 60 insertions(+), 107 deletions(-) diff --git a/storage/discovery.nim b/storage/discovery.nim index c5943d88..002c4377 100644 --- a/storage/discovery.nim +++ b/storage/discovery.nim @@ -22,6 +22,7 @@ import pkg/codexdht/discv5/[routing_table, protocol as discv5] from pkg/nimcrypto import keccak256 import ./rng as storage_rng +import ./utils/addrutils import ./errors import ./logutils @@ -175,29 +176,45 @@ method removeProvider*( warn "Error removing provider", peerId = peerId, exc = exc.msg raiseAssert("Unexpected Exception in removeProvider") +proc updateRecords*( + d: Discovery, announceAddrs: openArray[MultiAddress], discoveryPort: Port +) = + ## Update both provider and DHT records from TCP announce addresses. + ## Discovery (UDP) addresses are derived by remapping announceAddrs to UDP with discoveryPort. + ## Updates the discv5 SPR once with the full set of addresses. + let tcpAddrs = @announceAddrs + let udpAddrs = + tcpAddrs.mapIt(it.remapAddr(protocol = some("udp"), port = some(discoveryPort))) + + debug "Updating addresses", tcpAddrs, udpAddrs + + d.announceAddrs = tcpAddrs + d.providerRecord = SignedPeerRecord + .init(d.key, PeerRecord.init(d.peerId, tcpAddrs)) + .expect("Should construct signed record").some + d.dhtRecord = SignedPeerRecord + .init(d.key, PeerRecord.init(d.peerId, tcpAddrs & udpAddrs)) + .expect("Should construct signed record").some + + if not d.protocol.isNil: + d.protocol.updateRecord(d.dhtRecord).expect("Should update SPR") + proc updateAnnounceRecord*(d: Discovery, addrs: openArray[MultiAddress]) = - ## Update providers record - ## - d.announceAddrs = @addrs - info "Updating announce record", addrs = d.announceAddrs d.providerRecord = SignedPeerRecord .init(d.key, PeerRecord.init(d.peerId, d.announceAddrs)) .expect("Should construct signed record").some - if not d.protocol.isNil: d.protocol.updateRecord(d.providerRecord).expect("Should update SPR") -proc updateDhtRecord*(d: Discovery, addrs: openArray[MultiAddress]) = - ## Update providers record - ## - +proc updateDhtRecord*( + d: Discovery, addrs: openArray[MultiAddress] +) {.deprecated: "use updateRecords instead".} = info "Updating Dht record", addrs = addrs d.dhtRecord = SignedPeerRecord .init(d.key, PeerRecord.init(d.peerId, @addrs)) .expect("Should construct signed record").some - if not d.protocol.isNil: d.protocol.updateRecord(d.dhtRecord).expect("Should update SPR") @@ -249,7 +266,7 @@ proc new*( key: key, peerId: PeerId.init(key).expect("Should construct PeerId"), store: store ) - self.updateAnnounceRecord(announceAddrs) + self.updateRecords(@[], Port(0)) let discoveryConfig = DiscoveryConfig(tableIpLimits: tableIpLimits, bitsPerHop: DefaultBitsPerHop) diff --git a/storage/nat.nim b/storage/nat.nim index 9e4e591b..49977830 100644 --- a/storage/nat.nim +++ b/storage/nat.nim @@ -16,7 +16,6 @@ import import pkg/chronos import pkg/chronicles import pkg/libp2p -import pkg/libp2p/protocols/connectivity/autonatv2/service import pkg/libp2p/services/autorelayservice import ./utils @@ -332,10 +331,7 @@ proc handleNatStatus*( if not await autoRelayService.stop(switch): debug "AutoRelayService stop method returned false" - let discAddr = - dialBackAddr.get.remapAddr(protocol = some("udp"), port = some(discoveryPort)) - discovery.updateAnnounceRecord(@[dialBackAddr.get]) - discovery.updateDhtRecord(@[discAddr]) + discovery.updateRecords(@[dialBackAddr.get], discoveryPort) # TODO: switch DHT to server mode of NotReachable: var hasPortMapping = false @@ -348,8 +344,6 @@ proc handleNatStatus*( if maybePorts.isSome: let (tcpPort, udpPort) = maybePorts.get() let announceAddress = dialBackAddr.get.remapAddr(port = some(tcpPort)) - let discoveryAddrs = - dialBackAddr.get.remapAddr(protocol = some("udp"), port = some(udpPort)) # TODO: Try a dial me to make sure we are reachable @@ -357,8 +351,7 @@ proc handleNatStatus*( if not await autoRelayService.stop(switch): debug "AutoRelayService stop method returned false" - discovery.updateAnnounceRecord(@[announceAddress]) - discovery.updateDhtRecord(@[discoveryAddrs]) + discovery.updateRecords(@[announceAddress], udpPort) hasPortMapping = true diff --git a/storage/storage.nim b/storage/storage.nim index 2ad6dcc1..4dca8995 100644 --- a/storage/storage.nim +++ b/storage/storage.nim @@ -36,7 +36,6 @@ import ./blockexchange import ./utils/fileutils import ./discovery import ./utils/addrutils -import ./utils/natutils import ./namespaces import ./storagetypes import ./logutils @@ -81,22 +80,26 @@ proc start*(s: StorageServer) {.async.} = await s.storageNode.switch.start() - let announceIp = + let announceAddrs = if s.config.nat.hasExtIp: - some(s.config.nat.extIp) + # extip means that we assume the IP is reachable + # So we just take the first peer addr and remap it with extip to keep the port only + @[ + s.storageNode.switch.peerInfo.addrs[0].remapAddr( + ip = some(s.config.nat.extIp), port = none(Port) + ) + ] else: - getBestLocalAddress(s.config.listenIp) + # If extip is not set, we have 2 choices: + # 1- Announce the peer addrs contains detected addresses on the machine. + # 2- Wait for AutoNat + # The probleme with 1 is that you will certainly announce private addresses + # and if you advertise a CID, you will advertise these private addresses. + # TODO: DHT client mode + #s.storageNode.switch.peerInfo.addrs + @[] - if announceIp.isSome: - let ip = announceIp.get - let announceAddrs = s.storageNode.switch.peerInfo.addrs - .mapIt(it.remapAddr(ip = some(ip), port = none(Port))) - .deduplicate() - let discAddr = getMultiAddrWithIPAndUDPPort(ip, s.config.discoveryPort) - s.storageNode.discovery.updateAnnounceRecord(announceAddrs) - s.storageNode.discovery.updateDhtRecord(announceAddrs & @[discAddr]) - else: - warn "Unable to determine a local IP address to announce" + s.storageNode.discovery.updateRecords(announceAddrs, s.config.discoveryPort) await s.storageNode.start() @@ -337,8 +340,8 @@ proc new*( client = relayClient, onReservation = proc(addresses: seq[MultiAddress]) {.gcsafe, raises: [].} = debug "Relay reservation updated", addresses - discovery.updateAnnounceRecord(addresses) - discovery.updateDhtRecord(addresses), + # relay addresses are for download traffic only, not DHT routing + discovery.updateAnnounceRecord(addresses), rng = random.Rng.instance(), ) diff --git a/storage/utils/addrutils.nim b/storage/utils/addrutils.nim index 72d80ad3..47204889 100644 --- a/storage/utils/addrutils.nim +++ b/storage/utils/addrutils.nim @@ -14,7 +14,6 @@ import std/strutils import std/options import pkg/libp2p -import pkg/stew/endians2 func remapAddr*( address: MultiAddress, diff --git a/storage/utils/natutils.nim b/storage/utils/natutils.nim index b808d915..9d05dd58 100644 --- a/storage/utils/natutils.nim +++ b/storage/utils/natutils.nim @@ -1,59 +1,6 @@ {.push raises: [].} -import std/[net, options], pkg/results, chronos, chronicles - -import pkg/libp2p - type NatStrategy* = enum NatAuto NatUpnp NatPmp - -proc getRouteIpv4*(): Result[IpAddress, cstring] = - let - publicAddress = TransportAddress( - family: AddressFamily.IPv4, address_v4: [1'u8, 1, 1, 1], port: Port(0) - ) - route = getBestRoute(publicAddress) - - if route.source.isUnspecified(): - err("No best ipv4 route found") - else: - let ip = - try: - route.source.address() - except ValueError as e: - error "Address conversion error", exception = e.name, msg = e.msg - return err("Invalid IP address") - ok(ip) - -proc getRouteIpv6*(): Result[IpAddress, cstring] = - const googleDnsIpv6 = TransportAddress( - family: AddressFamily.IPv6, - address_v6: [32'u8, 1, 72, 96, 72, 96, 0, 0, 0, 0, 0, 0, 0, 0, 136, 136], - port: Port(0), - ) - let route = getBestRoute(googleDnsIpv6) - if route.source.isUnspecified(): - return err("No best ipv6 route found") - try: - ok(route.source.address()) - except ValueError as e: - error "Address conversion error", exception = e.name, msg = e.msg - err("Invalid IP address") - -proc getBestLocalAddress*(bindIp: IpAddress): Option[IpAddress] = - ## If bindIp is anyLocal (0.0.0.0 or ::), finds the best local IP via routing table. - ## Otherwise returns bindIp as-is. - let bindAddress = initTAddress(bindIp, Port(0)) - if bindAddress.isAnyLocal(): - let ip = - if bindIp.family == IpAddressFamily.IPv6: - getRouteIpv6() - else: - getRouteIpv4() - if ip.isOk(): - return some(ip.get()) - return none(IpAddress) - else: - return some(bindIp) diff --git a/tests/integration/multinodes.nim b/tests/integration/multinodes.nim index 034ee657..94b3ede3 100644 --- a/tests/integration/multinodes.nim +++ b/tests/integration/multinodes.nim @@ -213,6 +213,7 @@ template multinodesuite*(suiteName: string, body: untyped) = trace "Setting up test", suite = suiteName, test = currentTestName, nodeConfigs if var clients =? nodeConfigs.clients: failAndTeardownOnError "failed to start client nodes": + clients = clients.withExtIp() for config in clients.configs: let node = await startClientNode(config) running.add RunningNode(role: Role.Client, node: node) diff --git a/tests/integration/storageconfig.nim b/tests/integration/storageconfig.nim index 691daf34..7b218bd3 100644 --- a/tests/integration/storageconfig.nim +++ b/tests/integration/storageconfig.nim @@ -322,6 +322,15 @@ proc withNatScheduleInterval*( config.addCliOption("--nat-schedule-interval", $scheduleInterval) return startConfig +proc withExtIp*( + self: StorageConfigs, idx: int, ip = "127.0.0.1" +): StorageConfigs {.raises: [StorageConfigError].} = + self.checkBounds idx + + var startConfig = self + startConfig.configs[idx].addCliOption("--nat", "extip:" & ip) + return startConfig + proc withExtIp*( self: StorageConfigs, ip = "127.0.0.1" ): StorageConfigs {.raises: [StorageConfigError].} = diff --git a/tests/storage/helpers/nodeutils.nim b/tests/storage/helpers/nodeutils.nim index f084b423..4a957d77 100644 --- a/tests/storage/helpers/nodeutils.nim +++ b/tests/storage/helpers/nodeutils.nim @@ -11,8 +11,6 @@ import pkg/storage/stores import pkg/storage/blocktype as bt import pkg/storage/blockexchange import pkg/storage/systemclock -import pkg/storage/nat -import pkg/storage/utils/natutils import pkg/storage/merkletree import pkg/storage/manifest @@ -226,8 +224,7 @@ proc generateNodes*( if config.enableBootstrap: waitFor switch.peerInfo.update() - blockDiscovery.updateAnnounceRecord(switch.peerInfo.addrs) - blockDiscovery.updateDhtRecord(switch.peerInfo.addrs) + blockDiscovery.updateRecords(switch.peerInfo.addrs, bindPort.Port) if blockDiscovery.dhtRecord.isSome: bootstrapNodes.add !blockDiscovery.dhtRecord diff --git a/tests/storage/testnat.nim b/tests/storage/testnat.nim index 70b80572..f74c876f 100644 --- a/tests/storage/testnat.nim +++ b/tests/storage/testnat.nim @@ -14,7 +14,6 @@ import ../../storage/nat import ../../storage/discovery import ../../storage/rng import ../../storage/utils -import ../../storage/utils/natutils import ../../storage/utils/addrutils type MockNatMapper = ref object of NatMapper @@ -44,18 +43,6 @@ suite "nattedPorts": let natConfig = NatConfig(hasExtIp: true, extIp: parseIpAddress("8.8.8.8")) check nattedPorts(natConfig, Port(5000), Port(1234)).isNone -suite "hasPublicIp": - test "hasPublicIp returns true when the address is public": - let ma = MultiAddress.init("/ip4/8.8.8.8/tcp/8080").expect("valid") - check hasPublicIp(@[ma]) - - test "hasPublicIp returns false when the address is private": - let ma = MultiAddress.init("/ip4/192.168.1.1/tcp/8080").expect("valid") - check not hasPublicIp(@[ma]) - - test "hasPublicIp returns false when the address is empty": - check not hasPublicIp(@[]) - asyncchecksuite "handleNatStatus": var sw: Switch var key: PrivateKey From 4e9411390e5468d9106925a4ed01d30f1c7e5e65 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Wed, 6 May 2026 10:13:36 +0400 Subject: [PATCH 025/167] Add comment --- storage/discovery.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/storage/discovery.nim b/storage/discovery.nim index 002c4377..6e3dc473 100644 --- a/storage/discovery.nim +++ b/storage/discovery.nim @@ -266,6 +266,7 @@ proc new*( key: key, peerId: PeerId.init(key).expect("Should construct PeerId"), store: store ) + # Update with empty values to get a valid SPR self.updateRecords(@[], Port(0)) let discoveryConfig = From b0442d1d19afb8deb2c1304c33afce3ba6761a0f Mon Sep 17 00:00:00 2001 From: Arnaud Date: Thu, 4 Jun 2026 14:35:46 +0400 Subject: [PATCH 026/167] Refactor Upnp --- storage/nat.nim | 333 +++++-------------------------- storage/rest/api.nim | 12 +- storage/storage.nim | 66 +++--- storage/utils/natutils.nim | 201 +++++++++++++++++++ tests/integration/multinodes.nim | 4 +- tests/storage/testnat.nim | 77 +++++-- tests/storage/testnatutils.nim | 94 ++++++++- 7 files changed, 462 insertions(+), 325 deletions(-) diff --git a/storage/nat.nim b/storage/nat.nim index 49977830..616c64e7 100644 --- a/storage/nat.nim +++ b/storage/nat.nim @@ -8,10 +8,8 @@ {.push raises: [].} -import - std/[options, os, times, atomics, exitprocs], - nat_traversal/[miniupnpc, natpmp], - results +import std/[options, net] +import results import pkg/chronos import pkg/chronicles @@ -23,304 +21,52 @@ import ./utils/natutils import ./utils/addrutils import ./discovery -const - UPNP_TIMEOUT = 200 # ms - PORT_MAPPING_INTERVAL = 20 * 60 # seconds - NATPMP_LIFETIME = 60 * 60 # in seconds, must be longer than PORT_MAPPING_INTERVAL - -type PortMappings* = object - internalTcpPort: Port - externalTcpPort: Port - internalUdpPort: Port - externalUdpPort: Port - description: string - -type PortMappingArgs = - tuple[strategy: NatStrategy, tcpPort, udpPort: Port, description: string] +logScope: + topics = "nat" type NatConfig* = object case hasExtIp*: bool of true: extIp*: IpAddress of false: nat*: NatStrategy -var - upnp {.threadvar.}: Miniupnp - npmp {.threadvar.}: NatPmp - strategy = NatStrategy.NatAuto - natClosed: Atomic[bool] - activeMappings: seq[PortMappings] - natThreads: seq[Thread[PortMappingArgs]] = @[] - -logScope: - topics = "nat" - type NatMapper* = ref object of RootObj - -method mapNatPorts*(m: NatMapper): Option[(Port, Port)] {.base, gcsafe, raises: [].} = - none((Port, Port)) - -type DefaultNatMapper* = ref object of NatMapper natConfig*: NatConfig tcpPort*: Port discoveryPort*: Port + hasUpnpMapping: bool -## Initialises the UPnP or NAT-PMP threadvar and sets the `strategy` threadvar. -## Must be called before redirectPorts() in each thread. -proc initNatDevice(natStrategy: NatStrategy, quiet = false): bool = - if natStrategy == NatStrategy.NatAuto or natStrategy == NatStrategy.NatUpnp: - if upnp == nil: - upnp = newMiniupnp() - - upnp.discoverDelay = UPNP_TIMEOUT - let dres = upnp.discover() - if dres.isErr: - debug "UPnP", msg = dres.error - else: - var - msg: cstring - canContinue = true - case upnp.selectIGD() - of IGDNotFound: - msg = "Internet Gateway Device not found. Giving up." - canContinue = false - of IGDFound: - msg = "Internet Gateway Device found." - of IGDNotConnected: - msg = "Internet Gateway Device found but it's not connected. Trying anyway." - of NotAnIGD: - msg = - "Some device found, but it's not recognised as an Internet Gateway Device. Trying anyway." - of IGDIpNotRoutable: - msg = - "Internet Gateway Device found and is connected, but with a reserved or non-routable IP. Trying anyway." - if not quiet: - debug "UPnP", msg - if canContinue: - strategy = NatStrategy.NatUpnp - return true - - if natStrategy == NatStrategy.NatAuto or natStrategy == NatStrategy.NatPmp: - if npmp == nil: - npmp = newNatPmp() - let nres = npmp.init() - if nres.isErr: - debug "NAT-PMP", msg = nres.error - else: - strategy = NatStrategy.NatPmp - return true - - return false - -proc doPortMapping( - strategy: NatStrategy, tcpPort, udpPort: Port, description: string -): Option[(Port, Port)] {.gcsafe.} = - var - extTcpPort: Port - extUdpPort: Port - - if strategy == NatStrategy.NatUpnp: - for t in [(tcpPort, UPNPProtocol.TCP), (udpPort, UPNPProtocol.UDP)]: - let - (port, protocol) = t - pmres = upnp.addPortMapping( - externalPort = $port, - protocol = protocol, - internalHost = upnp.lanAddr, - internalPort = $port, - desc = description, - leaseDuration = 0, - ) - if pmres.isErr: - error "UPnP port mapping", msg = pmres.error, port - return - else: - # let's check it - let cres = - upnp.getSpecificPortMapping(externalPort = $port, protocol = protocol) - if cres.isErr: - warn "UPnP port mapping check failed. Assuming the check itself is broken and the port mapping was done.", - msg = cres.error - - info "UPnP: added port mapping", - externalPort = port, internalPort = port, protocol = protocol - case protocol - of UPNPProtocol.TCP: - extTcpPort = port - of UPNPProtocol.UDP: - extUdpPort = port - elif strategy == NatStrategy.NatPmp: - for t in [(tcpPort, NatPmpProtocol.TCP), (udpPort, NatPmpProtocol.UDP)]: - let - (port, protocol) = t - pmres = npmp.addPortMapping( - eport = port.cushort, - iport = port.cushort, - protocol = protocol, - lifetime = NATPMP_LIFETIME, - ) - if pmres.isErr: - error "NAT-PMP port mapping", msg = pmres.error, port - return - else: - let extPort = Port(pmres.value) - info "NAT-PMP: added port mapping", - externalPort = extPort, internalPort = port, protocol = protocol - case protocol - of NatPmpProtocol.TCP: - extTcpPort = extPort - of NatPmpProtocol.UDP: - extUdpPort = extPort - return some((extTcpPort, extUdpPort)) - -proc repeatPortMapping(args: PortMappingArgs) {.thread, raises: [ValueError].} = - ignoreSignalsInThread() - let - (strategy, tcpPort, udpPort, description) = args - interval = initDuration(seconds = PORT_MAPPING_INTERVAL) - sleepDuration = 1_000 # in ms, also the maximum delay after pressing Ctrl-C - - var lastUpdate = now() - - # We can't use copies of Miniupnp and NatPmp objects in this thread, because they share - # C pointers with other instances that have already been garbage collected, so - # we use threadvars instead and initialise them again here. - if initNatDevice(strategy, quiet = true): - while natClosed.load() == false: - let currTime = now() - if currTime >= (lastUpdate + interval): - discard doPortMapping(strategy, tcpPort, udpPort, description) - lastUpdate = currTime - - sleep(sleepDuration) - -proc stopNatThreads() {.noconv.} = - # stop the thread - debug "Stopping NAT port mapping renewal threads" - try: - natClosed.store(true) - joinThreads(natThreads) - except Exception as exc: - warn "Failed to stop NAT port mapping renewal thread", exc = exc.msg - - # delete our port mappings - - # FIXME: if the initial port mapping failed because it already existed for the - # required external port, we should not delete it. It might have been set up - # by another program. - - # In Windows, a new thread is created for the signal handler, so we need to - # initialise our threadvars again. - - if initNatDevice(strategy, quiet = true): - if strategy == NatStrategy.NatUpnp: - for entry in activeMappings: - for t in [ - (entry.externalTcpPort, entry.internalTcpPort, UPNPProtocol.TCP), - (entry.externalUdpPort, entry.internalUdpPort, UPNPProtocol.UDP), - ]: - let - (eport, iport, protocol) = t - pmres = upnp.deletePortMapping(externalPort = $eport, protocol = protocol) - if pmres.isErr: - error "UPnP port mapping deletion", msg = pmres.error - else: - debug "UPnP: deleted port mapping", - externalPort = eport, internalPort = iport, protocol = protocol - elif strategy == NatStrategy.NatPmp: - for entry in activeMappings: - for t in [ - (entry.externalTcpPort, entry.internalTcpPort, NatPmpProtocol.TCP), - (entry.externalUdpPort, entry.internalUdpPort, NatPmpProtocol.UDP), - ]: - let - (eport, iport, protocol) = t - pmres = npmp.deletePortMapping( - eport = eport.cushort, iport = iport.cushort, protocol = protocol - ) - if pmres.isErr: - error "NAT-PMP port mapping deletion", msg = pmres.error - else: - debug "NAT-PMP: deleted port mapping", - externalPort = eport, internalPort = iport, protocol = protocol - -proc redirectPorts*( - strategy: NatStrategy, tcpPort, udpPort: Port, description: string -): Option[(Port, Port)] = - result = doPortMapping(strategy, tcpPort, udpPort, description) - if result.isSome: - let (externalTcpPort, externalUdpPort) = result.get() - # needed by NAT-PMP on port mapping deletion - # Port mapping works. Let's launch a thread that repeats it, in case the - # NAT-PMP lease expires or the router is rebooted and forgets all about - # these mappings. - activeMappings.add( - PortMappings( - internalTcpPort: tcpPort, - externalTcpPort: externalTcpPort, - internalUdpPort: udpPort, - externalUdpPort: externalUdpPort, - description: description, - ) - ) - try: - natThreads.add(Thread[PortMappingArgs]()) - natThreads[^1].createThread( - repeatPortMapping, (strategy, externalTcpPort, externalUdpPort, description) - ) - # atexit() in disguise - if natThreads.len == 1: - # we should register the thread termination function only once - addExitProc(stopNatThreads) - except Exception as exc: - warn "Failed to create NAT port mapping renewal thread", exc = exc.msg - -proc setupNat*( - natStrategy: NatStrategy, tcpPort, udpPort: Port, clientId: string -): Option[(Port, Port)] = - ## Setup NAT port mapping. - ## Returns the external (tcpPort, udpPort) if port mapping succeeded, none otherwise. - ## TODO: Allow for tcp or udp port mapping to be optional. - if not initNatDevice(natStrategy): - warn "UPnP/NAT-PMP not available" +method mapNatPorts*(m: NatMapper): Option[(Port, Port)] {.base, gcsafe, raises: [].} = + if m.natConfig.hasExtIp: return none((Port, Port)) - let extPorts = ( - {.gcsafe.}: - redirectPorts( - strategy, tcpPort = tcpPort, udpPort = udpPort, description = clientId - ) - ) - if extPorts.isNone: - warn "UPnP/NAT-PMP available but port forwarding failed" - extPorts + # Devices are recreated on each call: discover() costs ~200ms but only fires + # when AutoNAT reports NotReachable, which is exactly when we want a fresh scan. + let upnpRes = UpnpDevice.init() + if upnpRes.isOk: + let ports = upnpRes.value.mapPorts(m.tcpPort, m.discoveryPort) + if ports.isSome: + m.hasUpnpMapping = true + return ports -proc findReachableNodes*(bootstrapNodes: seq[SignedPeerRecord]): seq[SignedPeerRecord] = - ## Returns the list of nodes known to be directly reachable. - ## Currently returns bootstrap nodes. In the future, any network participant - ## confirmed reachable by AutoNAT could be included. - bootstrapNodes + let pmpRes = PmpDevice.init() + if pmpRes.isOk: + let ports = pmpRes.value.mapPorts(m.tcpPort, m.discoveryPort) + if ports.isSome: + return ports -proc nattedPorts*(natConfig: NatConfig, tcpPort, udpPort: Port): Option[(Port, Port)] = - if natConfig.hasExtIp: - return none((Port, Port)) - let clientId = "storage" - return setupNat(natConfig.nat, tcpPort, udpPort, clientId) + none((Port, Port)) -method mapNatPorts*(m: DefaultNatMapper): Option[(Port, Port)] {.gcsafe, raises: [].} = - nattedPorts(m.natConfig, m.tcpPort, m.discoveryPort) - -proc handleNatStatus*( +method handleNatStatus*( + m: NatMapper, networkReachability: NetworkReachability, dialBackAddr: Opt[MultiAddress], discoveryPort: Port, - mapper: NatMapper, discovery: Discovery, switch: Switch, autoRelayService: AutoRelayService, -) {.async: (raises: [CancelledError]).} = +) {.async: (raises: [CancelledError]), base, gcsafe.} = case networkReachability of Unknown: - # Nothing to do here, not enough confidence score result discard of Reachable: if dialBackAddr.isNone: @@ -337,9 +83,9 @@ proc handleNatStatus*( var hasPortMapping = false if dialBackAddr.isNone: - warn "Got empty dialback address in AutoNat when node is Reachable" + warn "Got empty dialback address in AutoNat when node is NotReachable" else: - let maybePorts = mapper.mapNatPorts() + let maybePorts = m.mapNatPorts() if maybePorts.isSome: let (tcpPort, udpPort) = maybePorts.get() @@ -352,9 +98,34 @@ proc handleNatStatus*( debug "AutoRelayService stop method returned false" discovery.updateRecords(@[announceAddress], udpPort) - hasPortMapping = true if not hasPortMapping and not autoRelayService.isRunning: if not await autoRelayService.setup(switch): debug "AutoRelayService setup method returned false" + +proc close*(m: NatMapper, device = UpnpDevice()) = + # UPnP mappings are permanent (leaseDuration=0) and must be deleted explicitly. + # NAT-PMP mappings expire automatically after NATPMP_LIFETIME seconds. + if not m.hasUpnpMapping: + return + + # deletePortMapping requires the IGD control URL set during init + let deviceRes = device.init() + if deviceRes.isErr: + warn "UPnP reinit failed during cleanup, port mappings may remain", + msg = deviceRes.error + return + + for (port, proto) in [ + (m.tcpPort, NatIpProtocol.Tcp), (m.discoveryPort, NatIpProtocol.Udp) + ]: + let res = deviceRes.value.deletePortMapping(port, proto) + if res.isErr: + error "UPnP port mapping deletion failed", port, proto, msg = res.error + +proc findReachableNodes*(bootstrapNodes: seq[SignedPeerRecord]): seq[SignedPeerRecord] = + ## Returns the list of nodes known to be directly reachable. + ## Currently returns bootstrap nodes. In the future, any network participant + ## confirmed reachable by AutoNAT could be included. + bootstrapNodes diff --git a/storage/rest/api.nim b/storage/rest/api.nim index 6c54cc8b..5047686d 100644 --- a/storage/rest/api.nim +++ b/storage/rest/api.nim @@ -561,7 +561,7 @@ proc initNodeApi(node: StorageNodeRef, conf: StorageConf, router: var RestRouter proc initDebugApi( node: StorageNodeRef, conf: StorageConf, - autonat: AutonatV2Service, + autonat: Option[AutonatV2Service], router: var RestRouter, ) = let allowedOrigin = router.allowedOrigin @@ -583,7 +583,13 @@ proc initDebugApi( "announceAddresses": node.discovery.announceAddrs, "table": table, "storage": {"version": $storageVersion, "revision": $storageRevision}, - "nat": {"reachability": $autonat.networkReachability}, + "nat": { + "reachability": + if autonat.isSome: + $autonat.get.networkReachability + else: + "unknown" + }, } # return pretty json for human readability @@ -644,7 +650,7 @@ proc initRestApi*( node: StorageNodeRef, conf: StorageConf, repoStore: RepoStore, - autonat: AutonatV2Service, + autonat: Option[AutonatV2Service], corsAllowedOrigin: ?string, ): RestRouter = var router = RestRouter.init(validate, corsAllowedOrigin) diff --git a/storage/storage.nim b/storage/storage.nim index 4dca8995..d8c78c41 100644 --- a/storage/storage.nim +++ b/storage/storage.nim @@ -53,8 +53,9 @@ type repoStore: RepoStore maintenance: BlockMaintainer taskpool: Taskpool - autonatService*: AutonatV2Service + autonatService*: Option[AutonatV2Service] autoRelayService: AutoRelayService + natMapper: NatMapper isStarted: bool StoragePrivateKey* = libp2p.PrivateKey # alias @@ -123,6 +124,8 @@ proc stop*(s: StorageServer) {.async.} = notice "Stopping Storage node" + s.natMapper.close() + var futures = @[ s.storageNode.switch.stop(), s.storageNode.stop(), @@ -190,17 +193,23 @@ proc new*( let relayClient = relayClientModule.RelayClient.new(canHop = config.relay) let autonatClient = AutonatV2Client.new(random.Rng.instance()) - let autonatService = AutonatV2Service.new( - rng = random.Rng.instance(), - client = autonatClient, - config = AutonatV2ServiceConfig.new( - scheduleInterval = Opt.some(config.natScheduleInterval), - askNewConnectedPeers = true, - numPeersToAsk = config.natNumPeersToAsk, - maxQueueSize = config.natMaxQueueSize, - minConfidence = config.natMinConfidence, - ), - ) + let autonatService = + if config.nat.hasExtIp: + none(AutonatV2Service) + else: + some( + AutonatV2Service.new( + rng = random.Rng.instance(), + client = autonatClient, + config = AutonatV2ServiceConfig.new( + scheduleInterval = Opt.some(config.natScheduleInterval), + askNewConnectedPeers = true, + numPeersToAsk = config.natNumPeersToAsk, + maxQueueSize = config.natMaxQueueSize, + minConfidence = config.natMinConfidence, + ), + ) + ) let switch = SwitchBuilder .new() @@ -216,7 +225,12 @@ proc new*( .withTcpTransport({ServerFlags.ReuseAddr, ServerFlags.TcpNoDelay}) .withAutonatV2Server() .withCircuitRelay(relayClient) - .withServices(@[Service(autonatService)]) + .withServices( + if autonatService.isSome: + @[Service(autonatService.get)] + else: + @[] + ) .build() var taskPool: Taskpool @@ -362,21 +376,24 @@ proc new*( switch.mount(network) switch.mount(manifestProto) - let natMapper = DefaultNatMapper( + let natMapper = NatMapper( natConfig: config.nat, tcpPort: config.listenPort, discoveryPort: config.discoveryPort, ) - autonatService.setStatusAndConfidenceHandler( - proc( - networkReachability: NetworkReachability, confidence: Opt[float] - ) {.async: (raises: [CancelledError]).} = - debug "AutoNAT status", reachability = networkReachability, confidence - await handleNatStatus( - networkReachability, addrs, config.discoveryPort, natMapper, discovery, switch, - autoRelayService, - ) - ) + if autonatService.isSome: + autonatService.get.setStatusAndConfidenceHandler( + proc( + networkReachability: NetworkReachability, + confidence: Opt[float], + addrs: Opt[MultiAddress], + ) {.async: (raises: [CancelledError]).} = + debug "AutoNAT status", reachability = networkReachability, confidence + await natMapper.handleNatStatus( + networkReachability, addrs, config.discoveryPort, discovery, switch, + autoRelayService, + ) + ) StorageServer( config: config, @@ -388,4 +405,5 @@ proc new*( logFile: logFile, autonatService: autonatService, autoRelayService: autoRelayService, + natMapper: natMapper, ) diff --git a/storage/utils/natutils.nim b/storage/utils/natutils.nim index 9d05dd58..7364efe6 100644 --- a/storage/utils/natutils.nim +++ b/storage/utils/natutils.nim @@ -1,6 +1,207 @@ {.push raises: [].} +import std/[options, net] +import nat_traversal/[miniupnpc, natpmp] +import pkg/chronicles +import results + +export miniupnpc, natpmp, results, options, net + +logScope: + topics = "nat" + +const UPNP_TIMEOUT* = 200 # ms +const NATPMP_LIFETIME* = 60 * 60 # seconds + type NatStrategy* = enum NatAuto NatUpnp NatPmp + +type NatIpProtocol* = enum + Tcp + Udp + +# Generic Nat device can be UPnP or PmP +type NatDevice* = ref object of RootObj + +type UpnpDevice* = ref object of NatDevice + upnp: Miniupnp + +type PmpDevice* = ref object of NatDevice + npmp: NatPmp + +# appPortMapping is specific to the type of Nat device +method addPortMapping*( + d: NatDevice, port: Port, proto: NatIpProtocol +): Result[Port, string] {.base, gcsafe.} = + return err("not implemented") + +# Creates the mapping the the router and +# returns the opened ports. +method mapPorts*( + d: NatDevice, tcpPort, udpPort: Port +): Option[(Port, Port)] {.base, gcsafe.} = + var extTcpPort, extUdpPort: Port + + for t in [(tcpPort, NatIpProtocol.Tcp), (udpPort, NatIpProtocol.Udp)]: + let (port, proto) = t + let pmres = d.addPortMapping(port, proto) + + if pmres.isErr: + error "port mapping failed", msg = pmres.error + return none((Port, Port)) + + case proto + of Tcp: + extTcpPort = pmres.value + of Udp: + extUdpPort = pmres.value + + return some((extTcpPort, extUdpPort)) + +method getSpecificPortMapping*( + d: UpnpDevice, externalPort: string, protocol: UPNPProtocol +): Result[PortMappingRes, cstring] {.base, gcsafe.} = + if d.upnp == nil: + return err(cstring("upnp not initialized")) + + d.upnp.getSpecificPortMapping(externalPort = externalPort, protocol = protocol) + +method discover*(d: UpnpDevice): Result[int, cstring] {.base, gcsafe.} = + if d.upnp == nil: + return err(cstring("upnp not initialized")) + + return d.upnp.discover() + +method selectIGD*(d: UpnpDevice): SelectIGDResult {.base, gcsafe.} = + if d.upnp == nil: + return IGDNotFound + + return d.upnp.selectIGD() + +proc init*(T: type UpnpDevice): Result[UpnpDevice, string] {.gcsafe.} = + UpnpDevice().init() + +# Init UPnP device and create miniupnp instance. +# It call "discover" to retrieve the UPnP devices on the network, +# and then "selectIGD" to select a suitable device. +proc init*(d: UpnpDevice): Result[UpnpDevice, string] {.gcsafe.} = + if d.upnp == nil: + d.upnp = newMiniupnp() + + d.upnp.discoverDelay = UPNP_TIMEOUT + + let dres = d.discover() + if dres.isErr: + debug "UPnP", msg = dres.error + return err($dres.error) + + case d.selectIGD() + of IGDNotFound: + debug "UPnP", msg = "Internet Gateway Device not found. Giving up." + return err("IGD not found") + of IGDFound: + debug "UPnP", msg = "Internet Gateway Device found." + of IGDNotConnected: + debug "UPnP", + msg = "Internet Gateway Device found but it's not connected. Trying anyway." + of NotAnIGD: + debug "UPnP", + msg = + "Some device found, but it's not recognised as an Internet Gateway Device. Trying anyway." + of IGDIpNotRoutable: + debug "UPnP", + msg = + "Internet Gateway Device found and is connected, but with a reserved or non-routable IP. Trying anyway." + + return ok(d) + +# For UPnP, the external port is the same as the application port. +# This should work for most of the case. +# We could change this by using addAnyPortMapping for IGD2 compatible routers +# if needed. +method addPortMapping*( + d: UpnpDevice, port: Port, proto: NatIpProtocol +): Result[Port, string] {.gcsafe.} = + if d.upnp == nil: + return err("upnp not initialized") + + let protocol = if proto == NatIpProtocol.Tcp: UPNPProtocol.TCP else: UPNPProtocol.UDP + let pmres = d.upnp.addPortMapping( + externalPort = $port, + protocol = protocol, + internalHost = d.upnp.lanAddr, + internalPort = $port, + desc = "logos-storage", + leaseDuration = 0, + ) + if pmres.isErr: + return err($pmres.error) + + let cres = d.getSpecificPortMapping(externalPort = $port, protocol = protocol) + if cres.isErr: + # Eventually, the check could fail on some router even if the router is successful. + # So we log a warning but we still want to continue because it is not sure it is a failure. + warn "UPnP port mapping check failed. Assuming the check itself is broken and the port mapping was done.", + msg = cres.error + + info "UPnP: added port mapping", externalPort = port, internalPort = port + + return ok(port) + +method deletePortMapping*( + d: UpnpDevice, port: Port, proto: NatIpProtocol +): Result[void, string] {.base, gcsafe.} = + if d.upnp == nil: + return err("upnp not initialized") + + let protocol = if proto == NatIpProtocol.Tcp: UPNPProtocol.TCP else: UPNPProtocol.UDP + let res = d.upnp.deletePortMapping(externalPort = $port, protocol = protocol) + if res.isErr: + return err($res.error) + + debug "UPnP: deleted port mapping", port, proto + + ok() + +proc init*(T: type PmpDevice): Result[PmpDevice, string] {.gcsafe.} = + PmpDevice().init() + +# Create a NatPmP instance. +proc init*(d: PmpDevice): Result[PmpDevice, string] {.gcsafe.} = + if d.npmp == nil: + d.npmp = newNatPmp() + + let res = d.npmp.init() + if res.isErr: + debug "NAT-PMP", msg = res.error + return err($res.error) + + return ok(d) + +# Add a port mapping on NAT-PMP device. +# The application port might not be the external port. +# The latter is returned. +method addPortMapping*( + d: PmpDevice, port: Port, proto: NatIpProtocol +): Result[Port, string] {.gcsafe.} = + if d.npmp == nil: + return err("npmp not initialized") + + let protocol = + if proto == NatIpProtocol.Tcp: NatPmpProtocol.TCP else: NatPmpProtocol.UDP + let pmres = d.npmp.addPortMapping( + eport = port.cushort, + iport = port.cushort, + protocol = protocol, + lifetime = NATPMP_LIFETIME, + ) + if pmres.isErr: + return err(pmres.error) + + let extPort = Port(pmres.value) + + info "NAT-PMP: added port mapping", externalPort = extPort, internalPort = port + + return ok(extPort) diff --git a/tests/integration/multinodes.nim b/tests/integration/multinodes.nim index 94b3ede3..4d6d6f62 100644 --- a/tests/integration/multinodes.nim +++ b/tests/integration/multinodes.nim @@ -213,7 +213,9 @@ template multinodesuite*(suiteName: string, body: untyped) = trace "Setting up test", suite = suiteName, test = currentTestName, nodeConfigs if var clients =? nodeConfigs.clients: failAndTeardownOnError "failed to start client nodes": - clients = clients.withExtIp() + # Only the first node (bootstrap) gets a known extip. Other nodes use + # nat=auto so AutoNAT can run and determine their reachability. + clients = clients.withExtIp(0) for config in clients.configs: let node = await startClientNode(config) running.add RunningNode(role: Role.Client, node: node) diff --git a/tests/storage/testnat.nim b/tests/storage/testnat.nim index f74c876f..27ff6e8e 100644 --- a/tests/storage/testnat.nim +++ b/tests/storage/testnat.nim @@ -1,5 +1,6 @@ -import std/net +import std/[net, importutils] import pkg/chronos +import ../../storage/utils/natutils import pkg/libp2p/[multiaddress, multihash, multicodec] import pkg/libp2p/protocols/connectivity/autonatv2/service except setup import pkg/libp2p/protocols/connectivity/autonatv2/types @@ -16,12 +17,52 @@ import ../../storage/rng import ../../storage/utils import ../../storage/utils/addrutils +privateAccess(NatMapper) + +type MockUpnpDevice = ref object of UpnpDevice + deletedPorts: seq[(Port, NatIpProtocol)] + +method discover*(d: MockUpnpDevice): Result[int, cstring] {.gcsafe.} = + ok(1) + +method selectIGD*(d: MockUpnpDevice): SelectIGDResult {.gcsafe.} = + IGDFound + +method deletePortMapping*( + d: MockUpnpDevice, port: Port, proto: NatIpProtocol +): Result[void, string] {.gcsafe.} = + d.deletedPorts.add((port, proto)) + ok() + type MockNatMapper = ref object of NatMapper mappedPorts: Option[(Port, Port)] -method mapNatPorts*(m: MockNatMapper): Option[(Port, Port)] {.raises: [].} = +method mapNatPorts*(m: MockNatMapper): Option[(Port, Port)] {.gcsafe, raises: [].} = m.mappedPorts +suite "NatMapper.close": + test "does nothing when no upnp mapping": + let mapper = MockNatMapper( + natConfig: NatConfig(hasExtIp: false, nat: NatAuto), + tcpPort: Port(8080), + discoveryPort: Port(8090), + ) + let device = MockUpnpDevice() + mapper.close(device) + check device.deletedPorts.len == 0 + + test "deletes tcp and udp ports when upnp mapping exists": + let mapper = MockNatMapper( + natConfig: NatConfig(hasExtIp: false, nat: NatAuto), + tcpPort: Port(8080), + discoveryPort: Port(8090), + ) + mapper.hasUpnpMapping = true + let device = MockUpnpDevice() + mapper.close(device) + check device.deletedPorts == + @[(Port(8080), NatIpProtocol.Tcp), (Port(8090), NatIpProtocol.Udp)] + suite "remapAddr": test "replaces protocol tcp with udp": let ma = MultiAddress.init("/ip4/1.2.3.4/tcp/5000").expect("valid") @@ -38,11 +79,6 @@ suite "remapAddr": let remapped = ma.remapAddr(ip = some(parseIpAddress("8.8.8.8"))) check remapped == MultiAddress.init("/ip4/8.8.8.8/tcp/5000").expect("valid") -suite "nattedPorts": - test "returns none when extIp is configured (manual setup)": - let natConfig = NatConfig(hasExtIp: true, extIp: parseIpAddress("8.8.8.8")) - check nattedPorts(natConfig, Port(5000), Port(1234)).isNone - asyncchecksuite "handleNatStatus": var sw: Switch var key: PrivateKey @@ -68,8 +104,8 @@ asyncchecksuite "handleNatStatus": let dialBack = MultiAddress.init("/ip4/1.2.3.4/tcp/8080").expect("valid") let mapper = MockNatMapper(mappedPorts: some((Port(9000), Port(9001)))) - await handleNatStatus( - NotReachable, Opt.some(dialBack), discoveryPort, mapper, disc, sw, autoRelay + await mapper.handleNatStatus( + NotReachable, Opt.some(dialBack), discoveryPort, disc, sw, autoRelay ) check disc.announceAddrs == @@ -79,17 +115,28 @@ asyncchecksuite "handleNatStatus": test "handleNatStatus starts autoRelay when NotReachable and UPnP failed": let mapper = MockNatMapper(mappedPorts: none((Port, Port))) - await handleNatStatus( - NotReachable, Opt.none(MultiAddress), discoveryPort, mapper, disc, sw, autoRelay + await mapper.handleNatStatus( + NotReachable, Opt.none(MultiAddress), discoveryPort, disc, sw, autoRelay ) check autoRelay.isRunning + test "handleNatStatus starts autoRelay when NotReachable and mapping fails": + let dialBack = MultiAddress.init("/ip4/1.2.3.4/tcp/8080").expect("valid") + let mapper = MockNatMapper(mappedPorts: none((Port, Port))) + + await mapper.handleNatStatus( + NotReachable, Opt.some(dialBack), discoveryPort, disc, sw, autoRelay + ) + + check autoRelay.isRunning + check disc.announceAddrs == newSeq[MultiAddress]() + test "handleNatStatus does not announce address when Reachable and no dialBackAddr": let mapper = MockNatMapper(mappedPorts: none((Port, Port))) - await handleNatStatus( - Reachable, Opt.none(MultiAddress), discoveryPort, mapper, disc, sw, autoRelay + await mapper.handleNatStatus( + Reachable, Opt.none(MultiAddress), discoveryPort, disc, sw, autoRelay ) check disc.announceAddrs == newSeq[MultiAddress]() @@ -100,8 +147,8 @@ asyncchecksuite "handleNatStatus": let mapper = MockNatMapper(mappedPorts: none((Port, Port))) discard await autorelayservice.setup(autoRelay, sw) - await handleNatStatus( - Reachable, Opt.some(dialBack), discoveryPort, mapper, disc, sw, autoRelay + await mapper.handleNatStatus( + Reachable, Opt.some(dialBack), discoveryPort, disc, sw, autoRelay ) check not autoRelay.isRunning diff --git a/tests/storage/testnatutils.nim b/tests/storage/testnatutils.nim index 08a6621e..9eea951a 100644 --- a/tests/storage/testnatutils.nim +++ b/tests/storage/testnatutils.nim @@ -1 +1,93 @@ -discard +import std/[options, net] +import nat_traversal/[miniupnpc, natpmp] +import pkg/results +import ../asynctest +import ../../storage/utils/natutils + +type MockUpnpDev = ref object of UpnpDevice + discoverOk: bool + igdResult: SelectIGDResult + addPortMappingOk: bool + failOnProto: Option[NatIpProtocol] + +type MockPmpDev = ref object of PmpDevice + addPortMappingOk: bool + mappedPort: Port + +method discover*(d: MockUpnpDev): Result[int, cstring] {.gcsafe.} = + if d.discoverOk: + ok(1) + else: + err(cstring("discover failed")) + +method selectIGD*(d: MockUpnpDev): SelectIGDResult {.gcsafe.} = + d.igdResult + +method addPortMapping*( + d: MockUpnpDev, port: Port, proto: NatIpProtocol +): Result[Port, string] {.gcsafe.} = + if d.failOnProto == some(proto): + err("mapping failed") + elif d.addPortMappingOk: + ok(port) + else: + err("mapping failed") + +method getSpecificPortMapping*( + d: MockUpnpDev, externalPort: string, protocol: UPNPProtocol +): Result[PortMappingRes, cstring] {.gcsafe.} = + ok(PortMappingRes()) + +method addPortMapping*( + d: MockPmpDev, port: Port, proto: NatIpProtocol +): Result[Port, string] {.gcsafe.} = + if d.addPortMappingOk: + ok(d.mappedPort) + else: + err("mapping failed") + +suite "UpnpDevice.init": + test "returns err when discover fails": + check MockUpnpDev(discoverOk: false).init().isErr + + test "returns err when IGD not found": + check MockUpnpDev(discoverOk: true, igdResult: IGDNotFound).init().isErr + + test "returns ok when IGD found": + check MockUpnpDev(discoverOk: true, igdResult: IGDFound).init().isOk + + test "returns ok when IGD not connected": + check MockUpnpDev(discoverOk: true, igdResult: IGDNotConnected).init().isOk + + test "returns ok when not an IGD": + check MockUpnpDev(discoverOk: true, igdResult: NotAnIGD).init().isOk + + test "returns ok when IP not routable": + check MockUpnpDev(discoverOk: true, igdResult: IGDIpNotRoutable).init().isOk + +suite "UpnpDevice.mapPorts": + test "returns none when addPortMapping fails": + check MockUpnpDev(addPortMappingOk: false).mapPorts(Port(8080), Port(8090)).isNone + + test "returns mapped ports": + let res = MockUpnpDev(addPortMappingOk: true).mapPorts(Port(8080), Port(8090)) + check res.isSome + check res.get() == (Port(8080), Port(8090)) + + test "returns none when tcp mapping fails": + let d = MockUpnpDev(addPortMappingOk: true, failOnProto: some(NatIpProtocol.Tcp)) + check d.mapPorts(Port(8080), Port(8090)).isNone + + test "returns none when udp mapping fails": + let d = MockUpnpDev(addPortMappingOk: true, failOnProto: some(NatIpProtocol.Udp)) + check d.mapPorts(Port(8080), Port(8090)).isNone + +suite "PmpDevice.mapPorts": + test "returns none when mapping fails": + check MockPmpDev(addPortMappingOk: false).mapPorts(Port(8080), Port(8090)).isNone + + test "returns assigned external ports": + let d = MockPmpDev(addPortMappingOk: true, mappedPort: Port(9000)) + let res = d.mapPorts(Port(8080), Port(8090)) + check res.isSome + check res.get() == (Port(9000), Port(9000)) From bffd13c1602093b0725dfb64bcef555a996fb679 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Fri, 8 May 2026 14:56:42 +0400 Subject: [PATCH 027/167] Use thread to set the nat ports --- storage/nat.nim | 83 ++++++++++++++++++++++++++++++++++----- tests/storage/testnat.nim | 4 +- 2 files changed, 76 insertions(+), 11 deletions(-) diff --git a/storage/nat.nim b/storage/nat.nim index 616c64e7..acbcbcbe 100644 --- a/storage/nat.nim +++ b/storage/nat.nim @@ -12,6 +12,7 @@ import std/[options, net] import results import pkg/chronos +import pkg/chronos/threadsync import pkg/chronicles import pkg/libp2p import pkg/libp2p/services/autorelayservice @@ -24,6 +25,8 @@ import ./discovery logScope: topics = "nat" +const NatPortMappingTimeout = 5.seconds + type NatConfig* = object case hasExtIp*: bool of true: extIp*: IpAddress @@ -35,26 +38,86 @@ type NatMapper* = ref object of RootObj discoveryPort*: Port hasUpnpMapping: bool -method mapNatPorts*(m: NatMapper): Option[(Port, Port)] {.base, gcsafe, raises: [].} = - if m.natConfig.hasExtIp: - return none((Port, Port)) +type MapNatPortsCtx = object + natConfig: NatConfig + tcpPort: Port + discoveryPort: Port + signal: ThreadSignalPtr + result: Option[(Port, Port)] + hasUpnpMapping: bool + +proc mapNatPortsThread(ctx: ptr MapNatPortsCtx) {.thread.} = + if ctx.natConfig.hasExtIp: + discard ctx.signal.fireSync() + return # Devices are recreated on each call: discover() costs ~200ms but only fires # when AutoNAT reports NotReachable, which is exactly when we want a fresh scan. let upnpRes = UpnpDevice.init() if upnpRes.isOk: - let ports = upnpRes.value.mapPorts(m.tcpPort, m.discoveryPort) + let ports = upnpRes.value.mapPorts(ctx.tcpPort, ctx.discoveryPort) if ports.isSome: - m.hasUpnpMapping = true - return ports + ctx.hasUpnpMapping = true + ctx.result = ports + discard ctx.signal.fireSync() + return let pmpRes = PmpDevice.init() if pmpRes.isOk: - let ports = pmpRes.value.mapPorts(m.tcpPort, m.discoveryPort) + let ports = pmpRes.value.mapPorts(ctx.tcpPort, ctx.discoveryPort) if ports.isSome: - return ports + ctx.result = ports - none((Port, Port)) + discard ctx.signal.fireSync() + +method mapNatPorts*( + m: NatMapper +): Future[Option[(Port, Port)]] {.async: (raises: [CancelledError]), base, gcsafe.} = + let signal = ThreadSignalPtr.new().valueOr: + warn "Failed to create ThreadSignalPtr for NAT port mapping" + return none((Port, Port)) + + var ctx = cast[ptr MapNatPortsCtx](createShared(MapNatPortsCtx)) + ctx[] = MapNatPortsCtx( + natConfig: m.natConfig, + tcpPort: m.tcpPort, + discoveryPort: m.discoveryPort, + signal: signal, + ) + + var thread: Thread[ptr MapNatPortsCtx] + var threadStarted = false + defer: + if threadStarted: + # Blocking the event loop here is acceptable: UPnP discover() is bounded + # by UPNP_TIMEOUT (200ms), so the worst-case stall is ~200ms. + joinThread(thread) + # Always sync hasUpnpMapping back, even on timeout or cancellation. + # If the thread mapped ports just after the timeout, close() will + # still clean them up on the router. + if ctx.hasUpnpMapping: + m.hasUpnpMapping = true + freeShared(ctx) + discard signal.close() + + try: + createThread(thread, mapNatPortsThread, ctx) + threadStarted = true + except ValueError, ResourceExhaustedError: + warn "Failed to create thread for NAT port mapping" + return none((Port, Port)) + + try: + if not await signal.wait().withTimeout(NatPortMappingTimeout): + warn "NAT port mapping thread timed out" + return none((Port, Port)) + except CancelledError as exc: + raise exc + except AsyncError as exc: + warn "Error waiting for NAT port mapping thread", error = exc.msg + return none((Port, Port)) + + return ctx.result method handleNatStatus*( m: NatMapper, @@ -85,7 +148,7 @@ method handleNatStatus*( if dialBackAddr.isNone: warn "Got empty dialback address in AutoNat when node is NotReachable" else: - let maybePorts = m.mapNatPorts() + let maybePorts = await m.mapNatPorts() if maybePorts.isSome: let (tcpPort, udpPort) = maybePorts.get() diff --git a/tests/storage/testnat.nim b/tests/storage/testnat.nim index 27ff6e8e..78cf07f1 100644 --- a/tests/storage/testnat.nim +++ b/tests/storage/testnat.nim @@ -37,7 +37,9 @@ method deletePortMapping*( type MockNatMapper = ref object of NatMapper mappedPorts: Option[(Port, Port)] -method mapNatPorts*(m: MockNatMapper): Option[(Port, Port)] {.gcsafe, raises: [].} = +method mapNatPorts*( + m: MockNatMapper +): Future[Option[(Port, Port)]] {.async: (raises: [CancelledError]), gcsafe.} = m.mappedPorts suite "NatMapper.close": From 3460a3b266c84064c94d15c31197af83aa008660 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Thu, 4 Jun 2026 14:36:12 +0400 Subject: [PATCH 028/167] Small fixes --- .../requests/node_debug_request.nim | 9 +++- storage/discovery.nim | 8 +-- storage/storage.nim | 12 +++-- tests/storage/testnat.nim | 50 ++++++++++--------- tests/storage/testnatutils.nim | 6 +-- 5 files changed, 50 insertions(+), 35 deletions(-) diff --git a/library/storage_thread_requests/requests/node_debug_request.nim b/library/storage_thread_requests/requests/node_debug_request.nim index fe9bd389..8c7dec7d 100644 --- a/library/storage_thread_requests/requests/node_debug_request.nim +++ b/library/storage_thread_requests/requests/node_debug_request.nim @@ -9,7 +9,6 @@ import std/[options] import chronos import chronicles import codexdht/discv5/spr -import pkg/libp2p/protocols/connectivity/autonat/service import ../../alloc import ../../../storage/conf import ../../../storage/rest/json @@ -60,7 +59,13 @@ proc getDebug( if node.discovery.dhtRecord.isSome: node.discovery.dhtRecord.get.toURI else: "", "announceAddresses": node.discovery.announceAddrs, "table": table, - "nat": {"reachability": $storage[].autonatService.networkReachability}, + "nat": { + "reachability": + if storage[].autonatService.isSome: + $storage[].autonatService.get.networkReachability + else: + "unknown" + }, } return ok($json) diff --git a/storage/discovery.nim b/storage/discovery.nim index 6e3dc473..47ee538b 100644 --- a/storage/discovery.nim +++ b/storage/discovery.nim @@ -200,6 +200,8 @@ proc updateRecords*( d.protocol.updateRecord(d.dhtRecord).expect("Should update SPR") proc updateAnnounceRecord*(d: Discovery, addrs: openArray[MultiAddress]) = + # Updates announce addresses only, not the DHT routing record. + # Relay addresses should not pollute DHT routing. d.announceAddrs = @addrs info "Updating announce record", addrs = d.announceAddrs d.providerRecord = SignedPeerRecord @@ -254,7 +256,8 @@ proc new*( key: PrivateKey, bindIp = IPv4_any(), bindPort = 0.Port, - announceAddrs: openArray[MultiAddress], + announceAddrs: openArray[MultiAddress] = [], + discoveryPort = 0.Port, bootstrapNodes: openArray[SignedPeerRecord] = [], store: Datastore = SQLiteDatastore.new(Memory).expect("Should not fail!"), tableIpLimits: TableIpLimits = DefaultTableIpLimits, @@ -266,8 +269,7 @@ proc new*( key: key, peerId: PeerId.init(key).expect("Should construct PeerId"), store: store ) - # Update with empty values to get a valid SPR - self.updateRecords(@[], Port(0)) + self.updateRecords(announceAddrs, discoveryPort) let discoveryConfig = DiscoveryConfig(tableIpLimits: tableIpLimits, bitsPerHop: DefaultBitsPerHop) diff --git a/storage/storage.nim b/storage/storage.nim index d8c78c41..302056b3 100644 --- a/storage/storage.nim +++ b/storage/storage.nim @@ -85,6 +85,9 @@ proc start*(s: StorageServer) {.async.} = if s.config.nat.hasExtIp: # extip means that we assume the IP is reachable # So we just take the first peer addr and remap it with extip to keep the port only + if s.storageNode.switch.peerInfo.addrs.len == 0: + raise + newException(StorageError, "extip is set but switch has no listen addresses") @[ s.storageNode.switch.peerInfo.addrs[0].remapAddr( ip = some(s.config.nat.extIp), port = none(Port) @@ -94,7 +97,7 @@ proc start*(s: StorageServer) {.async.} = # If extip is not set, we have 2 choices: # 1- Announce the peer addrs contains detected addresses on the machine. # 2- Wait for AutoNat - # The probleme with 1 is that you will certainly announce private addresses + # The problem with 1 is that you will certainly announce private addresses # and if you advertise a CID, you will advertise these private addresses. # TODO: DHT client mode #s.storageNode.switch.peerInfo.addrs @@ -124,7 +127,8 @@ proc stop*(s: StorageServer) {.async.} = notice "Stopping Storage node" - s.natMapper.close() + if s.natMapper != nil: + s.natMapper.close() var futures = @[ s.storageNode.switch.stop(), @@ -286,9 +290,11 @@ proc new*( discovery = Discovery.new( switch.peerInfo.privateKey, - announceAddrs = @[listenMultiAddr], + announceAddrs = @[], bindPort = config.discoveryPort, bootstrapNodes = bootstrapNodes, + discoveryPort = config.discoveryPort, + bootstrapNodes = config.bootstrapNodes, store = discoveryStore, ) diff --git a/tests/storage/testnat.nim b/tests/storage/testnat.nim index 78cf07f1..f843acf2 100644 --- a/tests/storage/testnat.nim +++ b/tests/storage/testnat.nim @@ -1,9 +1,8 @@ -import std/[net, importutils] +import std/[net, importutils, envvars] import pkg/chronos import ../../storage/utils/natutils import pkg/libp2p/[multiaddress, multihash, multicodec] -import pkg/libp2p/protocols/connectivity/autonatv2/service except setup -import pkg/libp2p/protocols/connectivity/autonatv2/types +import pkg/libp2p/protocols/connectivity/autonat/types import pkg/libp2p/protocols/connectivity/relay/client as relayClientModule import pkg/libp2p/services/autorelayservice except setup @@ -15,7 +14,6 @@ import ../../storage/nat import ../../storage/discovery import ../../storage/rng import ../../storage/utils -import ../../storage/utils/addrutils privateAccess(NatMapper) @@ -42,7 +40,7 @@ method mapNatPorts*( ): Future[Option[(Port, Port)]] {.async: (raises: [CancelledError]), gcsafe.} = m.mappedPorts -suite "NatMapper.close": +suite "NAT - NatMapper.close": test "does nothing when no upnp mapping": let mapper = MockNatMapper( natConfig: NatConfig(hasExtIp: false, nat: NatAuto), @@ -65,30 +63,15 @@ suite "NatMapper.close": check device.deletedPorts == @[(Port(8080), NatIpProtocol.Tcp), (Port(8090), NatIpProtocol.Udp)] -suite "remapAddr": - test "replaces protocol tcp with udp": - let ma = MultiAddress.init("/ip4/1.2.3.4/tcp/5000").expect("valid") - let remapped = ma.remapAddr(protocol = some("udp"), port = some(Port(9000))) - check remapped == MultiAddress.init("/ip4/1.2.3.4/udp/9000").expect("valid") - - test "replaces only port, keeping protocol": - let ma = MultiAddress.init("/ip4/1.2.3.4/tcp/5000").expect("valid") - let remapped = ma.remapAddr(port = some(Port(9000))) - check remapped == MultiAddress.init("/ip4/1.2.3.4/tcp/9000").expect("valid") - - test "replaces only ip, keeping protocol and port": - let ma = MultiAddress.init("/ip4/1.2.3.4/tcp/5000").expect("valid") - let remapped = ma.remapAddr(ip = some(parseIpAddress("8.8.8.8"))) - check remapped == MultiAddress.init("/ip4/8.8.8.8/tcp/5000").expect("valid") - -asyncchecksuite "handleNatStatus": +asyncchecksuite "NAT - handleNatStatus": var sw: Switch var key: PrivateKey var disc: Discovery - let autoRelay = - AutoRelayService.new(1, relayClientModule.RelayClient.new(), nil, Rng.instance()) + var autoRelay: AutoRelayService setup: + autoRelay = + AutoRelayService.new(1, relayClientModule.RelayClient.new(), nil, Rng.instance()) key = PrivateKey.random(Rng.instance[]).get() disc = Discovery.new(key, announceAddrs = @[]) sw = newStandardSwitch() @@ -155,3 +138,22 @@ asyncchecksuite "handleNatStatus": check not autoRelay.isRunning check disc.announceAddrs == @[dialBack] + +suite "NAT - UPnP port mapping (requires NAT_TEST_UPNP=1)": + test "mapPorts and cleanup": + if getEnv("NAT_TEST_UPNP") != "1": + skip() + + let res = UpnpDevice.init() + check res.isOk + + let device = res.value + let ports = device.mapPorts(Port(8101), Port(8090)) + check ports.isSome + + let (tcp, udp) = ports.get() + check tcp == Port(8101) + check udp == Port(8090) + + check device.deletePortMapping(Port(8101), NatIpProtocol.Tcp).isOk + check device.deletePortMapping(Port(8090), NatIpProtocol.Udp).isOk diff --git a/tests/storage/testnatutils.nim b/tests/storage/testnatutils.nim index 9eea951a..10de4d18 100644 --- a/tests/storage/testnatutils.nim +++ b/tests/storage/testnatutils.nim @@ -46,7 +46,7 @@ method addPortMapping*( else: err("mapping failed") -suite "UpnpDevice.init": +suite "NAT - UpnpDevice.init": test "returns err when discover fails": check MockUpnpDev(discoverOk: false).init().isErr @@ -65,7 +65,7 @@ suite "UpnpDevice.init": test "returns ok when IP not routable": check MockUpnpDev(discoverOk: true, igdResult: IGDIpNotRoutable).init().isOk -suite "UpnpDevice.mapPorts": +suite "NAT - UpnpDevice.mapPorts": test "returns none when addPortMapping fails": check MockUpnpDev(addPortMappingOk: false).mapPorts(Port(8080), Port(8090)).isNone @@ -82,7 +82,7 @@ suite "UpnpDevice.mapPorts": let d = MockUpnpDev(addPortMappingOk: true, failOnProto: some(NatIpProtocol.Udp)) check d.mapPorts(Port(8080), Port(8090)).isNone -suite "PmpDevice.mapPorts": +suite "NAT - PmpDevice.mapPorts": test "returns none when mapping fails": check MockPmpDev(addPortMappingOk: false).mapPorts(Port(8080), Port(8090)).isNone From 6a99e218d745fa8d9f0f91598800b85c70ffec9d Mon Sep 17 00:00:00 2001 From: Arnaud Date: Fri, 8 May 2026 17:20:04 +0400 Subject: [PATCH 029/167] Remove NatStrategy upnp and pmp --- storage/conf.nim | 6 +----- storage/utils/natutils.nim | 2 -- tests/storage/testnat.nim | 1 + 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/storage/conf.nim b/storage/conf.nim index abc5a219..71e2753b 100644 --- a/storage/conf.nim +++ b/storage/conf.nim @@ -158,7 +158,7 @@ type nat* {. desc: "Specify method to use for determining public address. " & - "Must be one of: auto, upnp, pmp, extip:.", + "Must be one of: auto, extip:.", defaultValue: defaultNatConfig(), defaultValueDesc: "auto", name: "nat" @@ -419,10 +419,6 @@ func parse*(T: type NatConfig, p: string): Result[NatConfig, string] = case p.toLowerAscii of "auto": return ok(NatConfig(hasExtIp: false, nat: NatStrategy.NatAuto)) - of "upnp": - return ok(NatConfig(hasExtIp: false, nat: NatStrategy.NatUpnp)) - of "pmp": - return ok(NatConfig(hasExtIp: false, nat: NatStrategy.NatPmp)) else: if p.startsWith("extip:"): try: diff --git a/storage/utils/natutils.nim b/storage/utils/natutils.nim index 7364efe6..45abd178 100644 --- a/storage/utils/natutils.nim +++ b/storage/utils/natutils.nim @@ -15,8 +15,6 @@ const NATPMP_LIFETIME* = 60 * 60 # seconds type NatStrategy* = enum NatAuto - NatUpnp - NatPmp type NatIpProtocol* = enum Tcp diff --git a/tests/storage/testnat.nim b/tests/storage/testnat.nim index f843acf2..ff3da3cb 100644 --- a/tests/storage/testnat.nim +++ b/tests/storage/testnat.nim @@ -143,6 +143,7 @@ suite "NAT - UPnP port mapping (requires NAT_TEST_UPNP=1)": test "mapPorts and cleanup": if getEnv("NAT_TEST_UPNP") != "1": skip() + return let res = UpnpDevice.init() check res.isOk From 67155fa2a917ec5f7c7810e9cd580e426e2f4c43 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Tue, 12 May 2026 09:59:04 +0400 Subject: [PATCH 030/167] Add debug logs --- storage/nat.nim | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/storage/nat.nim b/storage/nat.nim index acbcbcbe..7001d087 100644 --- a/storage/nat.nim +++ b/storage/nat.nim @@ -139,6 +139,8 @@ method handleNatStatus*( if autoRelayService.isRunning: if not await autoRelayService.stop(switch): debug "AutoRelayService stop method returned false" + else: + debug "AutoRelayService stopped" discovery.updateRecords(@[dialBackAddr.get], discoveryPort) # TODO: switch DHT to server mode @@ -148,10 +150,15 @@ method handleNatStatus*( if dialBackAddr.isNone: warn "Got empty dialback address in AutoNat when node is NotReachable" else: + debug "Node is not reachable trying UPnP / PMP now" + let maybePorts = await m.mapNatPorts() if maybePorts.isSome: let (tcpPort, udpPort) = maybePorts.get() + + info "Port mapping created successfully", tcpPort, udpPort + let announceAddress = dialBackAddr.get.remapAddr(port = some(tcpPort)) # TODO: Try a dial me to make sure we are reachable @@ -159,13 +166,19 @@ method handleNatStatus*( if autoRelayService.isRunning: if not await autoRelayService.stop(switch): debug "AutoRelayService stop method returned false" + else: + debug "AutoRelayService stopped" discovery.updateRecords(@[announceAddress], udpPort) hasPortMapping = true if not hasPortMapping and not autoRelayService.isRunning: + debug "No port mapping found let's start autorelay" + if not await autoRelayService.setup(switch): - debug "AutoRelayService setup method returned false" + warn "Cannot start autorelay service" + else: + debug "AutoRelayService started" proc close*(m: NatMapper, device = UpnpDevice()) = # UPnP mappings are permanent (leaseDuration=0) and must be deleted explicitly. From ebce725c0bb87a4446be152406c76102517f1a0b Mon Sep 17 00:00:00 2001 From: Arnaud Date: Tue, 12 May 2026 09:59:59 +0400 Subject: [PATCH 031/167] Add remapAddr tests --- tests/storage/testaddrutils.nim | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 tests/storage/testaddrutils.nim diff --git a/tests/storage/testaddrutils.nim b/tests/storage/testaddrutils.nim new file mode 100644 index 00000000..3444f55a --- /dev/null +++ b/tests/storage/testaddrutils.nim @@ -0,0 +1,20 @@ +import std/[net, options] +import pkg/libp2p/multiaddress +import ../asynctest +import ../../storage/utils/addrutils + +suite "addrutils - remapAddr": + test "replaces protocol tcp with udp": + let ma = MultiAddress.init("/ip4/1.2.3.4/tcp/5000").expect("valid") + let remapped = ma.remapAddr(protocol = some("udp"), port = some(Port(9000))) + check remapped == MultiAddress.init("/ip4/1.2.3.4/udp/9000").expect("valid") + + test "replaces only port, keeping protocol": + let ma = MultiAddress.init("/ip4/1.2.3.4/tcp/5000").expect("valid") + let remapped = ma.remapAddr(port = some(Port(9000))) + check remapped == MultiAddress.init("/ip4/1.2.3.4/tcp/9000").expect("valid") + + test "replaces only ip, keeping protocol and port": + let ma = MultiAddress.init("/ip4/1.2.3.4/tcp/5000").expect("valid") + let remapped = ma.remapAddr(ip = some(parseIpAddress("8.8.8.8"))) + check remapped == MultiAddress.init("/ip4/8.8.8.8/tcp/5000").expect("valid") From deeb3f9ce5662a979422227d26a29dd693a9c523 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Tue, 12 May 2026 10:03:54 +0400 Subject: [PATCH 032/167] Fix import order --- tests/storage/testnat.nim | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/storage/testnat.nim b/tests/storage/testnat.nim index ff3da3cb..e669e9af 100644 --- a/tests/storage/testnat.nim +++ b/tests/storage/testnat.nim @@ -1,15 +1,14 @@ import std/[net, importutils, envvars] import pkg/chronos -import ../../storage/utils/natutils import pkg/libp2p/[multiaddress, multihash, multicodec] import pkg/libp2p/protocols/connectivity/autonat/types import pkg/libp2p/protocols/connectivity/relay/client as relayClientModule import pkg/libp2p/services/autorelayservice except setup - import pkg/results import ./helpers import ../asynctest +import ../../storage/utils/natutils import ../../storage/nat import ../../storage/discovery import ../../storage/rng From b8dc03b4aa3df5de68ded2068dc3f7ae47604ed4 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Tue, 12 May 2026 10:05:16 +0400 Subject: [PATCH 033/167] Add nat simulation util --- storage/conf.nim | 8 ++ storage/utils/natsimulation.nim | 141 ++++++++++++++++++++++++++++ tests/storage/testnatsimulation.nim | 115 +++++++++++++++++++++++ 3 files changed, 264 insertions(+) create mode 100644 storage/utils/natsimulation.nim create mode 100644 tests/storage/testnatsimulation.nim diff --git a/storage/conf.nim b/storage/conf.nim index 71e2753b..f50bd29e 100644 --- a/storage/conf.nim +++ b/storage/conf.nim @@ -341,6 +341,14 @@ type name: "nat-max-relays" .}: int + natSimulation* {. + desc: + "Simulate NAT filtering behavior for testing: endpoint-independent, address-dependent, address-and-port-dependent", + defaultValue: string.none, + name: "nat-simulation", + hidden + .}: Option[string] + relay* {. desc: "Enable circuit relay server (hop) - use on publicly reachable nodes only", defaultValue: false, diff --git a/storage/utils/natsimulation.nim b/storage/utils/natsimulation.nim new file mode 100644 index 00000000..013e70ab --- /dev/null +++ b/storage/utils/natsimulation.nim @@ -0,0 +1,141 @@ +{.push raises: [].} + +import std/sequtils +import pkg/chronos +import pkg/results +import pkg/libp2p +import pkg/libp2p/transports/tcptransport +import pkg/libp2p/transports/transport +import pkg/libp2p/wire + +type FilteringBehavior* = enum + EndpointIndependent + AddressDependent + AddressAndPortDependent + +type NatRouter* = ref object + filtering*: FilteringBehavior + conntrack: seq[TransportAddress] + +type NatTransport* = ref object of Transport + tcp: TcpTransport + router: NatRouter + +proc fromString*(T: type FilteringBehavior, s: string): Result[FilteringBehavior, string] = + case s + of "endpoint-independent": ok(EndpointIndependent) + of "address-dependent": ok(AddressDependent) + of "address-and-port-dependent": ok(AddressAndPortDependent) + else: err("Unknown filtering behavior: " & s) + +proc new*(T: type NatRouter, filtering: FilteringBehavior): T = + T(filtering: filtering) + +proc setFiltering*(r: NatRouter, filtering: FilteringBehavior) = + r.filtering = filtering + r.conntrack = @[] + +proc allowInbound(r: NatRouter, remote: TransportAddress): bool = + case r.filtering + of EndpointIndependent: + true + of AddressDependent: + r.conntrack.anyIt( + try: + it.address == remote.address + except ValueError: + false + ) + of AddressAndPortDependent: + remote in r.conntrack + +proc new*( + T: type NatTransport, + router: NatRouter, + upgrade: Upgrade, + flags: set[ServerFlags] = {}, +): T = + let self = T(tcp: TcpTransport.new(flags, upgrade), upgrader: upgrade, router: router) + procCall Transport(self).initialize() + return self + +method start*( + self: NatTransport, addrs: seq[MultiAddress] +) {.async: (raises: [LPError, transport.TransportError, CancelledError]).} = + await self.tcp.start(addrs) + self.addrs = self.tcp.addrs + self.running = true + self.onRunning.fire() + +method stop*(self: NatTransport) {.async: (raises: []).} = + await self.tcp.stop() + self.running = false + self.onStop.fire() + +method dial*( + self: NatTransport, + hostname: string, + address: MultiAddress, + peerId: Opt[PeerId] = Opt.none(PeerId), +): Future[Connection] {.async: (raises: [transport.TransportError, CancelledError]).} = + ## establishes an outgoing TCP connection and records the remote address + ## so it can connect back to us later + let conn = await self.tcp.dial(hostname, address) + + if conn.observedAddr.isSome: + let transportAddr = initTAddress(conn.observedAddr.get) + if transportAddr.isOk: + self.router.conntrack.add(transportAddr.get) + + return conn + +proc dropAfterTimeout(conn: Connection) {.async: (raises: []).} = + # Hold the connection open long enough for the remote's dial to time out, + # then close it. This simulates a NAT that drops packets rather than RSTs + # them, which is what AutoNAT needs to detect NotReachable. + await noCancel sleepAsync(20.seconds) + await noCancel conn.close() + +method accept*( + self: NatTransport +): Future[Connection] {.async: (raises: [transport.TransportError, CancelledError]).} = + ## waits for an incoming TCP connection and applies the NAT filtering rules + while true: + let conn = await self.tcp.accept() + + if self.router.filtering == EndpointIndependent: + return conn + + if conn.observedAddr.isNone: + await conn.close() + continue + + let transportAddr = initTAddress(conn.observedAddr.get) + if transportAddr.isErr: + await conn.close() + continue + + if not self.router.allowInbound(transportAddr.get): + # Do not close immediately: let the remote's dial time out naturally, + # then clean up. Returning a fast RST would produce EDialRefused (Unknown) + # instead of EDialError (NotReachable) in AutoNAT. + asyncSpawn dropAfterTimeout(conn) + continue + + return conn + +method handles*( + self: NatTransport, address: MultiAddress +): bool {.gcsafe, raises: [].} = + ## returns true if this transport handles the given address (TCP only) + if procCall Transport(self).handles(address): + if address.protocols.isOk: + return TCP.match(address) + +proc withNatTransport*( + b: SwitchBuilder, router: NatRouter, flags: set[ServerFlags] = {} +): SwitchBuilder = + b.withTransport( + proc(config: TransportConfig): Transport = + NatTransport.new(router, config.upgr, flags) + ) diff --git a/tests/storage/testnatsimulation.nim b/tests/storage/testnatsimulation.nim new file mode 100644 index 00000000..1563446a --- /dev/null +++ b/tests/storage/testnatsimulation.nim @@ -0,0 +1,115 @@ +import pkg/chronos + +import ./helpers +import ../asynctest +import ../../storage/rng +import ../../storage/utils/natsimulation + +const flags = {ServerFlags.ReuseAddr} +const listenAddr = "/ip4/127.0.0.1/tcp/0" + +proc newSwitch(rng: Rng): Switch = + SwitchBuilder + .new() + .withRng(rng) + .withPrivateKey(PrivateKey.random(rng[]).get()) + .withAddresses(@[MultiAddress.init(listenAddr).get()]) + .withTcpTransport(flags) + .withNoise() + .withYamux() + .build() + +proc newNatSwitch(router: NatRouter, rng: Rng): Switch = + SwitchBuilder + .new() + .withRng(rng) + .withPrivateKey(PrivateKey.random(rng[]).get()) + .withAddresses(@[MultiAddress.init(listenAddr).get()]) + .withNatTransport(router, flags) + .withNoise() + .withYamux() + .build() + +asyncchecksuite "NatTransport - Endpoint-Independent Filtering": + var bootstrap, natNode: Switch + + setup: + let router = NatRouter.new(EndpointIndependent) + bootstrap = newSwitch(Rng.instance()) + natNode = newNatSwitch(router, Rng.instance()) + await bootstrap.start() + await natNode.start() + + teardown: + await bootstrap.stop() + await natNode.stop() + + test "bootstrap can connect to nat node without any prior outbound": + await bootstrap.connect(natNode.peerInfo.peerId, natNode.peerInfo.addrs) + check bootstrap.isConnected(natNode.peerInfo.peerId) + +asyncchecksuite "NatTransport - Address-Dependent Filtering": + var bootstrap, thirdNode, natNode: Switch + + setup: + let router = NatRouter.new(AddressDependent) + bootstrap = newSwitch(Rng.instance()) + thirdNode = newSwitch(Rng.instance()) + natNode = newNatSwitch(router, Rng.instance()) + await bootstrap.start() + await thirdNode.start() + await natNode.start() + + teardown: + await bootstrap.stop() + await thirdNode.stop() + await natNode.stop() + + test "bootstrap can connect to nat node with a pre-existing connection": + await natNode.connect(bootstrap.peerInfo.peerId, bootstrap.peerInfo.addrs) + check natNode.isConnected(bootstrap.peerInfo.peerId) + + await bootstrap.connect(natNode.peerInfo.peerId, natNode.peerInfo.addrs) + check bootstrap.isConnected(natNode.peerInfo.peerId) + + test "third node can connect to nat node after nat node connected to bootstrap": + await natNode.connect(bootstrap.peerInfo.peerId, bootstrap.peerInfo.addrs) + await thirdNode.connect(natNode.peerInfo.peerId, natNode.peerInfo.addrs) + check thirdNode.isConnected(natNode.peerInfo.peerId) + + test "bootstrap cannot connect to nat node without a pre-existing connection": + expect(LPError): + await bootstrap.connect(natNode.peerInfo.peerId, natNode.peerInfo.addrs) + +asyncchecksuite "NatTransport - Address-and-Port-Dependent Filtering": + var bootstrap, thirdNode, natNode: Switch + + setup: + let router = NatRouter.new(AddressAndPortDependent) + bootstrap = newSwitch(Rng.instance()) + thirdNode = newSwitch(Rng.instance()) + natNode = newNatSwitch(router, Rng.instance()) + await bootstrap.start() + await thirdNode.start() + await natNode.start() + + teardown: + await bootstrap.stop() + await thirdNode.stop() + await natNode.stop() + + test "bootstrap can connect to nat node with a pre-existing connection": + await natNode.connect(bootstrap.peerInfo.peerId, bootstrap.peerInfo.addrs) + check natNode.isConnected(bootstrap.peerInfo.peerId) + + await bootstrap.connect(natNode.peerInfo.peerId, natNode.peerInfo.addrs) + check bootstrap.isConnected(natNode.peerInfo.peerId) + + test "bootstrap cannot connect to nat node without a pre-existing connection": + expect(LPError): + await bootstrap.connect(natNode.peerInfo.peerId, natNode.peerInfo.addrs) + + test "third node cannot connect to nat node even after nat node connected to bootstrap": + await natNode.connect(bootstrap.peerInfo.peerId, bootstrap.peerInfo.addrs) + expect(LPError): + await thirdNode.connect(natNode.peerInfo.peerId, natNode.peerInfo.addrs) From e8335c14ebdfc550e2575dd91a50b46d8029e640 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Tue, 12 May 2026 10:05:58 +0400 Subject: [PATCH 034/167] Integrate nat simulation and expose auto relayse service to api --- storage/rest/api.nim | 31 ++++++++++++- storage/storage.nim | 107 +++++++++++++++++++++++++++++-------------- 2 files changed, 101 insertions(+), 37 deletions(-) diff --git a/storage/rest/api.nim b/storage/rest/api.nim index 5047686d..2079d3bc 100644 --- a/storage/rest/api.nim +++ b/storage/rest/api.nim @@ -24,6 +24,7 @@ import pkg/confutils import pkg/libp2p import pkg/libp2p/routing_record import pkg/libp2p/protocols/connectivity/autonatv2/service +import pkg/libp2p/services/autorelayservice import pkg/codexdht/discv5/spr as spr import ../logutils @@ -38,6 +39,7 @@ import ../stores/repostore import ../blockexchange import ../units import ../utils/options +import ../utils/natsimulation import ./coders import ./json @@ -562,6 +564,8 @@ proc initDebugApi( node: StorageNodeRef, conf: StorageConf, autonat: Option[AutonatV2Service], + autoRelay: Option[AutoRelayService], + natRouter: Option[NatRouter], router: var RestRouter, ) = let allowedOrigin = router.allowedOrigin @@ -588,7 +592,8 @@ proc initDebugApi( if autonat.isSome: $autonat.get.networkReachability else: - "unknown" + "unknown", + "relayRunning": autoRelay.isSome and autoRelay.get.isRunning, }, } @@ -626,6 +631,26 @@ proc initDebugApi( trace "Excepting processing request", exc = exc.msg return RestApiResponse.error(Http500, headers = headers) + router.api(MethodPost, "/api/storage/v1/debug/nat/filtering") do( + filtering: Option[string] + ) -> RestApiResponse: + var headers = buildCorsHeaders("POST", allowedOrigin) + + without natSimulation =? natRouter: + return RestApiResponse.error( + Http400, "NAT simulation not active on this node", headers = headers + ) + + without res =? filtering and filtering =? res: + return + RestApiResponse.error(Http400, "Missing filtering value", headers = headers) + + let behavior = FilteringBehavior.fromString(filtering).valueOr: + return RestApiResponse.error(Http400, "Invalid filtering value", headers = headers) + + natSimulation.setFiltering(behavior) + return RestApiResponse.response("", headers = headers) + when storage_enable_api_debug_peers: router.api(MethodGet, "/api/storage/v1/debug/peer/{peerId}") do( peerId: PeerId @@ -651,12 +676,14 @@ proc initRestApi*( conf: StorageConf, repoStore: RepoStore, autonat: Option[AutonatV2Service], + autoRelay: Option[AutoRelayService], + natRouter: Option[NatRouter], corsAllowedOrigin: ?string, ): RestRouter = var router = RestRouter.init(validate, corsAllowedOrigin) initDataApi(node, repoStore, router) initNodeApi(node, conf, router) - initDebugApi(node, conf, autonat, router) + initDebugApi(node, conf, autonat, autoRelay, natRouter, router) return router diff --git a/storage/storage.nim b/storage/storage.nim index 302056b3..d9ca82c4 100644 --- a/storage/storage.nim +++ b/storage/storage.nim @@ -19,6 +19,7 @@ import pkg/presto import pkg/libp2p import pkg/libp2p/protocols/connectivity/autonatv2/[service, client] import pkg/libp2p/protocols/connectivity/relay/client as relayClientModule +import pkg/libp2p/protocols/connectivity/relay/relay as relayModule import pkg/libp2p/services/autorelayservice import pkg/confutils import pkg/confutils/defs @@ -40,6 +41,7 @@ import ./namespaces import ./storagetypes import ./logutils import ./nat +import ./utils/natsimulation logScope: topics = "storage node" @@ -53,9 +55,11 @@ type repoStore: RepoStore maintenance: BlockMaintainer taskpool: Taskpool + # Expose to make reachability accessible from rest api autonatService*: Option[AutonatV2Service] - autoRelayService: AutoRelayService - natMapper: NatMapper + autoRelayService: Option[AutoRelayService] + natMapper: Option[NatMapper] + natRouter*: Option[NatRouter] isStarted: bool StoragePrivateKey* = libp2p.PrivateKey # alias @@ -127,8 +131,8 @@ proc stop*(s: StorageServer) {.async.} = notice "Stopping Storage node" - if s.natMapper != nil: - s.natMapper.close() + if s.natMapper.isSome: + s.natMapper.get.close() var futures = @[ s.storageNode.switch.stop(), @@ -137,6 +141,12 @@ proc stop*(s: StorageServer) {.async.} = s.maintenance.stop(), ] + if s.autoRelayService.isSome and s.autoRelayService.get.isRunning: + proc stopAutoRelay(): Future[void] {.async: (raises: []).} = + discard await noCancel s.autoRelayService.get.stop(s.storageNode.switch) + + futures.add(stopAutoRelay()) + if s.restServer != nil: futures.add(s.restServer.stop()) @@ -194,8 +204,6 @@ proc new*( ## create StorageServer including setting up datastore, repostore, etc let listenMultiAddr = getMultiAddrWithIpAndTcpPort(config.listenIp, config.listenPort) - let relayClient = relayClientModule.RelayClient.new(canHop = config.relay) - let autonatClient = AutonatV2Client.new(random.Rng.instance()) let autonatService = if config.nat.hasExtIp: @@ -215,7 +223,14 @@ proc new*( ) ) - let switch = SwitchBuilder + let relayClient = RelayClient.new() + let relay: Relay = + if config.relay: + Relay.new() + else: + relayClient + + let switchBuilder = SwitchBuilder .new() .withPrivateKey(privateKey) .withAddresses(@[listenMultiAddr], enableWildcardResolver = true) @@ -226,16 +241,29 @@ proc new*( .withMaxConnections(config.maxPeers) .withAgentVersion(config.agentString) .withSignedPeerRecord(true) - .withTcpTransport({ServerFlags.ReuseAddr, ServerFlags.TcpNoDelay}) .withAutonatV2Server() - .withCircuitRelay(relayClient) + .withCircuitRelay(relay) .withServices( if autonatService.isSome: @[Service(autonatService.get)] else: @[] ) - .build() + + var natRouter: Option[NatRouter] + let switch = + if config.natSimulation.isSome: + let filtering = FilteringBehavior.fromString(config.natSimulation.get) + .valueOr(AddressAndPortDependent) + let router = NatRouter.new(filtering) + natRouter = some(router) + switchBuilder + .withNatTransport(router, {ServerFlags.ReuseAddr, ServerFlags.TcpNoDelay}) + .build() + else: + switchBuilder + .withTcpTransport({ServerFlags.ReuseAddr, ServerFlags.TcpNoDelay}) + .build() var taskPool: Taskpool autonatClient.setup(switch) @@ -355,7 +383,18 @@ proc new*( taskPool = taskPool, ) - autoRelayService = AutoRelayService.new( + var natMapper: Option[NatMapper] + var autoRelayService: Option[AutoRelayService] + + if autonatService.isSome: + natMapper = some( + NatMapper( + natConfig: config.nat, + tcpPort: config.listenPort, + discoveryPort: config.discoveryPort, + ) + ) + let relayService = AutoRelayService.new( maxNumRelays = config.natMaxRelays, client = relayClient, onReservation = proc(addresses: seq[MultiAddress]) {.gcsafe, raises: [].} = @@ -365,29 +404,8 @@ proc new*( rng = random.Rng.instance(), ) - var restServer: RestServerRef = nil + autoRelayService = some(relayService) - if config.apiBindAddress.isSome: - restServer = RestServerRef - .new( - storageNode.initRestApi( - config, repoStore, autonatService, config.apiCorsAllowedOrigin - ), - initTAddress(config.apiBindAddress.get(), config.apiPort), - bufferSize = (1024 * 64), - maxRequestBodySize = int.high, - ) - .expect("Should create rest server!") - - switch.mount(network) - switch.mount(manifestProto) - - let natMapper = NatMapper( - natConfig: config.nat, - tcpPort: config.listenPort, - discoveryPort: config.discoveryPort, - ) - if autonatService.isSome: autonatService.get.setStatusAndConfidenceHandler( proc( networkReachability: NetworkReachability, @@ -395,12 +413,30 @@ proc new*( addrs: Opt[MultiAddress], ) {.async: (raises: [CancelledError]).} = debug "AutoNAT status", reachability = networkReachability, confidence - await natMapper.handleNatStatus( + await natMapper.get.handleNatStatus( networkReachability, addrs, config.discoveryPort, discovery, switch, - autoRelayService, + relayService, ) ) + switch.mount(network) + switch.mount(manifestProto) + + var restServer: RestServerRef = nil + + if config.apiBindAddress.isSome: + restServer = RestServerRef + .new( + storageNode.initRestApi( + config, repoStore, autonatService, autoRelayService, natRouter, + config.apiCorsAllowedOrigin, + ), + initTAddress(config.apiBindAddress.get(), config.apiPort), + bufferSize = (1024 * 64), + maxRequestBodySize = int.high, + ) + .expect("Should create rest server!") + StorageServer( config: config, storageNode: storageNode, @@ -412,4 +448,5 @@ proc new*( autonatService: autonatService, autoRelayService: autoRelayService, natMapper: natMapper, + natRouter: natRouter, ) From 0ce3139a823955f1edeacecaacacae4ca3f6c0cb Mon Sep 17 00:00:00 2001 From: Arnaud Date: Tue, 12 May 2026 10:06:23 +0400 Subject: [PATCH 035/167] Provide tests for nat --- tests/integration/1_minute/testnat.nim | 150 ++++++++++++++++++++++++- tests/integration/storageclient.nim | 20 ++++ tests/integration/storageconfig.nim | 32 ++++++ 3 files changed, 196 insertions(+), 6 deletions(-) diff --git a/tests/integration/1_minute/testnat.nim b/tests/integration/1_minute/testnat.nim index 50c0467d..4a76e31c 100644 --- a/tests/integration/1_minute/testnat.nim +++ b/tests/integration/1_minute/testnat.nim @@ -1,5 +1,6 @@ import std/json import std/options +import std/sequtils import pkg/chronos import pkg/questionable/results @@ -7,7 +8,13 @@ import ../multinodes import ../storageclient import ../storageconfig -multinodesuite "AutoNAT integration": +const + DetectionTimeout = 15_000 + RelayTimeout = 30_000 + PollInterval = 1_000 + +# Reminder: multinodesuite setup the first node as bootstrap node +multinodesuite "AutoNAT detection": let natConfig = NodeConfigs( clients: StorageConfigs .init(nodes = 2) @@ -19,14 +26,145 @@ multinodesuite "AutoNAT integration": # .withLogLevel("DEBUG") .some ) - - # Reminder: multinodesuite setup the first node as bootstrap node test "node is reachable when using bootstrap node on same network", natConfig: - let node1 = clients()[0] let node2 = clients()[1] check eventuallySafe( (await node2.client.natReachability()).get() == "Reachable", - timeout = 30_000, - pollInterval = 500, + timeout = RelayTimeout, + pollInterval = PollInterval, + ) + + check eventuallySafe( + not (await node2.client.natRelayRunning()).get(), + timeout = RelayTimeout, + pollInterval = PollInterval, + ) + + let autonatConfig = NodeConfigs( + clients: StorageConfigs + .init(nodes = 2) + .withRelay(idx = 0) + .withNatSimulation(idx = 1) + .withNatNumPeersToAsk(1) + .withNatMinConfidence(0.5) + .withNatScheduleInterval(10.seconds) + .withNatMaxQueueSize(1) + # .withLogLevel("DEBUG") + # .debug() + # .withLogFile() + .some + ) + # node2 is behind simulated NAT: AutoNAT peers try to dial back but are blocked. + test "nat node is detected as not reachable and starts relay", autonatConfig: + let node2 = clients()[1] + + check eventuallySafe( + (await node2.client.natReachability()).get() == "NotReachable", + timeout = DetectionTimeout, + pollInterval = PollInterval, + ) + + check eventuallySafe( + (await node2.client.natRelayRunning()).get(), + timeout = RelayTimeout, + pollInterval = PollInterval, + ) + + check eventuallySafe( + block: + let addrs = (await node2.client.info()).get["addrs"].getElems.mapIt(it.getStr) + addrs.anyIt("p2p-circuit" in it), + timeout = RelayTimeout, + pollInterval = PollInterval, + ) + + let transitionConfig = NodeConfigs( + clients: StorageConfigs + .init(nodes = 2) + .withRelay(idx = 0) + .withNatSimulation(idx = 1) + .withNatNumPeersToAsk(1) + .withNatMinConfidence(0.5) + .withNatScheduleInterval(5.seconds) + .withNatMaxQueueSize(1).some + ) + # node2 starts behind simulated NAT (NotReachable + relay), then NAT is lifted + # and AutoNAT detects Reachable on the next scheduled check. + test "nat node recovers to reachable when nat is lifted", transitionConfig: + let node2 = clients()[1] + + check eventuallySafe( + (await node2.client.natReachability()).get() == "NotReachable", + timeout = DetectionTimeout, + pollInterval = PollInterval, + ) + + check eventuallySafe( + (await node2.client.natRelayRunning()).get(), + timeout = RelayTimeout, + pollInterval = PollInterval, + ) + + check eventuallySafe( + block: + let addrs = (await node2.client.info()).get["addrs"].getElems.mapIt(it.getStr) + addrs.anyIt("p2p-circuit" in it), + timeout = RelayTimeout, + pollInterval = PollInterval, + ) + + check (await node2.client.setNatFiltering("endpoint-independent")).isOk + + check eventuallySafe( + (await node2.client.natReachability()).get() == "Reachable", + timeout = RelayTimeout, + pollInterval = PollInterval, + ) + + check eventuallySafe( + not (await node2.client.natRelayRunning()).get(), + timeout = RelayTimeout, + pollInterval = PollInterval, + ) + + let natToSimConfig = NodeConfigs( + clients: StorageConfigs + .init(nodes = 2) + .withRelay(idx = 0) + .withNatSimulation(idx = 1, "endpoint-independent") + .withNatNumPeersToAsk(1) + .withNatMinConfidence(0.5) + .withNatScheduleInterval(5.seconds) + .withNatMaxQueueSize(1).some + ) + # node2 starts reachable (endpoint-independent NAT sim = pass-through), + # then NAT is tightened to block dial-backs and AutoNAT detects NotReachable. + test "reachable node becomes not reachable when nat is applied", natToSimConfig: + let node2 = clients()[1] + + check eventuallySafe( + (await node2.client.natReachability()).get() == "Reachable", + timeout = RelayTimeout, + pollInterval = PollInterval, + ) + + check eventuallySafe( + not (await node2.client.natRelayRunning()).get(), + timeout = RelayTimeout, + pollInterval = PollInterval, + ) + + check (await node2.client.setNatFiltering("address-and-port-dependent")).isOk + + check eventuallySafe( + (await node2.client.natReachability()).get() == "NotReachable", + timeout = RelayTimeout, + pollInterval = PollInterval, + ) + + check eventuallySafe( + (await node2.client.natRelayRunning()).get(), + timeout = RelayTimeout, + pollInterval = PollInterval, ) diff --git a/tests/integration/storageclient.nim b/tests/integration/storageclient.nim index 62d72917..2c0ec356 100644 --- a/tests/integration/storageclient.nim +++ b/tests/integration/storageclient.nim @@ -281,3 +281,23 @@ proc natReachability*( return info.get()["nat"]["reachability"].getStr().success except KeyError as e: return failure e.msg + +proc natRelayRunning*( + client: StorageClient +): Future[?!bool] {.async: (raises: [CancelledError, HttpError]).} = + let info = await client.info() + if info.isErr: + return failure "Failed to get node info" + try: + return info.get()["nat"]["relayRunning"].getBool().success + except KeyError as e: + return failure e.msg + +proc setNatFiltering*( + client: StorageClient, filtering: string +): Future[?!void] {.async: (raises: [CancelledError, HttpError]).} = + let response = + await client.post(client.baseurl & "/debug/nat/filtering?filtering=" & filtering) + if response.status != 200: + return failure "Failed to set NAT filtering: " & $response.status + return success() diff --git a/tests/integration/storageconfig.nim b/tests/integration/storageconfig.nim index 7b218bd3..ed839b1f 100644 --- a/tests/integration/storageconfig.nim +++ b/tests/integration/storageconfig.nim @@ -338,3 +338,35 @@ proc withExtIp*( for config in startConfig.configs.mitems: config.addCliOption("--nat", "extip:" & ip) return startConfig + +proc withRelay*( + self: StorageConfigs, idx: int +): StorageConfigs {.raises: [StorageConfigError].} = + self.checkBounds idx + + var startConfig = self + startConfig.configs[idx].addCliOption("--relay") + return startConfig + +proc withRelay*(self: StorageConfigs): StorageConfigs {.raises: [StorageConfigError].} = + var startConfig = self + for config in startConfig.configs.mitems: + config.addCliOption("--relay") + return startConfig + +proc withNatSimulation*( + self: StorageConfigs, idx: int, filtering = "address-and-port-dependent" +): StorageConfigs {.raises: [StorageConfigError].} = + self.checkBounds idx + + var startConfig = self + startConfig.configs[idx].addCliOption("--nat-simulation", filtering) + return startConfig + +proc withNatSimulation*( + self: StorageConfigs, filtering = "address-and-port-dependent" +): StorageConfigs {.raises: [StorageConfigError].} = + var startConfig = self + for config in startConfig.configs.mitems: + config.addCliOption("--nat-simulation", filtering) + return startConfig From adf1b799309bee58b708370e94352ee0efcf7356 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Tue, 12 May 2026 10:25:02 +0400 Subject: [PATCH 036/167] Fix format --- storage/rest/api.nim | 3 ++- storage/storage.nim | 5 +++-- storage/utils/natsimulation.nim | 16 +++++++++++----- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/storage/rest/api.nim b/storage/rest/api.nim index 2079d3bc..9bd5ce9c 100644 --- a/storage/rest/api.nim +++ b/storage/rest/api.nim @@ -646,7 +646,8 @@ proc initDebugApi( RestApiResponse.error(Http400, "Missing filtering value", headers = headers) let behavior = FilteringBehavior.fromString(filtering).valueOr: - return RestApiResponse.error(Http400, "Invalid filtering value", headers = headers) + return + RestApiResponse.error(Http400, "Invalid filtering value", headers = headers) natSimulation.setFiltering(behavior) return RestApiResponse.response("", headers = headers) diff --git a/storage/storage.nim b/storage/storage.nim index d9ca82c4..a4c3dc65 100644 --- a/storage/storage.nim +++ b/storage/storage.nim @@ -253,8 +253,9 @@ proc new*( var natRouter: Option[NatRouter] let switch = if config.natSimulation.isSome: - let filtering = FilteringBehavior.fromString(config.natSimulation.get) - .valueOr(AddressAndPortDependent) + let filtering = FilteringBehavior.fromString(config.natSimulation.get).valueOr( + AddressAndPortDependent + ) let router = NatRouter.new(filtering) natRouter = some(router) switchBuilder diff --git a/storage/utils/natsimulation.nim b/storage/utils/natsimulation.nim index 013e70ab..bce08a2a 100644 --- a/storage/utils/natsimulation.nim +++ b/storage/utils/natsimulation.nim @@ -21,12 +21,18 @@ type NatTransport* = ref object of Transport tcp: TcpTransport router: NatRouter -proc fromString*(T: type FilteringBehavior, s: string): Result[FilteringBehavior, string] = +proc fromString*( + T: type FilteringBehavior, s: string +): Result[FilteringBehavior, string] = case s - of "endpoint-independent": ok(EndpointIndependent) - of "address-dependent": ok(AddressDependent) - of "address-and-port-dependent": ok(AddressAndPortDependent) - else: err("Unknown filtering behavior: " & s) + of "endpoint-independent": + ok(EndpointIndependent) + of "address-dependent": + ok(AddressDependent) + of "address-and-port-dependent": + ok(AddressAndPortDependent) + else: + err("Unknown filtering behavior: " & s) proc new*(T: type NatRouter, filtering: FilteringBehavior): T = T(filtering: filtering) From 9b3c79037a2a2e2030fa09c4c187685226177c81 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Tue, 12 May 2026 10:25:57 +0400 Subject: [PATCH 037/167] Update openapi info --- openapi.yaml | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/openapi.yaml b/openapi.yaml index 041127ee..f35c54b2 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -128,6 +128,22 @@ components: $ref: "#/components/schemas/PeersTable" storage: $ref: "#/components/schemas/StorageVersion" + nat: + $ref: "#/components/schemas/NatInfo" + + NatInfo: + type: object + required: + - reachability + - relayRunning + properties: + reachability: + type: string + enum: [Unknown, Reachable, NotReachable] + description: AutoNAT reachability status + relayRunning: + type: boolean + description: Whether the AutoRelay service is currently running DataList: type: object @@ -534,6 +550,29 @@ paths: "500": description: Well it was bad-bad + "/debug/nat/filtering": + post: + summary: "Set NAT simulation filtering behavior at runtime" + description: "Only available on nodes started with --nat-simulation. Used for testing NAT transitions." + tags: [Debug] + operationId: setNatFiltering + + parameters: + - in: query + name: filtering + required: true + schema: + type: string + enum: [endpoint-independent, address-dependent, address-and-port-dependent] + + responses: + "200": + description: Filtering behavior updated successfully + "400": + description: Missing or invalid filtering value, or NAT simulation not active + "500": + description: Internal error + "/debug/info": get: summary: "Gets node information" From fac21133812f369e82ecf1b0aa56d08bef8ec0ea Mon Sep 17 00:00:00 2001 From: Arnaud Date: Tue, 12 May 2026 15:06:22 +0400 Subject: [PATCH 038/167] Add spr method to compute with UDP and TCP records --- .../requests/node_debug_request.nim | 4 +-- .../requests/node_info_request.nim | 2 +- storage/discovery.nim | 34 +++++++++++++++---- storage/rest/api.nim | 9 ++--- storage/storage.nim | 8 ++--- tests/storage/helpers/nodeutils.nim | 4 +-- 6 files changed, 39 insertions(+), 22 deletions(-) diff --git a/library/storage_thread_requests/requests/node_debug_request.nim b/library/storage_thread_requests/requests/node_debug_request.nim index 8c7dec7d..15addb96 100644 --- a/library/storage_thread_requests/requests/node_debug_request.nim +++ b/library/storage_thread_requests/requests/node_debug_request.nim @@ -51,12 +51,12 @@ proc getDebug( ): Future[Result[string, string]] {.async: (raises: []).} = let node = storage[].node let table = RestRoutingTable.init(node.discovery.protocol.routingTable) + let nodeSpr = node.discovery.getSpr() let json = %*{ "id": $node.switch.peerInfo.peerId, "addrs": node.switch.peerInfo.addrs.mapIt($it), - "spr": - if node.discovery.dhtRecord.isSome: node.discovery.dhtRecord.get.toURI else: "", + "spr": if nodeSpr.isSome: nodeSpr.get.toURI else: "", "announceAddresses": node.discovery.announceAddrs, "table": table, "nat": { diff --git a/library/storage_thread_requests/requests/node_info_request.nim b/library/storage_thread_requests/requests/node_info_request.nim index 7e755a3a..567435f4 100644 --- a/library/storage_thread_requests/requests/node_info_request.nim +++ b/library/storage_thread_requests/requests/node_info_request.nim @@ -38,7 +38,7 @@ proc getRepo( proc getSpr( storage: ptr StorageServer ): Future[Result[string, string]] {.async: (raises: []).} = - let spr = storage[].node.discovery.dhtRecord + let spr = storage[].node.discovery.getSpr() if spr.isNone: return err("Failed to get SPR: no SPR record found.") diff --git a/storage/discovery.nim b/storage/discovery.nim index 47ee538b..02278af3 100644 --- a/storage/discovery.nim +++ b/storage/discovery.nim @@ -176,6 +176,27 @@ method removeProvider*( warn "Error removing provider", peerId = peerId, exc = exc.msg raiseAssert("Unexpected Exception in removeProvider") +proc getSpr*(d: Discovery): ?SignedPeerRecord = + ## Combined TCP+UDP record for bootstrap use by connecting nodes. + without providerRecord =? d.providerRecord: + return none(SignedPeerRecord) + + without dhtRecord =? d.dhtRecord: + return none(SignedPeerRecord) + + let tcpAddrs = providerRecord.data.addresses.mapIt(it.address) + let udpAddrs = dhtRecord.data.addresses.mapIt(it.address) + + SignedPeerRecord + .init(d.key, PeerRecord.init(d.peerId, tcpAddrs & udpAddrs)) + .expect("Should construct signed record").some + +proc updateSpr(d: Discovery) = + if not d.protocol.isNil: + let spr = d.getSpr() + if spr.isSome: + d.protocol.updateRecord(spr).expect("Should update SPR") + proc updateRecords*( d: Discovery, announceAddrs: openArray[MultiAddress], discoveryPort: Port ) = @@ -193,11 +214,10 @@ proc updateRecords*( .init(d.key, PeerRecord.init(d.peerId, tcpAddrs)) .expect("Should construct signed record").some d.dhtRecord = SignedPeerRecord - .init(d.key, PeerRecord.init(d.peerId, tcpAddrs & udpAddrs)) + .init(d.key, PeerRecord.init(d.peerId, udpAddrs)) .expect("Should construct signed record").some - if not d.protocol.isNil: - d.protocol.updateRecord(d.dhtRecord).expect("Should update SPR") + d.updateSpr() proc updateAnnounceRecord*(d: Discovery, addrs: openArray[MultiAddress]) = # Updates announce addresses only, not the DHT routing record. @@ -207,8 +227,8 @@ proc updateAnnounceRecord*(d: Discovery, addrs: openArray[MultiAddress]) = d.providerRecord = SignedPeerRecord .init(d.key, PeerRecord.init(d.peerId, d.announceAddrs)) .expect("Should construct signed record").some - if not d.protocol.isNil: - d.protocol.updateRecord(d.providerRecord).expect("Should update SPR") + + d.updateSpr() proc updateDhtRecord*( d: Discovery, addrs: openArray[MultiAddress] @@ -217,8 +237,8 @@ proc updateDhtRecord*( d.dhtRecord = SignedPeerRecord .init(d.key, PeerRecord.init(d.peerId, @addrs)) .expect("Should construct signed record").some - if not d.protocol.isNil: - d.protocol.updateRecord(d.dhtRecord).expect("Should update SPR") + + d.updateSpr() proc start*(d: Discovery) {.async: (raises: []).} = try: diff --git a/storage/rest/api.nim b/storage/rest/api.nim index 9bd5ce9c..a04cb326 100644 --- a/storage/rest/api.nim +++ b/storage/rest/api.nim @@ -25,10 +25,11 @@ import pkg/libp2p import pkg/libp2p/routing_record import pkg/libp2p/protocols/connectivity/autonatv2/service import pkg/libp2p/services/autorelayservice -import pkg/codexdht/discv5/spr as spr +import pkg/codexdht/discv5/spr import ../logutils import ../node +import ../discovery import ../blocktype import ../storagetypes import ../conf @@ -487,7 +488,7 @@ proc initNodeApi(node: StorageNodeRef, conf: StorageConf, router: var RestRouter var headers = buildCorsHeaders("GET", allowedOrigin) try: - without spr =? node.discovery.dhtRecord: + without spr =? node.discovery.getSpr(): return RestApiResponse.response( "", status = Http503, contentType = "application/json", headers = headers ) @@ -577,13 +578,13 @@ proc initDebugApi( try: let table = RestRoutingTable.init(node.discovery.protocol.routingTable) + let nodeSpr = node.discovery.getSpr() let json = %*{ "id": $node.switch.peerInfo.peerId, "addrs": node.switch.peerInfo.addrs.mapIt($it), "repo": $conf.dataDir, - "spr": - if node.discovery.dhtRecord.isSome: node.discovery.dhtRecord.get.toURI else: "", + "spr": if nodeSpr.isSome: nodeSpr.get.toURI else: "", "announceAddresses": node.discovery.announceAddrs, "table": table, "storage": {"version": $storageVersion, "revision": $storageRevision}, diff --git a/storage/storage.nim b/storage/storage.nim index a4c3dc65..3d51578c 100644 --- a/storage/storage.nim +++ b/storage/storage.nim @@ -98,13 +98,8 @@ proc start*(s: StorageServer) {.async.} = ) ] else: - # If extip is not set, we have 2 choices: - # 1- Announce the peer addrs contains detected addresses on the machine. - # 2- Wait for AutoNat - # The problem with 1 is that you will certainly announce private addresses - # and if you advertise a CID, you will advertise these private addresses. + # Don't announce address and wait for AutoNat # TODO: DHT client mode - #s.storageNode.switch.peerInfo.addrs @[] s.storageNode.discovery.updateRecords(announceAddrs, s.config.discoveryPort) @@ -114,6 +109,7 @@ proc start*(s: StorageServer) {.async.} = for spr in findReachableNodes(s.config.bootstrapNodes): try: let addrs = spr.data.addresses.mapIt(it.address) + echo "addrs", addrs await s.storageNode.switch.connect(spr.data.peerId, addrs) except CatchableError as e: warn "Cannot connect to bootstrap node", error = e.msg diff --git a/tests/storage/helpers/nodeutils.nim b/tests/storage/helpers/nodeutils.nim index 4a957d77..95d7a8c6 100644 --- a/tests/storage/helpers/nodeutils.nim +++ b/tests/storage/helpers/nodeutils.nim @@ -225,8 +225,8 @@ proc generateNodes*( if config.enableBootstrap: waitFor switch.peerInfo.update() blockDiscovery.updateRecords(switch.peerInfo.addrs, bindPort.Port) - if blockDiscovery.dhtRecord.isSome: - bootstrapNodes.add !blockDiscovery.dhtRecord + if blockDiscovery.getSpr().isSome: + bootstrapNodes.add !blockDiscovery.getSpr() fullNode else: From a53d93f703e271aebc42748cb62059ae6d0483f5 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Tue, 12 May 2026 15:14:01 +0400 Subject: [PATCH 039/167] Add auto relay in c binding --- .../storage_thread_requests/requests/node_debug_request.nim | 4 +++- storage/storage.nim | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/library/storage_thread_requests/requests/node_debug_request.nim b/library/storage_thread_requests/requests/node_debug_request.nim index 15addb96..2edb5e13 100644 --- a/library/storage_thread_requests/requests/node_debug_request.nim +++ b/library/storage_thread_requests/requests/node_debug_request.nim @@ -64,7 +64,9 @@ proc getDebug( if storage[].autonatService.isSome: $storage[].autonatService.get.networkReachability else: - "unknown" + "unknown", + "relayRunning": + storage[].autoRelayService.isSome and storage[].autoRelayService.get.isRunning, }, } diff --git a/storage/storage.nim b/storage/storage.nim index 3d51578c..2a28d7ee 100644 --- a/storage/storage.nim +++ b/storage/storage.nim @@ -57,7 +57,7 @@ type taskpool: Taskpool # Expose to make reachability accessible from rest api autonatService*: Option[AutonatV2Service] - autoRelayService: Option[AutoRelayService] + autoRelayService*: Option[AutoRelayService] natMapper: Option[NatMapper] natRouter*: Option[NatRouter] isStarted: bool From 48f502ec83a866ae92d3850b01c9061748cddd41 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Tue, 12 May 2026 20:43:08 +0400 Subject: [PATCH 040/167] Fix imports --- library/storage_thread_requests/requests/node_debug_request.nim | 2 ++ library/storage_thread_requests/requests/node_info_request.nim | 1 + 2 files changed, 3 insertions(+) diff --git a/library/storage_thread_requests/requests/node_debug_request.nim b/library/storage_thread_requests/requests/node_debug_request.nim index 2edb5e13..29c3a187 100644 --- a/library/storage_thread_requests/requests/node_debug_request.nim +++ b/library/storage_thread_requests/requests/node_debug_request.nim @@ -9,12 +9,14 @@ import std/[options] import chronos import chronicles import codexdht/discv5/spr +import pkg/libp2p/services/autorelayservice import ../../alloc import ../../../storage/conf import ../../../storage/rest/json import ../../../storage/node from ../../../storage/storage import StorageServer, node +import ../../../storage/discovery logScope: topics = "libstorage libstoragedebug" diff --git a/library/storage_thread_requests/requests/node_info_request.nim b/library/storage_thread_requests/requests/node_info_request.nim index 567435f4..f6c8c93c 100644 --- a/library/storage_thread_requests/requests/node_info_request.nim +++ b/library/storage_thread_requests/requests/node_info_request.nim @@ -10,6 +10,7 @@ import ../../../storage/rest/json import ../../../storage/node from ../../../storage/storage import StorageServer, config, node +import ../../../storage/discovery logScope: topics = "libstorage libstorageinfo" From f015fe31a933ad877fdebd8ae799c566a2a1bcf6 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Tue, 12 May 2026 20:43:18 +0400 Subject: [PATCH 041/167] Remove echo --- storage/storage.nim | 1 - 1 file changed, 1 deletion(-) diff --git a/storage/storage.nim b/storage/storage.nim index 2a28d7ee..9b2e3de6 100644 --- a/storage/storage.nim +++ b/storage/storage.nim @@ -109,7 +109,6 @@ proc start*(s: StorageServer) {.async.} = for spr in findReachableNodes(s.config.bootstrapNodes): try: let addrs = spr.data.addresses.mapIt(it.address) - echo "addrs", addrs await s.storageNode.switch.connect(spr.data.peerId, addrs) except CatchableError as e: warn "Cannot connect to bootstrap node", error = e.msg From 3e982b4af325e8f167d990471d7d8521095428c5 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Tue, 12 May 2026 20:44:49 +0400 Subject: [PATCH 042/167] Improve tests --- tests/integration/1_minute/testnat.nim | 125 +++++++++++++++--- .../integration/5_minutes/testnatdownload.nim | 65 +++++++++ tests/integration/multinodes.nim | 3 +- tests/integration/storageconfig.nim | 17 ++- tests/integration/twonodes.nim | 2 +- 5 files changed, 186 insertions(+), 26 deletions(-) create mode 100644 tests/integration/5_minutes/testnatdownload.nim diff --git a/tests/integration/1_minute/testnat.nim b/tests/integration/1_minute/testnat.nim index 4a76e31c..0edfeedd 100644 --- a/tests/integration/1_minute/testnat.nim +++ b/tests/integration/1_minute/testnat.nim @@ -18,13 +18,11 @@ multinodesuite "AutoNAT detection": let natConfig = NodeConfigs( clients: StorageConfigs .init(nodes = 2) + .withRelay(0) .withNatNumPeersToAsk(1) .withNatMinConfidence(0.5) .withNatScheduleInterval(10.seconds) - .withNatMaxQueueSize(1) - # .withLogFile() - # .withLogLevel("DEBUG") - .some + .withNatMaxQueueSize(1).some ) test "node is reachable when using bootstrap node on same network", natConfig: let node2 = clients()[1] @@ -41,22 +39,45 @@ multinodesuite "AutoNAT detection": pollInterval = PollInterval, ) + let endpointIndependentConfig = NodeConfigs( + clients: StorageConfigs + .init(nodes = 2) + .withRelay(0) + .withNatSimulation(idx = 1, "endpoint-independent") + .withNatNumPeersToAsk(1) + .withNatMinConfidence(0.5) + .withNatScheduleInterval(10.seconds) + .withNatMaxQueueSize(1).some + ) + # EIF = Endpoint Independent Filtering + test "node with simulated EIF nat is detected as reachable", endpointIndependentConfig: + let node2 = clients()[1] + + check eventuallySafe( + (await node2.client.natReachability()).get() == "Reachable", + timeout = RelayTimeout, + pollInterval = PollInterval, + ) + + check eventuallySafe( + not (await node2.client.natRelayRunning()).get(), + timeout = RelayTimeout, + pollInterval = PollInterval, + ) + let autonatConfig = NodeConfigs( clients: StorageConfigs .init(nodes = 2) - .withRelay(idx = 0) - .withNatSimulation(idx = 1) + .withRelay(0) + .withNatSimulation(idx = 1, "address-and-port-dependent") .withNatNumPeersToAsk(1) .withNatMinConfidence(0.5) .withNatScheduleInterval(10.seconds) - .withNatMaxQueueSize(1) - # .withLogLevel("DEBUG") - # .debug() - # .withLogFile() - .some + .withNatMaxQueueSize(1).some ) - # node2 is behind simulated NAT: AutoNAT peers try to dial back but are blocked. - test "nat node is detected as not reachable and starts relay", autonatConfig: + # APDF = Address and Port-Dependent Filtering + test "node with simulated APDF nat is detected as not reachable and starts relay", + autonatConfig: let node2 = clients()[1] check eventuallySafe( @@ -82,16 +103,17 @@ multinodesuite "AutoNAT detection": let transitionConfig = NodeConfigs( clients: StorageConfigs .init(nodes = 2) - .withRelay(idx = 0) - .withNatSimulation(idx = 1) + .withRelay(0) + .withNatSimulation(idx = 1, "address-and-port-dependent") .withNatNumPeersToAsk(1) .withNatMinConfidence(0.5) .withNatScheduleInterval(5.seconds) .withNatMaxQueueSize(1).some ) - # node2 starts behind simulated NAT (NotReachable + relay), then NAT is lifted - # and AutoNAT detects Reachable on the next scheduled check. - test "nat node recovers to reachable when nat is lifted", transitionConfig: + # APDF = Address and Port-Dependent Filtering + # EIF = Endpoint Independent Filtering + test "node with simulated APDF nat recovers to reachable and stops relay when nat switches to EIF nat", + transitionConfig: let node2 = clients()[1] check eventuallySafe( @@ -131,16 +153,17 @@ multinodesuite "AutoNAT detection": let natToSimConfig = NodeConfigs( clients: StorageConfigs .init(nodes = 2) - .withRelay(idx = 0) + .withRelay(0) .withNatSimulation(idx = 1, "endpoint-independent") .withNatNumPeersToAsk(1) .withNatMinConfidence(0.5) .withNatScheduleInterval(5.seconds) .withNatMaxQueueSize(1).some ) - # node2 starts reachable (endpoint-independent NAT sim = pass-through), - # then NAT is tightened to block dial-backs and AutoNAT detects NotReachable. - test "reachable node becomes not reachable when nat is applied", natToSimConfig: + + # APDF = Address and Port-Dependent Filtering + test "reachable node becomes not reachable and starts relay when nat switches to APDF nat", + natToSimConfig: let node2 = clients()[1] check eventuallySafe( @@ -168,3 +191,61 @@ multinodesuite "AutoNAT detection": timeout = RelayTimeout, pollInterval = PollInterval, ) + + let multiNatConfig = NodeConfigs( + clients: StorageConfigs + .init(nodes = 3) + .withRelay(0) + .withNatSimulation(idx = 1, "address-and-port-dependent") + .withNatSimulation(idx = 2, "address-and-port-dependent") + .withNatNumPeersToAsk(1) + .withNatMinConfidence(0.5) + .withNatScheduleInterval(5.seconds) + .withNatMaxQueueSize(1).some + ) + + # APDF = Address and Port-Dependent Filtering + test "two nodes with simulated APDF nat starts relay through the same relay node", + multiNatConfig: + let node2 = clients()[1] + let node3 = clients()[2] + + check eventuallySafe( + (await node2.client.natReachability()).get() == "NotReachable", + timeout = RelayTimeout, + pollInterval = PollInterval, + ) + + check eventuallySafe( + (await node3.client.natReachability()).get() == "NotReachable", + timeout = RelayTimeout, + pollInterval = PollInterval, + ) + + check eventuallySafe( + (await node2.client.natRelayRunning()).get(), + timeout = RelayTimeout, + pollInterval = PollInterval, + ) + + check eventuallySafe( + (await node3.client.natRelayRunning()).get(), + timeout = RelayTimeout, + pollInterval = PollInterval, + ) + + check eventuallySafe( + block: + let addrs = (await node2.client.info()).get["addrs"].getElems.mapIt(it.getStr) + addrs.anyIt("p2p-circuit" in it), + timeout = RelayTimeout, + pollInterval = PollInterval, + ) + + check eventuallySafe( + block: + let addrs = (await node3.client.info()).get["addrs"].getElems.mapIt(it.getStr) + addrs.anyIt("p2p-circuit" in it), + timeout = RelayTimeout, + pollInterval = PollInterval, + ) diff --git a/tests/integration/5_minutes/testnatdownload.nim b/tests/integration/5_minutes/testnatdownload.nim new file mode 100644 index 00000000..365217d2 --- /dev/null +++ b/tests/integration/5_minutes/testnatdownload.nim @@ -0,0 +1,65 @@ +import std/json +import std/sequtils +import pkg/chronos +import pkg/questionable/results + +import ../multinodes +import ../storageclient +import ../storageconfig + +const + RelayTimeout = 30_000 + PollInterval = 1_000 + +multinodesuite "NAT download": + let natDownloadConfig = NodeConfigs( + clients: StorageConfigs + .init(nodes = 3) + .withRelay(idx = 0) + .withNatSimulation(idx = 2, "address-and-port-dependent") + .withNatNumPeersToAsk(1) + .withNatMinConfidence(0.5) + .withNatScheduleInterval(5.seconds) + .withNatMaxQueueSize(1).some + ) + # APDF = Address and Port-Dependent Filtering + test "node 3 with simulated APDF downloads content from reachable seed node 2", + natDownloadConfig: + let seed = clients()[1] + let natNode = clients()[2] + + let content = "content for nat download test" + let cid = (await seed.client.upload(content)).get + + check eventuallySafe( + (await natNode.client.download(cid)).isOk, + timeout = RelayTimeout, + pollInterval = PollInterval, + ) + + check (await natNode.client.download(cid)).get == content + + # APDF = Address and Port-Dependent Filtering + test "reachable node 2 downloads content from node 3 with simulated APDF via relay", + natDownloadConfig: + let seed = clients()[1] + let natNode = clients()[2] + + check eventuallySafe( + block: + let addrs = (await natNode.client.info()).get["addrs"].getElems.mapIt(it.getStr) + addrs.anyIt("p2p-circuit" in it), + timeout = RelayTimeout, + pollInterval = PollInterval, + ) + + let content = "content seeded from nat node" + let cid = (await natNode.client.upload(content)).get + + check eventuallySafe( + (await seed.client.download(cid)).isOk, + timeout = RelayTimeout, + pollInterval = PollInterval, + ) + + check (await seed.client.download(cid)).get == content diff --git a/tests/integration/multinodes.nim b/tests/integration/multinodes.nim index 4d6d6f62..a0c70cc9 100644 --- a/tests/integration/multinodes.nim +++ b/tests/integration/multinodes.nim @@ -219,7 +219,8 @@ template multinodesuite*(suiteName: string, body: untyped) = for config in clients.configs: let node = await startClientNode(config) running.add RunningNode(role: Role.Client, node: node) - await StorageProcess(node).updateBootstrapNodes() + if config.isBootstrapNode: + await StorageProcess(node).updateBootstrapNodes() teardown: await teardownImpl() diff --git a/tests/integration/storageconfig.nim b/tests/integration/storageconfig.nim index ed839b1f..934f77d0 100644 --- a/tests/integration/storageconfig.nim +++ b/tests/integration/storageconfig.nim @@ -354,8 +354,21 @@ proc withRelay*(self: StorageConfigs): StorageConfigs {.raises: [StorageConfigEr config.addCliOption("--relay") return startConfig +# For testing, a node with extip (not behind nat) is a bootstrap node +proc isBootstrapNode*(config: StorageConfig): bool {.raises: [].} = + let opts = config.cliOptions.getOrDefault(StartUpCmd.noCmd) + + try: + if "--nat" in opts and "extip" in opts["--nat"].value: + return true + except KeyError: + warn "Failed to look at the extip config" + return false + + return false + proc withNatSimulation*( - self: StorageConfigs, idx: int, filtering = "address-and-port-dependent" + self: StorageConfigs, idx: int, filtering: string ): StorageConfigs {.raises: [StorageConfigError].} = self.checkBounds idx @@ -364,7 +377,7 @@ proc withNatSimulation*( return startConfig proc withNatSimulation*( - self: StorageConfigs, filtering = "address-and-port-dependent" + self: StorageConfigs, filtering: string ): StorageConfigs {.raises: [StorageConfigError].} = var startConfig = self for config in startConfig.configs.mitems: diff --git a/tests/integration/twonodes.nim b/tests/integration/twonodes.nim index 7fa55244..2398873a 100644 --- a/tests/integration/twonodes.nim +++ b/tests/integration/twonodes.nim @@ -12,7 +12,7 @@ export multinodes template twonodessuite*(name: string, body: untyped) = multinodesuite name: let twoNodesConfig {.inject, used.} = - NodeConfigs(clients: StorageConfigs.init(nodes = 2).some) + NodeConfigs(clients: StorageConfigs.init(nodes = 2).withExtIp(1).some) var node1 {.inject, used.}: StorageProcess var node2 {.inject, used.}: StorageProcess From 294899a391a9c1e58f506b626b51ca4f35d8ebba Mon Sep 17 00:00:00 2001 From: Arnaud Date: Wed, 13 May 2026 10:04:50 +0400 Subject: [PATCH 043/167] Improve tests checking --- tests/integration/1_minute/testnat.nim | 177 ++++-------------- .../integration/5_minutes/testnatdownload.nim | 4 +- 2 files changed, 41 insertions(+), 140 deletions(-) diff --git a/tests/integration/1_minute/testnat.nim b/tests/integration/1_minute/testnat.nim index 0edfeedd..7402eb74 100644 --- a/tests/integration/1_minute/testnat.nim +++ b/tests/integration/1_minute/testnat.nim @@ -13,6 +13,28 @@ const RelayTimeout = 30_000 PollInterval = 1_000 +proc checkNatReachability*(client: StorageClient, reachability: string) {.async.} = + check eventuallySafe( + (await client.natReachability()).get() == reachability, + timeout = RelayTimeout, + pollInterval = PollInterval, + ) + +proc checkRelayIsRunning*(client: StorageClient, isRunning: bool) {.async.} = + check eventuallySafe( + (await client.natRelayRunning()).get() == isRunning, + timeout = RelayTimeout, + pollInterval = PollInterval, + ) + + check eventuallySafe( + block: + let addrs = (await client.info()).get["addrs"].getElems.mapIt(it.getStr) + addrs.anyIt("p2p-circuit" in it) == isRunning, + timeout = RelayTimeout, + pollInterval = PollInterval, + ) + # Reminder: multinodesuite setup the first node as bootstrap node multinodesuite "AutoNAT detection": let natConfig = NodeConfigs( @@ -26,18 +48,8 @@ multinodesuite "AutoNAT detection": ) test "node is reachable when using bootstrap node on same network", natConfig: let node2 = clients()[1] - - check eventuallySafe( - (await node2.client.natReachability()).get() == "Reachable", - timeout = RelayTimeout, - pollInterval = PollInterval, - ) - - check eventuallySafe( - not (await node2.client.natRelayRunning()).get(), - timeout = RelayTimeout, - pollInterval = PollInterval, - ) + await node2.client.checkNatReachability("Reachable") + await node2.client.checkRelayIsRunning(false) let endpointIndependentConfig = NodeConfigs( clients: StorageConfigs @@ -52,18 +64,8 @@ multinodesuite "AutoNAT detection": # EIF = Endpoint Independent Filtering test "node with simulated EIF nat is detected as reachable", endpointIndependentConfig: let node2 = clients()[1] - - check eventuallySafe( - (await node2.client.natReachability()).get() == "Reachable", - timeout = RelayTimeout, - pollInterval = PollInterval, - ) - - check eventuallySafe( - not (await node2.client.natRelayRunning()).get(), - timeout = RelayTimeout, - pollInterval = PollInterval, - ) + await node2.client.checkNatReachability("Reachable") + await node2.client.checkRelayIsRunning(false) let autonatConfig = NodeConfigs( clients: StorageConfigs @@ -79,26 +81,8 @@ multinodesuite "AutoNAT detection": test "node with simulated APDF nat is detected as not reachable and starts relay", autonatConfig: let node2 = clients()[1] - - check eventuallySafe( - (await node2.client.natReachability()).get() == "NotReachable", - timeout = DetectionTimeout, - pollInterval = PollInterval, - ) - - check eventuallySafe( - (await node2.client.natRelayRunning()).get(), - timeout = RelayTimeout, - pollInterval = PollInterval, - ) - - check eventuallySafe( - block: - let addrs = (await node2.client.info()).get["addrs"].getElems.mapIt(it.getStr) - addrs.anyIt("p2p-circuit" in it), - timeout = RelayTimeout, - pollInterval = PollInterval, - ) + await node2.client.checkNatReachability("NotReachable") + await node2.client.checkRelayIsRunning(true) let transitionConfig = NodeConfigs( clients: StorageConfigs @@ -116,39 +100,13 @@ multinodesuite "AutoNAT detection": transitionConfig: let node2 = clients()[1] - check eventuallySafe( - (await node2.client.natReachability()).get() == "NotReachable", - timeout = DetectionTimeout, - pollInterval = PollInterval, - ) - - check eventuallySafe( - (await node2.client.natRelayRunning()).get(), - timeout = RelayTimeout, - pollInterval = PollInterval, - ) - - check eventuallySafe( - block: - let addrs = (await node2.client.info()).get["addrs"].getElems.mapIt(it.getStr) - addrs.anyIt("p2p-circuit" in it), - timeout = RelayTimeout, - pollInterval = PollInterval, - ) + await node2.client.checkNatReachability("NotReachable") + await node2.client.checkRelayIsRunning(true) check (await node2.client.setNatFiltering("endpoint-independent")).isOk - check eventuallySafe( - (await node2.client.natReachability()).get() == "Reachable", - timeout = RelayTimeout, - pollInterval = PollInterval, - ) - - check eventuallySafe( - not (await node2.client.natRelayRunning()).get(), - timeout = RelayTimeout, - pollInterval = PollInterval, - ) + await node2.client.checkNatReachability("Reachable") + await node2.client.checkRelayIsRunning(false) let natToSimConfig = NodeConfigs( clients: StorageConfigs @@ -160,37 +118,18 @@ multinodesuite "AutoNAT detection": .withNatScheduleInterval(5.seconds) .withNatMaxQueueSize(1).some ) - # APDF = Address and Port-Dependent Filtering test "reachable node becomes not reachable and starts relay when nat switches to APDF nat", natToSimConfig: let node2 = clients()[1] - check eventuallySafe( - (await node2.client.natReachability()).get() == "Reachable", - timeout = RelayTimeout, - pollInterval = PollInterval, - ) - - check eventuallySafe( - not (await node2.client.natRelayRunning()).get(), - timeout = RelayTimeout, - pollInterval = PollInterval, - ) + await node2.client.checkNatReachability("Reachable") + await node2.client.checkRelayIsRunning(false) check (await node2.client.setNatFiltering("address-and-port-dependent")).isOk - check eventuallySafe( - (await node2.client.natReachability()).get() == "NotReachable", - timeout = RelayTimeout, - pollInterval = PollInterval, - ) - - check eventuallySafe( - (await node2.client.natRelayRunning()).get(), - timeout = RelayTimeout, - pollInterval = PollInterval, - ) + await node2.client.checkNatReachability("NotReachable") + await node2.client.checkRelayIsRunning(true) let multiNatConfig = NodeConfigs( clients: StorageConfigs @@ -203,49 +142,13 @@ multinodesuite "AutoNAT detection": .withNatScheduleInterval(5.seconds) .withNatMaxQueueSize(1).some ) - # APDF = Address and Port-Dependent Filtering test "two nodes with simulated APDF nat starts relay through the same relay node", multiNatConfig: let node2 = clients()[1] let node3 = clients()[2] - check eventuallySafe( - (await node2.client.natReachability()).get() == "NotReachable", - timeout = RelayTimeout, - pollInterval = PollInterval, - ) - - check eventuallySafe( - (await node3.client.natReachability()).get() == "NotReachable", - timeout = RelayTimeout, - pollInterval = PollInterval, - ) - - check eventuallySafe( - (await node2.client.natRelayRunning()).get(), - timeout = RelayTimeout, - pollInterval = PollInterval, - ) - - check eventuallySafe( - (await node3.client.natRelayRunning()).get(), - timeout = RelayTimeout, - pollInterval = PollInterval, - ) - - check eventuallySafe( - block: - let addrs = (await node2.client.info()).get["addrs"].getElems.mapIt(it.getStr) - addrs.anyIt("p2p-circuit" in it), - timeout = RelayTimeout, - pollInterval = PollInterval, - ) - - check eventuallySafe( - block: - let addrs = (await node3.client.info()).get["addrs"].getElems.mapIt(it.getStr) - addrs.anyIt("p2p-circuit" in it), - timeout = RelayTimeout, - pollInterval = PollInterval, - ) + await node2.client.checkNatReachability("NotReachable") + await node3.client.checkNatReachability("NotReachable") + await node2.client.checkRelayIsRunning(true) + await node3.client.checkRelayIsRunning(true) diff --git a/tests/integration/5_minutes/testnatdownload.nim b/tests/integration/5_minutes/testnatdownload.nim index 365217d2..8407ebb5 100644 --- a/tests/integration/5_minutes/testnatdownload.nim +++ b/tests/integration/5_minutes/testnatdownload.nim @@ -46,9 +46,7 @@ multinodesuite "NAT download": let natNode = clients()[2] check eventuallySafe( - block: - let addrs = (await natNode.client.info()).get["addrs"].getElems.mapIt(it.getStr) - addrs.anyIt("p2p-circuit" in it), + (await natNode.client.natRelayRunning()).get(), timeout = RelayTimeout, pollInterval = PollInterval, ) From e5b9b73bb8642bcb1d8138029af3d8a41110f9b2 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Wed, 13 May 2026 18:04:07 +0400 Subject: [PATCH 044/167] Expose port mapping type --- .../requests/node_debug_request.nim | 9 ++++++++ storage/nat.nim | 18 ++++++++++----- storage/rest/api.nim | 12 +++++++++- storage/storage.nim | 4 ++-- tests/integration/storageclient.nim | 11 ++++++++++ tests/integration/storageconfig.nim | 8 +++++++ tests/storage/testnat.nim | 22 +------------------ 7 files changed, 54 insertions(+), 30 deletions(-) diff --git a/library/storage_thread_requests/requests/node_debug_request.nim b/library/storage_thread_requests/requests/node_debug_request.nim index 29c3a187..1925e05b 100644 --- a/library/storage_thread_requests/requests/node_debug_request.nim +++ b/library/storage_thread_requests/requests/node_debug_request.nim @@ -16,6 +16,7 @@ import ../../../storage/rest/json import ../../../storage/node from ../../../storage/storage import StorageServer, node +import ../../../storage/nat import ../../../storage/discovery logScope: @@ -69,6 +70,14 @@ proc getDebug( "unknown", "relayRunning": storage[].autoRelayService.isSome and storage[].autoRelayService.get.isRunning, + "portMapping": + if storage[].natMapper.isNone or + storage[].natMapper.get.portMappingType == NoMapping: + "none" + elif storage[].natMapper.get.portMappingType == UpnpMapping: + "upnp" + else: + "pmp", }, } diff --git a/storage/nat.nim b/storage/nat.nim index 7001d087..53b474a7 100644 --- a/storage/nat.nim +++ b/storage/nat.nim @@ -32,11 +32,16 @@ type NatConfig* = object of true: extIp*: IpAddress of false: nat*: NatStrategy +type PortMappingType* = enum + NoMapping + UpnpMapping + PmpMapping + type NatMapper* = ref object of RootObj natConfig*: NatConfig tcpPort*: Port discoveryPort*: Port - hasUpnpMapping: bool + portMappingType*: PortMappingType type MapNatPortsCtx = object natConfig: NatConfig @@ -44,7 +49,7 @@ type MapNatPortsCtx = object discoveryPort: Port signal: ThreadSignalPtr result: Option[(Port, Port)] - hasUpnpMapping: bool + portMappingType: PortMappingType proc mapNatPortsThread(ctx: ptr MapNatPortsCtx) {.thread.} = if ctx.natConfig.hasExtIp: @@ -57,7 +62,7 @@ proc mapNatPortsThread(ctx: ptr MapNatPortsCtx) {.thread.} = if upnpRes.isOk: let ports = upnpRes.value.mapPorts(ctx.tcpPort, ctx.discoveryPort) if ports.isSome: - ctx.hasUpnpMapping = true + ctx.portMappingType = UpnpMapping ctx.result = ports discard ctx.signal.fireSync() return @@ -66,6 +71,7 @@ proc mapNatPortsThread(ctx: ptr MapNatPortsCtx) {.thread.} = if pmpRes.isOk: let ports = pmpRes.value.mapPorts(ctx.tcpPort, ctx.discoveryPort) if ports.isSome: + ctx.portMappingType = PmpMapping ctx.result = ports discard ctx.signal.fireSync() @@ -95,8 +101,8 @@ method mapNatPorts*( # Always sync hasUpnpMapping back, even on timeout or cancellation. # If the thread mapped ports just after the timeout, close() will # still clean them up on the router. - if ctx.hasUpnpMapping: - m.hasUpnpMapping = true + if ctx.portMappingType != NoMapping: + m.portMappingType = ctx.portMappingType freeShared(ctx) discard signal.close() @@ -183,7 +189,7 @@ method handleNatStatus*( proc close*(m: NatMapper, device = UpnpDevice()) = # UPnP mappings are permanent (leaseDuration=0) and must be deleted explicitly. # NAT-PMP mappings expire automatically after NATPMP_LIFETIME seconds. - if not m.hasUpnpMapping: + if m.portMappingType != UpnpMapping: return # deletePortMapping requires the IGD control URL set during init diff --git a/storage/rest/api.nim b/storage/rest/api.nim index a04cb326..03e593c7 100644 --- a/storage/rest/api.nim +++ b/storage/rest/api.nim @@ -41,6 +41,7 @@ import ../blockexchange import ../units import ../utils/options import ../utils/natsimulation +import ../nat import ./coders import ./json @@ -566,6 +567,7 @@ proc initDebugApi( conf: StorageConf, autonat: Option[AutonatV2Service], autoRelay: Option[AutoRelayService], + natMapper: Option[NatMapper], natRouter: Option[NatRouter], router: var RestRouter, ) = @@ -595,6 +597,13 @@ proc initDebugApi( else: "unknown", "relayRunning": autoRelay.isSome and autoRelay.get.isRunning, + "portMapping": + if natMapper.isNone or natMapper.get.portMappingType == NoMapping: + "none" + elif natMapper.get.portMappingType == UpnpMapping: + "upnp" + else: + "pmp", }, } @@ -679,6 +688,7 @@ proc initRestApi*( repoStore: RepoStore, autonat: Option[AutonatV2Service], autoRelay: Option[AutoRelayService], + natMapper: Option[NatMapper], natRouter: Option[NatRouter], corsAllowedOrigin: ?string, ): RestRouter = @@ -686,6 +696,6 @@ proc initRestApi*( initDataApi(node, repoStore, router) initNodeApi(node, conf, router) - initDebugApi(node, conf, autonat, autoRelay, natRouter, router) + initDebugApi(node, conf, autonat, autoRelay, natMapper, natRouter, router) return router diff --git a/storage/storage.nim b/storage/storage.nim index 9b2e3de6..053e7bc2 100644 --- a/storage/storage.nim +++ b/storage/storage.nim @@ -58,7 +58,7 @@ type # Expose to make reachability accessible from rest api autonatService*: Option[AutonatV2Service] autoRelayService*: Option[AutoRelayService] - natMapper: Option[NatMapper] + natMapper*: Option[NatMapper] natRouter*: Option[NatRouter] isStarted: bool @@ -424,7 +424,7 @@ proc new*( restServer = RestServerRef .new( storageNode.initRestApi( - config, repoStore, autonatService, autoRelayService, natRouter, + config, repoStore, autonatService, autoRelayService, natMapper, natRouter, config.apiCorsAllowedOrigin, ), initTAddress(config.apiBindAddress.get(), config.apiPort), diff --git a/tests/integration/storageclient.nim b/tests/integration/storageclient.nim index 2c0ec356..b12f388a 100644 --- a/tests/integration/storageclient.nim +++ b/tests/integration/storageclient.nim @@ -293,6 +293,17 @@ proc natRelayRunning*( except KeyError as e: return failure e.msg +proc natPortMapping*( + client: StorageClient +): Future[?!string] {.async: (raises: [CancelledError, HttpError]).} = + let info = await client.info() + if info.isErr: + return failure "Failed to get node info" + try: + return info.get()["nat"]["portMapping"].getStr().success + except KeyError as e: + return failure e.msg + proc setNatFiltering*( client: StorageClient, filtering: string ): Future[?!void] {.async: (raises: [CancelledError, HttpError]).} = diff --git a/tests/integration/storageconfig.nim b/tests/integration/storageconfig.nim index 934f77d0..3508b31e 100644 --- a/tests/integration/storageconfig.nim +++ b/tests/integration/storageconfig.nim @@ -290,6 +290,14 @@ proc withListenIp*( config.addCliOption("--listen-ip", ip) return startConfig +proc withListenPort*( + self: StorageConfigs, idx: int, port: int +): StorageConfigs {.raises: [StorageConfigError].} = + self.checkBounds idx + var startConfig = self + startConfig.configs[idx].addCliOption("--listen-port", $port) + return startConfig + proc withNatNumPeersToAsk*( self: StorageConfigs, numPeersToAsk: int ): StorageConfigs {.raises: [StorageConfigError].} = diff --git a/tests/storage/testnat.nim b/tests/storage/testnat.nim index e669e9af..5aaca402 100644 --- a/tests/storage/testnat.nim +++ b/tests/storage/testnat.nim @@ -56,7 +56,7 @@ suite "NAT - NatMapper.close": tcpPort: Port(8080), discoveryPort: Port(8090), ) - mapper.hasUpnpMapping = true + mapper.portMappingType = UpnpMapping let device = MockUpnpDevice() mapper.close(device) check device.deletedPorts == @@ -137,23 +137,3 @@ asyncchecksuite "NAT - handleNatStatus": check not autoRelay.isRunning check disc.announceAddrs == @[dialBack] - -suite "NAT - UPnP port mapping (requires NAT_TEST_UPNP=1)": - test "mapPorts and cleanup": - if getEnv("NAT_TEST_UPNP") != "1": - skip() - return - - let res = UpnpDevice.init() - check res.isOk - - let device = res.value - let ports = device.mapPorts(Port(8101), Port(8090)) - check ports.isSome - - let (tcp, udp) = ports.get() - check tcp == Port(8101) - check udp == Port(8090) - - check device.deletePortMapping(Port(8101), NatIpProtocol.Tcp).isOk - check device.deletePortMapping(Port(8090), NatIpProtocol.Udp).isOk From 48abbe20fab58f820773fe9c0c3bb499dba6cd9a Mon Sep 17 00:00:00 2001 From: Arnaud Date: Wed, 13 May 2026 18:04:33 +0400 Subject: [PATCH 045/167] Add integration test for UPNP to test on real router --- tests/integration/1_minute/testnatupnp.nim | 69 ++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 tests/integration/1_minute/testnatupnp.nim diff --git a/tests/integration/1_minute/testnatupnp.nim b/tests/integration/1_minute/testnatupnp.nim new file mode 100644 index 00000000..4839797e --- /dev/null +++ b/tests/integration/1_minute/testnatupnp.nim @@ -0,0 +1,69 @@ +import std/[envvars, json, strutils, sequtils] +import pkg/chronos +import pkg/questionable/results +import nat_traversal/miniupnpc + +import ../multinodes +import ../storageclient +import ../storageconfig +import ../../../storage/utils/natutils + +from ./testnat.nim import checkNatReachability, checkRelayIsRunning + +const + DetectionTimeout = 15_000 + RelayTimeout = 30_000 + PollInterval = 1_000 + +# Requires a real UPnP router on the network (NAT_TEST_UPNP=1) +multinodesuite "AutoNAT UPnP port mapping": + let upnpConfig = NodeConfigs( + clients: StorageConfigs + .init(nodes = 2) + .withRelay(0) + .withNatSimulation(idx = 1, "address-and-port-dependent") + .withListenPort(idx = 1, 8102) + .withNatNumPeersToAsk(1) + .withNatMinConfidence(0.5) + .withNatScheduleInterval(10.seconds) + .withNatMaxQueueSize(1) + .withLogFile() + .withLogLevel(idx = 1, LogLevel.DEBUG).some + ) + + test "node behind NAT maps ports via UPnP and exposes mapping in debug info", + upnpConfig: + if getEnv("NAT_TEST_UPNP") != "1": + skip() + return + + let node2 = clients()[1] + + await node2.client.checkNatReachability("NotReachable") + + check eventuallySafe( + block: + let res = await node2.client.natPortMapping() + res.isOk and res.get == "upnp", + timeout = RelayTimeout, + pollInterval = PollInterval, + ) + + # Ideally we should find a way to test that the node is Reachable now + + await node2.client.checkRelayIsRunning(false) + + # Extract mapped TCP port from announce addresses and verify it exists on the IGD + let announceAddrs = + (await node2.client.info()).get["announceAddresses"].getElems.mapIt(it.getStr) + let tcpAddr = announceAddrs.filterIt(it.startsWith("/ip4/") and "/tcp/" in it) + check tcpAddr.len > 0 + + let mappedPort = tcpAddr[0].split("/")[4] + let device = UpnpDevice.init() + check device.isOk + check device.get.getSpecificPortMapping(mappedPort, UPNPProtocol.TCP).isOk + + await node2.stop() + + check device.get.getSpecificPortMapping(mappedPort, UPNPProtocol.TCP).isErr From 641b9f0443dde6338d7e6bcbfd33815ce9434437 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Thu, 14 May 2026 10:56:56 +0400 Subject: [PATCH 046/167] Add DHT client mode --- openapi.yaml | 9 +++ storage/nat.nim | 5 +- storage/rest/api.nim | 1 + storage/storage.nim | 6 +- tests/integration/1_minute/testnat.nim | 56 ++++++++----------- tests/integration/1_minute/testnatupnp.nim | 8 ++- .../integration/5_minutes/testnatdownload.nim | 1 - tests/integration/storageclient.nim | 11 ---- tests/storage/testnat.nim | 7 ++- 9 files changed, 52 insertions(+), 52 deletions(-) diff --git a/openapi.yaml b/openapi.yaml index f35c54b2..eed45eba 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -135,15 +135,24 @@ components: type: object required: - reachability + - clientMode - relayRunning + - portMapping properties: reachability: type: string enum: [Unknown, Reachable, NotReachable] description: AutoNAT reachability status + clientMode: + type: boolean + description: Whether the DHT is running in client mode (not added to remote routing tables) relayRunning: type: boolean description: Whether the AutoRelay service is currently running + portMapping: + type: string + enum: [none, upnp, pmp] + description: Active NAT port mapping type DataList: type: object diff --git a/storage/nat.nim b/storage/nat.nim index 53b474a7..fa9d51b4 100644 --- a/storage/nat.nim +++ b/storage/nat.nim @@ -149,10 +149,12 @@ method handleNatStatus*( debug "AutoRelayService stopped" discovery.updateRecords(@[dialBackAddr.get], discoveryPort) - # TODO: switch DHT to server mode + discovery.protocol.clientMode = false of NotReachable: var hasPortMapping = false + discovery.protocol.clientMode = true + if dialBackAddr.isNone: warn "Got empty dialback address in AutoNat when node is NotReachable" else: @@ -176,6 +178,7 @@ method handleNatStatus*( debug "AutoRelayService stopped" discovery.updateRecords(@[announceAddress], udpPort) + discovery.protocol.clientMode = false hasPortMapping = true if not hasPortMapping and not autoRelayService.isRunning: diff --git a/storage/rest/api.nim b/storage/rest/api.nim index 03e593c7..6acdfd61 100644 --- a/storage/rest/api.nim +++ b/storage/rest/api.nim @@ -596,6 +596,7 @@ proc initDebugApi( $autonat.get.networkReachability else: "unknown", + "clientMode": node.discovery.protocol.clientMode, "relayRunning": autoRelay.isSome and autoRelay.get.isRunning, "portMapping": if natMapper.isNone or natMapper.get.portMappingType == NoMapping: diff --git a/storage/storage.nim b/storage/storage.nim index 053e7bc2..d89c76a9 100644 --- a/storage/storage.nim +++ b/storage/storage.nim @@ -99,9 +99,13 @@ proc start*(s: StorageServer) {.async.} = ] else: # Don't announce address and wait for AutoNat - # TODO: DHT client mode @[] + if not s.config.nat.hasExtIp: + # Nodes with autonat start with client mode. + # It will be updated if reachable. + s.storageNode.discovery.protocol.clientMode = true + s.storageNode.discovery.updateRecords(announceAddrs, s.config.discoveryPort) await s.storageNode.start() diff --git a/tests/integration/1_minute/testnat.nim b/tests/integration/1_minute/testnat.nim index 7402eb74..795ed6b6 100644 --- a/tests/integration/1_minute/testnat.nim +++ b/tests/integration/1_minute/testnat.nim @@ -13,28 +13,25 @@ const RelayTimeout = 30_000 PollInterval = 1_000 -proc checkNatReachability*(client: StorageClient, reachability: string) {.async.} = - check eventuallySafe( - (await client.natReachability()).get() == reachability, - timeout = RelayTimeout, - pollInterval = PollInterval, - ) - -proc checkRelayIsRunning*(client: StorageClient, isRunning: bool) {.async.} = - check eventuallySafe( - (await client.natRelayRunning()).get() == isRunning, - timeout = RelayTimeout, - pollInterval = PollInterval, - ) - +proc checkNatStatus*( + client: StorageClient, reachability: string, relayRunning: bool +) {.async.} = check eventuallySafe( block: - let addrs = (await client.info()).get["addrs"].getElems.mapIt(it.getStr) - addrs.anyIt("p2p-circuit" in it) == isRunning, + let info = (await client.info()).get + let nat = info["nat"] + let addrs = info["addrs"].getElems.mapIt(it.getStr) + nat["reachability"].getStr() == reachability and + nat["clientMode"].getBool() == relayRunning and + nat["relayRunning"].getBool() == relayRunning and + addrs.anyIt("p2p-circuit" in it) == relayRunning, timeout = RelayTimeout, pollInterval = PollInterval, ) +proc checkNatStatus*(client: StorageClient, reachability: string) {.async.} = + await client.checkNatStatus(reachability, reachability == "NotReachable") + # Reminder: multinodesuite setup the first node as bootstrap node multinodesuite "AutoNAT detection": let natConfig = NodeConfigs( @@ -48,8 +45,7 @@ multinodesuite "AutoNAT detection": ) test "node is reachable when using bootstrap node on same network", natConfig: let node2 = clients()[1] - await node2.client.checkNatReachability("Reachable") - await node2.client.checkRelayIsRunning(false) + await node2.client.checkNatStatus("Reachable") let endpointIndependentConfig = NodeConfigs( clients: StorageConfigs @@ -64,8 +60,7 @@ multinodesuite "AutoNAT detection": # EIF = Endpoint Independent Filtering test "node with simulated EIF nat is detected as reachable", endpointIndependentConfig: let node2 = clients()[1] - await node2.client.checkNatReachability("Reachable") - await node2.client.checkRelayIsRunning(false) + await node2.client.checkNatStatus("Reachable") let autonatConfig = NodeConfigs( clients: StorageConfigs @@ -81,8 +76,7 @@ multinodesuite "AutoNAT detection": test "node with simulated APDF nat is detected as not reachable and starts relay", autonatConfig: let node2 = clients()[1] - await node2.client.checkNatReachability("NotReachable") - await node2.client.checkRelayIsRunning(true) + await node2.client.checkNatStatus("NotReachable") let transitionConfig = NodeConfigs( clients: StorageConfigs @@ -100,13 +94,11 @@ multinodesuite "AutoNAT detection": transitionConfig: let node2 = clients()[1] - await node2.client.checkNatReachability("NotReachable") - await node2.client.checkRelayIsRunning(true) + await node2.client.checkNatStatus("NotReachable") check (await node2.client.setNatFiltering("endpoint-independent")).isOk - await node2.client.checkNatReachability("Reachable") - await node2.client.checkRelayIsRunning(false) + await node2.client.checkNatStatus("Reachable") let natToSimConfig = NodeConfigs( clients: StorageConfigs @@ -123,13 +115,11 @@ multinodesuite "AutoNAT detection": natToSimConfig: let node2 = clients()[1] - await node2.client.checkNatReachability("Reachable") - await node2.client.checkRelayIsRunning(false) + await node2.client.checkNatStatus("Reachable") check (await node2.client.setNatFiltering("address-and-port-dependent")).isOk - await node2.client.checkNatReachability("NotReachable") - await node2.client.checkRelayIsRunning(true) + await node2.client.checkNatStatus("NotReachable") let multiNatConfig = NodeConfigs( clients: StorageConfigs @@ -148,7 +138,5 @@ multinodesuite "AutoNAT detection": let node2 = clients()[1] let node3 = clients()[2] - await node2.client.checkNatReachability("NotReachable") - await node3.client.checkNatReachability("NotReachable") - await node2.client.checkRelayIsRunning(true) - await node3.client.checkRelayIsRunning(true) + await node2.client.checkNatStatus("NotReachable") + await node3.client.checkNatStatus("NotReachable") diff --git a/tests/integration/1_minute/testnatupnp.nim b/tests/integration/1_minute/testnatupnp.nim index 4839797e..3c09c9c5 100644 --- a/tests/integration/1_minute/testnatupnp.nim +++ b/tests/integration/1_minute/testnatupnp.nim @@ -8,7 +8,7 @@ import ../storageclient import ../storageconfig import ../../../storage/utils/natutils -from ./testnat.nim import checkNatReachability, checkRelayIsRunning +from ./testnat.nim import checkNatStatus const DetectionTimeout = 15_000 @@ -39,7 +39,8 @@ multinodesuite "AutoNAT UPnP port mapping": let node2 = clients()[1] - await node2.client.checkNatReachability("NotReachable") + let isRelayRunning = false + await node2.client.checkNatStatus("NotReachable", isRelayRunning) check eventuallySafe( block: @@ -51,7 +52,8 @@ multinodesuite "AutoNAT UPnP port mapping": # Ideally we should find a way to test that the node is Reachable now - await node2.client.checkRelayIsRunning(false) + let isRelayRunning = false + await node2.client.checkNatStatus("NotReachable", isRelayRunning) # Extract mapped TCP port from announce addresses and verify it exists on the IGD let announceAddrs = diff --git a/tests/integration/5_minutes/testnatdownload.nim b/tests/integration/5_minutes/testnatdownload.nim index 8407ebb5..0216c000 100644 --- a/tests/integration/5_minutes/testnatdownload.nim +++ b/tests/integration/5_minutes/testnatdownload.nim @@ -1,5 +1,4 @@ import std/json -import std/sequtils import pkg/chronos import pkg/questionable/results diff --git a/tests/integration/storageclient.nim b/tests/integration/storageclient.nim index b12f388a..e1bf04a8 100644 --- a/tests/integration/storageclient.nim +++ b/tests/integration/storageclient.nim @@ -271,17 +271,6 @@ proc connectPeer*( let response = await client.get(url) assert response.status == 200 -proc natReachability*( - client: StorageClient -): Future[?!string] {.async: (raises: [CancelledError, HttpError]).} = - let info = await client.info() - if info.isErr: - return failure "Failed to get node info" - try: - return info.get()["nat"]["reachability"].getStr().success - except KeyError as e: - return failure e.msg - proc natRelayRunning*( client: StorageClient ): Future[?!bool] {.async: (raises: [CancelledError, HttpError]).} = diff --git a/tests/storage/testnat.nim b/tests/storage/testnat.nim index 5aaca402..888279ee 100644 --- a/tests/storage/testnat.nim +++ b/tests/storage/testnat.nim @@ -1,4 +1,4 @@ -import std/[net, importutils, envvars] +import std/[net, importutils] import pkg/chronos import pkg/libp2p/[multiaddress, multihash, multicodec] import pkg/libp2p/protocols/connectivity/autonat/types @@ -95,6 +95,7 @@ asyncchecksuite "NAT - handleNatStatus": check disc.announceAddrs == @[MultiAddress.init("/ip4/1.2.3.4/tcp/9000").expect("valid")] check not autoRelay.isRunning + check not disc.protocol.clientMode test "handleNatStatus starts autoRelay when NotReachable and UPnP failed": let mapper = MockNatMapper(mappedPorts: none((Port, Port))) @@ -104,6 +105,7 @@ asyncchecksuite "NAT - handleNatStatus": ) check autoRelay.isRunning + check disc.protocol.clientMode test "handleNatStatus starts autoRelay when NotReachable and mapping fails": let dialBack = MultiAddress.init("/ip4/1.2.3.4/tcp/8080").expect("valid") @@ -115,6 +117,7 @@ asyncchecksuite "NAT - handleNatStatus": check autoRelay.isRunning check disc.announceAddrs == newSeq[MultiAddress]() + check disc.protocol.clientMode test "handleNatStatus does not announce address when Reachable and no dialBackAddr": let mapper = MockNatMapper(mappedPorts: none((Port, Port))) @@ -125,6 +128,7 @@ asyncchecksuite "NAT - handleNatStatus": check disc.announceAddrs == newSeq[MultiAddress]() check not autoRelay.isRunning + check not disc.protocol.clientMode test "handleNatStatus stops relay and announces dialBackAddr when Reachable": let dialBack = MultiAddress.init("/ip4/1.2.3.4/tcp/8080").expect("valid") @@ -137,3 +141,4 @@ asyncchecksuite "NAT - handleNatStatus": check not autoRelay.isRunning check disc.announceAddrs == @[dialBack] + check not disc.protocol.clientMode From 084f3dfa042c24c02d97ccaba6605b43ccb12493 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Thu, 14 May 2026 12:14:41 +0400 Subject: [PATCH 047/167] Refactoring to make udp port explicit --- storage/discovery.nim | 10 ++++------ storage/nat.nim | 16 +++++++++++----- storage/storage.nim | 2 +- tests/storage/helpers/nodeutils.nim | 2 +- 4 files changed, 17 insertions(+), 13 deletions(-) diff --git a/storage/discovery.nim b/storage/discovery.nim index 02278af3..70e8d1b2 100644 --- a/storage/discovery.nim +++ b/storage/discovery.nim @@ -198,14 +198,12 @@ proc updateSpr(d: Discovery) = d.protocol.updateRecord(spr).expect("Should update SPR") proc updateRecords*( - d: Discovery, announceAddrs: openArray[MultiAddress], discoveryPort: Port + d: Discovery, announceAddrs: openArray[MultiAddress], udpPort: Port ) = - ## Update both provider and DHT records from TCP announce addresses. - ## Discovery (UDP) addresses are derived by remapping announceAddrs to UDP with discoveryPort. - ## Updates the discv5 SPR once with the full set of addresses. + # UDP addresses are derived from TCP announce addresses by remapping protocol and port. let tcpAddrs = @announceAddrs let udpAddrs = - tcpAddrs.mapIt(it.remapAddr(protocol = some("udp"), port = some(discoveryPort))) + tcpAddrs.mapIt(it.remapAddr(protocol = some("udp"), port = some(udpPort))) debug "Updating addresses", tcpAddrs, udpAddrs @@ -289,7 +287,7 @@ proc new*( key: key, peerId: PeerId.init(key).expect("Should construct PeerId"), store: store ) - self.updateRecords(announceAddrs, discoveryPort) + self.updateRecords(announceAddrs, udpPort = discoveryPort) let discoveryConfig = DiscoveryConfig(tableIpLimits: tableIpLimits, bitsPerHop: DefaultBitsPerHop) diff --git a/storage/nat.nim b/storage/nat.nim index fa9d51b4..0cf06e9c 100644 --- a/storage/nat.nim +++ b/storage/nat.nim @@ -148,7 +148,7 @@ method handleNatStatus*( else: debug "AutoRelayService stopped" - discovery.updateRecords(@[dialBackAddr.get], discoveryPort) + discovery.updateRecords(@[dialBackAddr.get], udpPort = discoveryPort) discovery.protocol.clientMode = false of NotReachable: var hasPortMapping = false @@ -160,6 +160,9 @@ method handleNatStatus*( else: debug "Node is not reachable trying UPnP / PMP now" + # Here we should check first that a mapping exists. + # If it does exist but Autonat still report as Not Reachable + # we should fallback to relay. let maybePorts = await m.mapNatPorts() if maybePorts.isSome: @@ -169,16 +172,19 @@ method handleNatStatus*( let announceAddress = dialBackAddr.get.remapAddr(port = some(tcpPort)) - # TODO: Try a dial me to make sure we are reachable - if autoRelayService.isRunning: + # Here we stop the relay because the node *should* be reachable if not await autoRelayService.stop(switch): debug "AutoRelayService stop method returned false" else: debug "AutoRelayService stopped" - discovery.updateRecords(@[announceAddress], udpPort) - discovery.protocol.clientMode = false + # Note that we update the DHT records but we don't set the client mode + # to false because we are not sure the node is reachable. + # The client mode will be updated on the next iteration of autonat. + # Trying to check manually that the node is reachable is not trivial, + # this is exactly what Autonat does. + discovery.updateRecords(@[announceAddress], udpPort = udpPort) hasPortMapping = true if not hasPortMapping and not autoRelayService.isRunning: diff --git a/storage/storage.nim b/storage/storage.nim index d89c76a9..61e8edc1 100644 --- a/storage/storage.nim +++ b/storage/storage.nim @@ -106,7 +106,7 @@ proc start*(s: StorageServer) {.async.} = # It will be updated if reachable. s.storageNode.discovery.protocol.clientMode = true - s.storageNode.discovery.updateRecords(announceAddrs, s.config.discoveryPort) + s.storageNode.discovery.updateRecords(announceAddrs, udpPort = s.config.discoveryPort) await s.storageNode.start() diff --git a/tests/storage/helpers/nodeutils.nim b/tests/storage/helpers/nodeutils.nim index 95d7a8c6..fd9e658f 100644 --- a/tests/storage/helpers/nodeutils.nim +++ b/tests/storage/helpers/nodeutils.nim @@ -224,7 +224,7 @@ proc generateNodes*( if config.enableBootstrap: waitFor switch.peerInfo.update() - blockDiscovery.updateRecords(switch.peerInfo.addrs, bindPort.Port) + blockDiscovery.updateRecords(switch.peerInfo.addrs, udpPort = bindPort.Port) if blockDiscovery.getSpr().isSome: bootstrapNodes.add !blockDiscovery.getSpr() From aa0d9e812b789e055d75aa4a3c7e727f334b15f4 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Thu, 14 May 2026 12:15:14 +0400 Subject: [PATCH 048/167] Update tests --- tests/integration/1_minute/testnat.nim | 12 +++++++++--- tests/integration/1_minute/testnatupnp.nim | 10 ++++++---- tests/storage/testnat.nim | 2 +- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/tests/integration/1_minute/testnat.nim b/tests/integration/1_minute/testnat.nim index 795ed6b6..79efee9e 100644 --- a/tests/integration/1_minute/testnat.nim +++ b/tests/integration/1_minute/testnat.nim @@ -14,7 +14,10 @@ const PollInterval = 1_000 proc checkNatStatus*( - client: StorageClient, reachability: string, relayRunning: bool + client: StorageClient, + reachability: string, + relayRunning: bool, + clientMode: bool, ) {.async.} = check eventuallySafe( block: @@ -22,7 +25,7 @@ proc checkNatStatus*( let nat = info["nat"] let addrs = info["addrs"].getElems.mapIt(it.getStr) nat["reachability"].getStr() == reachability and - nat["clientMode"].getBool() == relayRunning and + nat["clientMode"].getBool() == clientMode and nat["relayRunning"].getBool() == relayRunning and addrs.anyIt("p2p-circuit" in it) == relayRunning, timeout = RelayTimeout, @@ -30,7 +33,10 @@ proc checkNatStatus*( ) proc checkNatStatus*(client: StorageClient, reachability: string) {.async.} = - await client.checkNatStatus(reachability, reachability == "NotReachable") + let notReachable = reachability == "NotReachable" + await client.checkNatStatus( + reachability, relayRunning = notReachable, clientMode = notReachable + ) # Reminder: multinodesuite setup the first node as bootstrap node multinodesuite "AutoNAT detection": diff --git a/tests/integration/1_minute/testnatupnp.nim b/tests/integration/1_minute/testnatupnp.nim index 3c09c9c5..933e6204 100644 --- a/tests/integration/1_minute/testnatupnp.nim +++ b/tests/integration/1_minute/testnatupnp.nim @@ -39,8 +39,9 @@ multinodesuite "AutoNAT UPnP port mapping": let node2 = clients()[1] - let isRelayRunning = false - await node2.client.checkNatStatus("NotReachable", isRelayRunning) + await node2.client.checkNatStatus( + "NotReachable", relayRunning = false, clientMode = true + ) check eventuallySafe( block: @@ -52,8 +53,9 @@ multinodesuite "AutoNAT UPnP port mapping": # Ideally we should find a way to test that the node is Reachable now - let isRelayRunning = false - await node2.client.checkNatStatus("NotReachable", isRelayRunning) + await node2.client.checkNatStatus( + "NotReachable", relayRunning = false, clientMode = true + ) # Extract mapped TCP port from announce addresses and verify it exists on the IGD let announceAddrs = diff --git a/tests/storage/testnat.nim b/tests/storage/testnat.nim index 888279ee..d4c14f23 100644 --- a/tests/storage/testnat.nim +++ b/tests/storage/testnat.nim @@ -95,7 +95,7 @@ asyncchecksuite "NAT - handleNatStatus": check disc.announceAddrs == @[MultiAddress.init("/ip4/1.2.3.4/tcp/9000").expect("valid")] check not autoRelay.isRunning - check not disc.protocol.clientMode + check disc.protocol.clientMode test "handleNatStatus starts autoRelay when NotReachable and UPnP failed": let mapper = MockNatMapper(mappedPorts: none((Port, Port))) From e907299a5392ca2553835fd3e9a23b03eafc96dd Mon Sep 17 00:00:00 2001 From: Arnaud Date: Thu, 14 May 2026 12:16:50 +0400 Subject: [PATCH 049/167] Format --- tests/integration/1_minute/testnat.nim | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/integration/1_minute/testnat.nim b/tests/integration/1_minute/testnat.nim index 79efee9e..b9c7079a 100644 --- a/tests/integration/1_minute/testnat.nim +++ b/tests/integration/1_minute/testnat.nim @@ -14,10 +14,7 @@ const PollInterval = 1_000 proc checkNatStatus*( - client: StorageClient, - reachability: string, - relayRunning: bool, - clientMode: bool, + client: StorageClient, reachability: string, relayRunning: bool, clientMode: bool ) {.async.} = check eventuallySafe( block: From b99a7c9ac3fc637253610ba7af22529ff307f7bb Mon Sep 17 00:00:00 2001 From: Arnaud Date: Thu, 4 Jun 2026 14:36:59 +0400 Subject: [PATCH 050/167] Add nim-libplum --- .gitmodules | 10 ++++------ vendor/nim-libplum | 1 + vendor/nim-nat-traversal | 1 - 3 files changed, 5 insertions(+), 7 deletions(-) create mode 160000 vendor/nim-libplum delete mode 160000 vendor/nim-nat-traversal diff --git a/.gitmodules b/.gitmodules index 8538670b..d986bff6 100644 --- a/.gitmodules +++ b/.gitmodules @@ -42,7 +42,7 @@ path = vendor/asynctest url = https://github.com/status-im/asynctest.git ignore = untracked - branch = main + branch = main [submodule "vendor/nim-presto"] path = vendor/nim-presto url = https://github.com/status-im/nim-presto.git @@ -53,11 +53,6 @@ url = https://github.com/status-im/nim-confutils.git ignore = untracked branch = master -[submodule "vendor/nim-nat-traversal"] - path = vendor/nim-nat-traversal - url = https://github.com/status-im/nim-nat-traversal.git - ignore = untracked - branch = master [submodule "vendor/nim-libbacktrace"] path = vendor/nim-libbacktrace url = https://github.com/status-im/nim-libbacktrace.git @@ -196,3 +191,6 @@ url = https://github.com/vacp2p/nim-lsquic.git ignore = untracked branch = main +[submodule "vendor/nim-libplum"] + path = vendor/nim-libplum + url = git@github.com:2-towns/nim-libplum.git diff --git a/vendor/nim-libplum b/vendor/nim-libplum new file mode 160000 index 00000000..269d9754 --- /dev/null +++ b/vendor/nim-libplum @@ -0,0 +1 @@ +Subproject commit 269d97548c9bb9350314883aa1211e717f22f1f2 diff --git a/vendor/nim-nat-traversal b/vendor/nim-nat-traversal deleted file mode 160000 index 860e18c3..00000000 --- a/vendor/nim-nat-traversal +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 860e18c37667b5dd005b94c63264560c35d88004 From 4c9bfc966dabcf068aea9012450c356d9e7fd946 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Thu, 4 Jun 2026 14:37:13 +0400 Subject: [PATCH 051/167] Update nim-libplum --- vendor/nim-libplum | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/nim-libplum b/vendor/nim-libplum index 269d9754..bf0ace8d 160000 --- a/vendor/nim-libplum +++ b/vendor/nim-libplum @@ -1 +1 @@ -Subproject commit 269d97548c9bb9350314883aa1211e717f22f1f2 +Subproject commit bf0ace8da2715b6aed1e1ed9f33614c8e3b83893 From b274629d69e90c450800b978175dba87944574e6 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Tue, 19 May 2026 14:59:23 +0400 Subject: [PATCH 052/167] Add integration with libplum --- .dockerignore | 1 - Makefile | 13 +- build.nims | 15 ++ .../requests/node_debug_request.nim | 15 +- storage/conf.nim | 18 ++ storage/nat.nim | 190 ++++++++--------- storage/rest/api.nim | 18 +- storage/storage.nim | 3 + storage/utils/natutils.nim | 196 +----------------- tests/integration/1_minute/testnat.nim | 30 +-- tests/integration/1_minute/testnatupnp.nim | 26 +-- tests/integration/nat/docker-entrypoint.sh | 28 +++ tests/integration/nat/miniupnpd_stub_rdr.c | 169 +++++++++++++++ tests/integration/nathelper.nim | 34 +++ tests/integration/storageconfig.nim | 24 +++ tests/storage/testnat.nim | 59 +----- tests/storage/testnatutils.nim | 93 --------- 17 files changed, 416 insertions(+), 516 deletions(-) create mode 100644 tests/integration/nat/docker-entrypoint.sh create mode 100644 tests/integration/nat/miniupnpd_stub_rdr.c create mode 100644 tests/integration/nathelper.nim delete mode 100644 tests/storage/testnatutils.nim diff --git a/.dockerignore b/.dockerignore index 6427da1d..e201c832 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,4 +3,3 @@ build docs metrics nimcache -tests diff --git a/Makefile b/Makefile index f7945253..2a1a05a3 100644 --- a/Makefile +++ b/Makefile @@ -82,10 +82,12 @@ endif coverage \ deps \ libbacktrace \ + libplum \ test \ testAll \ testIntegration \ testLibstorage \ + testNatIntegration \ update ifeq ($(NIM_PARAMS),) @@ -120,11 +122,14 @@ else NIM_PARAMS := $(NIM_PARAMS) -d:release endif -deps: | deps-common nat-libs +deps: | deps-common libplum ifneq ($(USE_LIBBACKTRACE), 0) deps: | libbacktrace endif +libplum: + + "$(MAKE)" -C vendor/nim-libplum/vendor/libplum libplum.a CC=$(CC) $(HANDLE_OUTPUT) + update: | update-common # detecting the os @@ -147,6 +152,12 @@ testIntegration: | build deps echo -e $(BUILD_MSG) "build/$@" && \ $(ENV_SCRIPT) nim testIntegration $(TEST_PARAMS) $(NIM_PARAMS) build.nims +# Builds and runs the UPnP NAT integration test inside a miniupnpd container +DOCKER := $(or $(shell which podman 2>/dev/null), $(shell which docker 2>/dev/null)) +testNatIntegration: + $(DOCKER) build -t miniupnpd-test -f tests/integration/nat/Dockerfile . + $(DOCKER) run --rm --cap-add NET_ADMIN miniupnpd-test + # Builds a C example that uses the libstorage C library and runs it testLibstorage: | build deps $(MAKE) $(if $(ncpu),-j$(ncpu),) libstorage diff --git a/build.nims b/build.nims index b74a931f..1ecc0319 100644 --- a/build.nims +++ b/build.nims @@ -78,6 +78,21 @@ task testIntegration, "Run integration tests": # test "testIntegration", params = "-d:chronicles_sinks=textlines[notimestamps,stdout],textlines[dynamic] " & # "-d:chronicles_enabled_topics:integration:TRACE" +task testNatPortMapping, "Run UPnP NAT integration test (requires miniupnpd container)": + buildBinary "storage", + outName = "storage", + params = "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE" + putEnv("STORAGE_INTEGRATION_TEST_INCLUDES", "integration/1_minute/testnatupnp.nim") + test "testIntegration", outName = "testIntegrationNat" + +# Used to build the testing binarie in Docker +task buildNatPortMappingBinaries, "Build UPnP NAT test binaries without running them": + buildBinary "storage", + outName = "storage", + params = "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE" + putEnv("STORAGE_INTEGRATION_TEST_INCLUDES", "integration/1_minute/testnatupnp.nim") + buildBinary "testIntegration", outName = "testIntegrationNat", srcDir = "tests/" + task build, "build Logos Storage binary": storageTask() diff --git a/library/storage_thread_requests/requests/node_debug_request.nim b/library/storage_thread_requests/requests/node_debug_request.nim index 1925e05b..a72d7143 100644 --- a/library/storage_thread_requests/requests/node_debug_request.nim +++ b/library/storage_thread_requests/requests/node_debug_request.nim @@ -63,21 +63,10 @@ proc getDebug( "announceAddresses": node.discovery.announceAddrs, "table": table, "nat": { - "reachability": - if storage[].autonatService.isSome: - $storage[].autonatService.get.networkReachability - else: - "unknown", + "reachability": reachabilityStr(storage[].autonatService), "relayRunning": storage[].autoRelayService.isSome and storage[].autoRelayService.get.isRunning, - "portMapping": - if storage[].natMapper.isNone or - storage[].natMapper.get.portMappingType == NoMapping: - "none" - elif storage[].natMapper.get.portMappingType == UpnpMapping: - "upnp" - else: - "pmp", + "portMapping": portMappingStr(storage[].natMapper), }, } diff --git a/storage/conf.nim b/storage/conf.nim index f50bd29e..54b89d41 100644 --- a/storage/conf.nim +++ b/storage/conf.nim @@ -341,6 +341,24 @@ type name: "nat-max-relays" .}: int + natPortMappingDiscoverTimeout* {. + desc: "Timeout in milliseconds for UPnP/NAT-PMP/PCP device discovery", + defaultValue: 500, + name: "nat-port-mapping-discover-timeout" + .}: int + + natPortMappingTimeout* {. + desc: "Timeout in milliseconds for creating a port mapping on the router", + defaultValue: 500, + name: "nat-port-mapping-timeout" + .}: int + + natPortMappingRecheckPeriod* {. + desc: "Period in milliseconds between rechecks of existing port mappings", + defaultValue: 300000, + name: "nat-port-mapping-recheck-period" + .}: int + natSimulation* {. desc: "Simulate NAT filtering behavior for testing: endpoint-independent, address-dependent, address-and-port-dependent", diff --git a/storage/nat.nim b/storage/nat.nim index 0cf06e9c..ce3fea5a 100644 --- a/storage/nat.nim +++ b/storage/nat.nim @@ -12,10 +12,10 @@ import std/[options, net] import results import pkg/chronos -import pkg/chronos/threadsync import pkg/chronicles import pkg/libp2p import pkg/libp2p/services/autorelayservice +import pkg/libp2p/protocols/connectivity/autonatv2/service import ./utils import ./utils/natutils @@ -25,105 +25,82 @@ import ./discovery logScope: topics = "nat" -const NatPortMappingTimeout = 5.seconds - type NatConfig* = object case hasExtIp*: bool of true: extIp*: IpAddress of false: nat*: NatStrategy -type PortMappingType* = enum - NoMapping - UpnpMapping - PmpMapping - type NatMapper* = ref object of RootObj natConfig*: NatConfig tcpPort*: Port discoveryPort*: Port - portMappingType*: PortMappingType - -type MapNatPortsCtx = object - natConfig: NatConfig - tcpPort: Port - discoveryPort: Port - signal: ThreadSignalPtr - result: Option[(Port, Port)] - portMappingType: PortMappingType - -proc mapNatPortsThread(ctx: ptr MapNatPortsCtx) {.thread.} = - if ctx.natConfig.hasExtIp: - discard ctx.signal.fireSync() - return - - # Devices are recreated on each call: discover() costs ~200ms but only fires - # when AutoNAT reports NotReachable, which is exactly when we want a fresh scan. - let upnpRes = UpnpDevice.init() - if upnpRes.isOk: - let ports = upnpRes.value.mapPorts(ctx.tcpPort, ctx.discoveryPort) - if ports.isSome: - ctx.portMappingType = UpnpMapping - ctx.result = ports - discard ctx.signal.fireSync() - return - - let pmpRes = PmpDevice.init() - if pmpRes.isOk: - let ports = pmpRes.value.mapPorts(ctx.tcpPort, ctx.discoveryPort) - if ports.isSome: - ctx.portMappingType = PmpMapping - ctx.result = ports - - discard ctx.signal.fireSync() + discoverTimeout*: int + mappingTimeout*: int + recheckPeriod*: int + tcpMappingId: Option[cint] + udpMappingId: Option[cint] + activeMappingProtocol*: Option[MappingProtocol] + activeTcpPort: Option[Port] + activeUdpPort: Option[Port] + plumInitialized: bool method mapNatPorts*( m: NatMapper -): Future[Option[(Port, Port)]] {.async: (raises: [CancelledError]), base, gcsafe.} = - let signal = ThreadSignalPtr.new().valueOr: - warn "Failed to create ThreadSignalPtr for NAT port mapping" - return none((Port, Port)) +): Future[Option[(Port, Port, MappingProtocol)]] {. + async: (raises: [CancelledError]), base, gcsafe +.} = + if m.natConfig.hasExtIp: + return none((Port, Port, MappingProtocol)) - var ctx = cast[ptr MapNatPortsCtx](createShared(MapNatPortsCtx)) - ctx[] = MapNatPortsCtx( - natConfig: m.natConfig, - tcpPort: m.tcpPort, - discoveryPort: m.discoveryPort, - signal: signal, - ) + # If both mappings are still active, return the stored ports without recreating. + if m.tcpMappingId.isSome and hasMapping(m.tcpMappingId.get) and m.udpMappingId.isSome and + hasMapping(m.udpMappingId.get): + return some((m.activeTcpPort.get, m.activeUdpPort.get, m.activeMappingProtocol.get)) - var thread: Thread[ptr MapNatPortsCtx] - var threadStarted = false - defer: - if threadStarted: - # Blocking the event loop here is acceptable: UPnP discover() is bounded - # by UPNP_TIMEOUT (200ms), so the worst-case stall is ~200ms. - joinThread(thread) - # Always sync hasUpnpMapping back, even on timeout or cancellation. - # If the thread mapped ports just after the timeout, close() will - # still clean them up on the router. - if ctx.portMappingType != NoMapping: - m.portMappingType = ctx.portMappingType - freeShared(ctx) - discard signal.close() + if not m.plumInitialized: + # 5s matches the old NatPortMappingTimeout used with miniupnpc/libnatpmp. + let res = init( + discoverTimeout = m.discoverTimeout, + mappingTimeout = m.mappingTimeout, + recheckPeriod = m.recheckPeriod, + ) + if res.isErr: + warn "Failed to initialize plum", msg = res.error + return none((Port, Port, MappingProtocol)) + m.plumInitialized = true - try: - createThread(thread, mapNatPortsThread, ctx) - threadStarted = true - except ValueError, ResourceExhaustedError: - warn "Failed to create thread for NAT port mapping" - return none((Port, Port)) + # If there is only one mapping, something went wrong somewhere + # so we delete the mappings to recreate them. + if m.tcpMappingId.isSome: + destroyMapping(m.tcpMappingId.get) + m.tcpMappingId = none(cint) - try: - if not await signal.wait().withTimeout(NatPortMappingTimeout): - warn "NAT port mapping thread timed out" - return none((Port, Port)) - except CancelledError as exc: - raise exc - except AsyncError as exc: - warn "Error waiting for NAT port mapping thread", error = exc.msg - return none((Port, Port)) + if m.udpMappingId.isSome: + destroyMapping(m.udpMappingId.get) + m.udpMappingId = none(cint) - return ctx.result + m.activeMappingProtocol = none(MappingProtocol) + m.activeTcpPort = none(Port) + m.activeUdpPort = none(Port) + + let tcpRes = await createMapping(TCP, m.tcpPort.uint16) + if tcpRes.isErr: + warn "TCP port mapping failed", msg = tcpRes.error + return none((Port, Port, MappingProtocol)) + + let udpRes = await createMapping(UDP, m.discoveryPort.uint16) + if udpRes.isErr: + warn "UDP port mapping failed", msg = udpRes.error + destroyMapping(tcpRes.value.id) + return none((Port, Port, MappingProtocol)) + + m.tcpMappingId = some(tcpRes.value.id) + m.udpMappingId = some(udpRes.value.id) + m.activeMappingProtocol = some(tcpRes.value.mapping.mappingProtocol) + m.activeTcpPort = some(Port(tcpRes.value.mapping.externalPort)) + m.activeUdpPort = some(Port(udpRes.value.mapping.externalPort)) + + some((m.activeTcpPort.get, m.activeUdpPort.get, m.activeMappingProtocol.get)) method handleNatStatus*( m: NatMapper, @@ -158,7 +135,7 @@ method handleNatStatus*( if dialBackAddr.isNone: warn "Got empty dialback address in AutoNat when node is NotReachable" else: - debug "Node is not reachable trying UPnP / PMP now" + debug "Node is not reachable trying port mapping now" # Here we should check first that a mapping exists. # If it does exist but Autonat still report as Not Reachable @@ -166,9 +143,9 @@ method handleNatStatus*( let maybePorts = await m.mapNatPorts() if maybePorts.isSome: - let (tcpPort, udpPort) = maybePorts.get() + let (tcpPort, udpPort, protocol) = maybePorts.get() - info "Port mapping created successfully", tcpPort, udpPort + info "Port mapping created successfully", tcpPort, udpPort, protocol let announceAddress = dialBackAddr.get.remapAddr(port = some(tcpPort)) @@ -195,25 +172,32 @@ method handleNatStatus*( else: debug "AutoRelayService started" -proc close*(m: NatMapper, device = UpnpDevice()) = - # UPnP mappings are permanent (leaseDuration=0) and must be deleted explicitly. - # NAT-PMP mappings expire automatically after NATPMP_LIFETIME seconds. - if m.portMappingType != UpnpMapping: - return +proc close*(m: NatMapper) = + if m.tcpMappingId.isSome: + destroyMapping(m.tcpMappingId.get) + m.tcpMappingId = none(cint) + if m.udpMappingId.isSome: + destroyMapping(m.udpMappingId.get) + m.udpMappingId = none(cint) + if m.plumInitialized: + discard cleanup() + m.plumInitialized = false - # deletePortMapping requires the IGD control URL set during init - let deviceRes = device.init() - if deviceRes.isErr: - warn "UPnP reinit failed during cleanup, port mappings may remain", - msg = deviceRes.error - return +proc reachabilityStr*(autonat: Option[AutonatV2Service]): string = + if autonat.isSome: + $autonat.get.networkReachability + else: + "unknown" - for (port, proto) in [ - (m.tcpPort, NatIpProtocol.Tcp), (m.discoveryPort, NatIpProtocol.Udp) - ]: - let res = deviceRes.value.deletePortMapping(port, proto) - if res.isErr: - error "UPnP port mapping deletion failed", port, proto, msg = res.error +proc portMappingStr*(natMapper: Option[NatMapper]): string = + if natMapper.isNone or natMapper.get.activeMappingProtocol.isNone: + return "none" + case natMapper.get.activeMappingProtocol.get + of MappingProtocol.UPnP: "upnp" + of MappingProtocol.NatPmp: "pmp" + of MappingProtocol.PCP: "pcp" + of MappingProtocol.Direct: "direct" + of MappingProtocol.Unknown: "none" proc findReachableNodes*(bootstrapNodes: seq[SignedPeerRecord]): seq[SignedPeerRecord] = ## Returns the list of nodes known to be directly reachable. diff --git a/storage/rest/api.nim b/storage/rest/api.nim index 6acdfd61..c125c7fb 100644 --- a/storage/rest/api.nim +++ b/storage/rest/api.nim @@ -535,8 +535,8 @@ proc initNodeApi(node: StorageNodeRef, conf: StorageConf, router: var RestRouter ## to invoke peer discovery, if it succeeds ## the returned addresses will be used to dial ## - ## `addrs` the listening addresses of the peers to dial, which is - ## /ip4/0.0.0.0/tcp/, where port is specified with the + ## `addrs` the listening addresses of the peers to dial, which is + ## /ip4/0.0.0.0/tcp/, where port is specified with the ## `--listen-port` CLI flag. ## var headers = buildCorsHeaders("GET", allowedOrigin) @@ -591,20 +591,10 @@ proc initDebugApi( "table": table, "storage": {"version": $storageVersion, "revision": $storageRevision}, "nat": { - "reachability": - if autonat.isSome: - $autonat.get.networkReachability - else: - "unknown", + "reachability": reachabilityStr(autonat), "clientMode": node.discovery.protocol.clientMode, "relayRunning": autoRelay.isSome and autoRelay.get.isRunning, - "portMapping": - if natMapper.isNone or natMapper.get.portMappingType == NoMapping: - "none" - elif natMapper.get.portMappingType == UpnpMapping: - "upnp" - else: - "pmp", + "portMapping": portMappingStr(natMapper), }, } diff --git a/storage/storage.nim b/storage/storage.nim index 61e8edc1..0c975b5d 100644 --- a/storage/storage.nim +++ b/storage/storage.nim @@ -392,6 +392,9 @@ proc new*( natConfig: config.nat, tcpPort: config.listenPort, discoveryPort: config.discoveryPort, + discoverTimeout: config.natPortMappingDiscoverTimeout, + mappingTimeout: config.natPortMappingTimeout, + recheckPeriod: config.natPortMappingRecheckPeriod, ) ) let relayService = AutoRelayService.new( diff --git a/storage/utils/natutils.nim b/storage/utils/natutils.nim index 45abd178..f96d4579 100644 --- a/storage/utils/natutils.nim +++ b/storage/utils/natutils.nim @@ -1,205 +1,15 @@ {.push raises: [].} import std/[options, net] -import nat_traversal/[miniupnpc, natpmp] import pkg/chronicles import results +import libplum/plum +import libplum/libplum -export miniupnpc, natpmp, results, options, net +export plum, libplum, results, options, net logScope: topics = "nat" -const UPNP_TIMEOUT* = 200 # ms -const NATPMP_LIFETIME* = 60 * 60 # seconds - type NatStrategy* = enum NatAuto - -type NatIpProtocol* = enum - Tcp - Udp - -# Generic Nat device can be UPnP or PmP -type NatDevice* = ref object of RootObj - -type UpnpDevice* = ref object of NatDevice - upnp: Miniupnp - -type PmpDevice* = ref object of NatDevice - npmp: NatPmp - -# appPortMapping is specific to the type of Nat device -method addPortMapping*( - d: NatDevice, port: Port, proto: NatIpProtocol -): Result[Port, string] {.base, gcsafe.} = - return err("not implemented") - -# Creates the mapping the the router and -# returns the opened ports. -method mapPorts*( - d: NatDevice, tcpPort, udpPort: Port -): Option[(Port, Port)] {.base, gcsafe.} = - var extTcpPort, extUdpPort: Port - - for t in [(tcpPort, NatIpProtocol.Tcp), (udpPort, NatIpProtocol.Udp)]: - let (port, proto) = t - let pmres = d.addPortMapping(port, proto) - - if pmres.isErr: - error "port mapping failed", msg = pmres.error - return none((Port, Port)) - - case proto - of Tcp: - extTcpPort = pmres.value - of Udp: - extUdpPort = pmres.value - - return some((extTcpPort, extUdpPort)) - -method getSpecificPortMapping*( - d: UpnpDevice, externalPort: string, protocol: UPNPProtocol -): Result[PortMappingRes, cstring] {.base, gcsafe.} = - if d.upnp == nil: - return err(cstring("upnp not initialized")) - - d.upnp.getSpecificPortMapping(externalPort = externalPort, protocol = protocol) - -method discover*(d: UpnpDevice): Result[int, cstring] {.base, gcsafe.} = - if d.upnp == nil: - return err(cstring("upnp not initialized")) - - return d.upnp.discover() - -method selectIGD*(d: UpnpDevice): SelectIGDResult {.base, gcsafe.} = - if d.upnp == nil: - return IGDNotFound - - return d.upnp.selectIGD() - -proc init*(T: type UpnpDevice): Result[UpnpDevice, string] {.gcsafe.} = - UpnpDevice().init() - -# Init UPnP device and create miniupnp instance. -# It call "discover" to retrieve the UPnP devices on the network, -# and then "selectIGD" to select a suitable device. -proc init*(d: UpnpDevice): Result[UpnpDevice, string] {.gcsafe.} = - if d.upnp == nil: - d.upnp = newMiniupnp() - - d.upnp.discoverDelay = UPNP_TIMEOUT - - let dres = d.discover() - if dres.isErr: - debug "UPnP", msg = dres.error - return err($dres.error) - - case d.selectIGD() - of IGDNotFound: - debug "UPnP", msg = "Internet Gateway Device not found. Giving up." - return err("IGD not found") - of IGDFound: - debug "UPnP", msg = "Internet Gateway Device found." - of IGDNotConnected: - debug "UPnP", - msg = "Internet Gateway Device found but it's not connected. Trying anyway." - of NotAnIGD: - debug "UPnP", - msg = - "Some device found, but it's not recognised as an Internet Gateway Device. Trying anyway." - of IGDIpNotRoutable: - debug "UPnP", - msg = - "Internet Gateway Device found and is connected, but with a reserved or non-routable IP. Trying anyway." - - return ok(d) - -# For UPnP, the external port is the same as the application port. -# This should work for most of the case. -# We could change this by using addAnyPortMapping for IGD2 compatible routers -# if needed. -method addPortMapping*( - d: UpnpDevice, port: Port, proto: NatIpProtocol -): Result[Port, string] {.gcsafe.} = - if d.upnp == nil: - return err("upnp not initialized") - - let protocol = if proto == NatIpProtocol.Tcp: UPNPProtocol.TCP else: UPNPProtocol.UDP - let pmres = d.upnp.addPortMapping( - externalPort = $port, - protocol = protocol, - internalHost = d.upnp.lanAddr, - internalPort = $port, - desc = "logos-storage", - leaseDuration = 0, - ) - if pmres.isErr: - return err($pmres.error) - - let cres = d.getSpecificPortMapping(externalPort = $port, protocol = protocol) - if cres.isErr: - # Eventually, the check could fail on some router even if the router is successful. - # So we log a warning but we still want to continue because it is not sure it is a failure. - warn "UPnP port mapping check failed. Assuming the check itself is broken and the port mapping was done.", - msg = cres.error - - info "UPnP: added port mapping", externalPort = port, internalPort = port - - return ok(port) - -method deletePortMapping*( - d: UpnpDevice, port: Port, proto: NatIpProtocol -): Result[void, string] {.base, gcsafe.} = - if d.upnp == nil: - return err("upnp not initialized") - - let protocol = if proto == NatIpProtocol.Tcp: UPNPProtocol.TCP else: UPNPProtocol.UDP - let res = d.upnp.deletePortMapping(externalPort = $port, protocol = protocol) - if res.isErr: - return err($res.error) - - debug "UPnP: deleted port mapping", port, proto - - ok() - -proc init*(T: type PmpDevice): Result[PmpDevice, string] {.gcsafe.} = - PmpDevice().init() - -# Create a NatPmP instance. -proc init*(d: PmpDevice): Result[PmpDevice, string] {.gcsafe.} = - if d.npmp == nil: - d.npmp = newNatPmp() - - let res = d.npmp.init() - if res.isErr: - debug "NAT-PMP", msg = res.error - return err($res.error) - - return ok(d) - -# Add a port mapping on NAT-PMP device. -# The application port might not be the external port. -# The latter is returned. -method addPortMapping*( - d: PmpDevice, port: Port, proto: NatIpProtocol -): Result[Port, string] {.gcsafe.} = - if d.npmp == nil: - return err("npmp not initialized") - - let protocol = - if proto == NatIpProtocol.Tcp: NatPmpProtocol.TCP else: NatPmpProtocol.UDP - let pmres = d.npmp.addPortMapping( - eport = port.cushort, - iport = port.cushort, - protocol = protocol, - lifetime = NATPMP_LIFETIME, - ) - if pmres.isErr: - return err(pmres.error) - - let extPort = Port(pmres.value) - - info "NAT-PMP: added port mapping", externalPort = extPort, internalPort = port - - return ok(extPort) diff --git a/tests/integration/1_minute/testnat.nim b/tests/integration/1_minute/testnat.nim index b9c7079a..abf40928 100644 --- a/tests/integration/1_minute/testnat.nim +++ b/tests/integration/1_minute/testnat.nim @@ -1,39 +1,15 @@ -import std/json import std/options -import std/sequtils import pkg/chronos import pkg/questionable/results import ../multinodes import ../storageclient import ../storageconfig +import ../nathelper -const - DetectionTimeout = 15_000 - RelayTimeout = 30_000 - PollInterval = 1_000 +export nathelper -proc checkNatStatus*( - client: StorageClient, reachability: string, relayRunning: bool, clientMode: bool -) {.async.} = - check eventuallySafe( - block: - let info = (await client.info()).get - let nat = info["nat"] - let addrs = info["addrs"].getElems.mapIt(it.getStr) - nat["reachability"].getStr() == reachability and - nat["clientMode"].getBool() == clientMode and - nat["relayRunning"].getBool() == relayRunning and - addrs.anyIt("p2p-circuit" in it) == relayRunning, - timeout = RelayTimeout, - pollInterval = PollInterval, - ) - -proc checkNatStatus*(client: StorageClient, reachability: string) {.async.} = - let notReachable = reachability == "NotReachable" - await client.checkNatStatus( - reachability, relayRunning = notReachable, clientMode = notReachable - ) +const DetectionTimeout = 15_000 # Reminder: multinodesuite setup the first node as bootstrap node multinodesuite "AutoNAT detection": diff --git a/tests/integration/1_minute/testnatupnp.nim b/tests/integration/1_minute/testnatupnp.nim index 933e6204..747246a9 100644 --- a/tests/integration/1_minute/testnatupnp.nim +++ b/tests/integration/1_minute/testnatupnp.nim @@ -1,21 +1,15 @@ -import std/[envvars, json, strutils, sequtils] +import std/[json, strutils, sequtils] import pkg/chronos import pkg/questionable/results -import nat_traversal/miniupnpc import ../multinodes import ../storageclient import ../storageconfig -import ../../../storage/utils/natutils -from ./testnat.nim import checkNatStatus +import ../nathelper -const - DetectionTimeout = 15_000 - RelayTimeout = 30_000 - PollInterval = 1_000 +const DetectionTimeout = 15_000 -# Requires a real UPnP router on the network (NAT_TEST_UPNP=1) multinodesuite "AutoNAT UPnP port mapping": let upnpConfig = NodeConfigs( clients: StorageConfigs @@ -33,10 +27,6 @@ multinodesuite "AutoNAT UPnP port mapping": test "node behind NAT maps ports via UPnP and exposes mapping in debug info", upnpConfig: - if getEnv("NAT_TEST_UPNP") != "1": - skip() - return - let node2 = clients()[1] await node2.client.checkNatStatus( @@ -51,23 +41,13 @@ multinodesuite "AutoNAT UPnP port mapping": pollInterval = PollInterval, ) - # Ideally we should find a way to test that the node is Reachable now - await node2.client.checkNatStatus( "NotReachable", relayRunning = false, clientMode = true ) - # Extract mapped TCP port from announce addresses and verify it exists on the IGD let announceAddrs = (await node2.client.info()).get["announceAddresses"].getElems.mapIt(it.getStr) let tcpAddr = announceAddrs.filterIt(it.startsWith("/ip4/") and "/tcp/" in it) check tcpAddr.len > 0 - let mappedPort = tcpAddr[0].split("/")[4] - let device = UpnpDevice.init() - check device.isOk - check device.get.getSpecificPortMapping(mappedPort, UPNPProtocol.TCP).isOk - await node2.stop() - - check device.get.getSpecificPortMapping(mappedPort, UPNPProtocol.TCP).isErr diff --git a/tests/integration/nat/docker-entrypoint.sh b/tests/integration/nat/docker-entrypoint.sh new file mode 100644 index 00000000..1be6b923 --- /dev/null +++ b/tests/integration/nat/docker-entrypoint.sh @@ -0,0 +1,28 @@ +#!/bin/bash +set -euo pipefail + +RUNDIR=/tmp/miniupnpd +mkdir -p "$RUNDIR" + +LAN_IF=$(ip route show default | awk '/default/{print $5; exit}') + +ip link add plum-wan type dummy +ip addr add 1.2.3.4/24 dev plum-wan +ip link set plum-wan up + +cat > "$RUNDIR/miniupnpd.conf" << EOF +ext_ifname=plum-wan +listening_ip=$LAN_IF +enable_pcp_pmp=no +port=0 +allow 1024-65535 0.0.0.0/0 1024-65535 +EOF + +if [[ "${DEBUG:-0}" == "1" ]]; then + miniupnpd -d -f "$RUNDIR/miniupnpd.conf" & +else + miniupnpd -d -f "$RUNDIR/miniupnpd.conf" > /dev/null 2>&1 & +fi +sleep 1 + +/app/build/testIntegrationNat diff --git a/tests/integration/nat/miniupnpd_stub_rdr.c b/tests/integration/nat/miniupnpd_stub_rdr.c new file mode 100644 index 00000000..15e45b0a --- /dev/null +++ b/tests/integration/nat/miniupnpd_stub_rdr.c @@ -0,0 +1,169 @@ +/* Stub firewall backend for miniupnpd. + * Replaces iptcrdr.o + iptpinhole.o + nfct_get.o. + * All mapping operations succeed without touching the kernel. */ + +#include +#include + +/* commonrdr.h interface */ + +int init_redirect(void) { return 0; } +void shutdown_redirect(void) {} + +int get_redirect_rule_count(const char *ifname) +{ (void)ifname; return 0; } + +int get_redirect_rule(const char *ifname, unsigned short eport, int proto, + char *iaddr, int iaddrlen, unsigned short *iport, + char *desc, int desclen, + char *rhost, int rhostlen, + unsigned int *timestamp, + uint64_t *packets, uint64_t *bytes) +{ (void)ifname; (void)eport; (void)proto; (void)iaddr; (void)iaddrlen; + (void)iport; (void)desc; (void)desclen; (void)rhost; (void)rhostlen; + (void)timestamp; (void)packets; (void)bytes; return -1; } + +int get_redirect_rule_by_index(int index, + char *ifname, unsigned short *eport, + char *iaddr, int iaddrlen, unsigned short *iport, + int *proto, char *desc, int desclen, + char *rhost, int rhostlen, + unsigned int *timestamp, + uint64_t *packets, uint64_t *bytes) +{ (void)index; (void)ifname; (void)eport; (void)iaddr; (void)iaddrlen; + (void)iport; (void)proto; (void)desc; (void)desclen; (void)rhost; + (void)rhostlen; (void)timestamp; (void)packets; (void)bytes; return -1; } + +unsigned short *get_portmappings_in_range(unsigned short startport, + unsigned short endport, + int proto, unsigned int *number) +{ (void)startport; (void)endport; (void)proto; *number = 0; return 0; } + +int update_portmapping(const char *ifname, unsigned short eport, int proto, + unsigned short iport, const char *desc, + unsigned int timestamp) +{ (void)ifname; (void)eport; (void)proto; (void)iport; (void)desc; + (void)timestamp; return 0; } + +int update_portmapping_desc_timestamp(const char *ifname, + unsigned short eport, int proto, + const char *desc, unsigned int timestamp) +{ (void)ifname; (void)eport; (void)proto; (void)desc; (void)timestamp; + return 0; } + +/* iptcrdr.h interface */ + +int add_redirect_rule2(const char *ifname, + const char *rhost, unsigned short eport, + const char *iaddr, unsigned short iport, int proto, + const char *desc, unsigned int timestamp) +{ (void)ifname; (void)rhost; (void)eport; (void)iaddr; (void)iport; + (void)proto; (void)desc; (void)timestamp; return 0; } + +int add_peer_redirect_rule2(const char *ifname, + const char *rhost, unsigned short rport, + const char *eaddr, unsigned short eport, + const char *iaddr, unsigned short iport, int proto, + const char *desc, unsigned int timestamp) +{ (void)ifname; (void)rhost; (void)rport; (void)eaddr; (void)eport; + (void)iaddr; (void)iport; (void)proto; (void)desc; (void)timestamp; + return 0; } + +int add_filter_rule2(const char *ifname, + const char *rhost, const char *iaddr, + unsigned short eport, unsigned short iport, + int proto, const char *desc) +{ (void)ifname; (void)rhost; (void)iaddr; (void)eport; (void)iport; + (void)proto; (void)desc; return 0; } + +int delete_redirect_and_filter_rules(unsigned short eport, int proto) +{ (void)eport; (void)proto; return 0; } + +int delete_filter_rule(const char *ifname, unsigned short port, int proto) +{ (void)ifname; (void)port; (void)proto; return 0; } + +int add_peer_dscp_rule2(const char *ifname, + const char *rhost, unsigned short rport, + unsigned char dscp, + const char *iaddr, unsigned short iport, int proto, + const char *desc, unsigned int timestamp) +{ (void)ifname; (void)rhost; (void)rport; (void)dscp; (void)iaddr; + (void)iport; (void)proto; (void)desc; (void)timestamp; return 0; } + +int get_peer_rule_by_index(int index, + char *ifname, unsigned short *eport, + char *iaddr, int iaddrlen, unsigned short *iport, + int *proto, char *desc, int desclen, + char *rhost, int rhostlen, unsigned short *rport, + unsigned int *timestamp, + uint64_t *packets, uint64_t *bytes) +{ (void)index; (void)ifname; (void)eport; (void)iaddr; (void)iaddrlen; + (void)iport; (void)proto; (void)desc; (void)desclen; (void)rhost; + (void)rhostlen; (void)rport; (void)timestamp; (void)packets; (void)bytes; + return -1; } + +int get_nat_redirect_rule(const char *nat_chain_name, const char *ifname, + unsigned short eport, int proto, + char *iaddr, int iaddrlen, unsigned short *iport, + char *desc, int desclen, + char *rhost, int rhostlen, + unsigned int *timestamp, + uint64_t *packets, uint64_t *bytes) +{ (void)nat_chain_name; (void)ifname; (void)eport; (void)proto; (void)iaddr; + (void)iaddrlen; (void)iport; (void)desc; (void)desclen; (void)rhost; + (void)rhostlen; (void)timestamp; (void)packets; (void)bytes; return -1; } + +int list_redirect_rule(const char *ifname) +{ (void)ifname; return 0; } + +/* commonrdr.h USE_NETFILTER interface */ + +int set_rdr_name(int param, const char *string) +{ (void)param; (void)string; return 0; } + +/* nfct_get.c interface */ + +int get_nat_ext_addr(struct sockaddr *src, struct sockaddr *dst, uint8_t proto, + struct sockaddr *ret_ext) +{ (void)src; (void)dst; (void)proto; (void)ret_ext; return -1; } + +/* iptpinhole.h interface */ + +int find_pinhole(const char *ifname, + const char *rem_host, unsigned short rem_port, + const char *int_client, unsigned short int_port, + int proto, char *desc, int desc_len, unsigned int *timestamp) +{ (void)ifname; (void)rem_host; (void)rem_port; (void)int_client; + (void)int_port; (void)proto; (void)desc; (void)desc_len; (void)timestamp; + return -1; } + +int add_pinhole(const char *ifname, + const char *rem_host, unsigned short rem_port, + const char *int_client, unsigned short int_port, + int proto, const char *desc, unsigned int timestamp) +{ (void)ifname; (void)rem_host; (void)rem_port; (void)int_client; + (void)int_port; (void)proto; (void)desc; (void)timestamp; return 0; } + +int update_pinhole(unsigned short uid, unsigned int timestamp) +{ (void)uid; (void)timestamp; return 0; } + +int delete_pinhole(unsigned short uid) +{ (void)uid; return 0; } + +int get_pinhole_info(unsigned short uid, + char *rem_host, int rem_hostlen, unsigned short *rem_port, + char *int_client, int int_clientlen, + unsigned short *int_port, + int *proto, char *desc, int desclen, + unsigned int *timestamp, + uint64_t *packets, uint64_t *bytes) +{ (void)uid; (void)rem_host; (void)rem_hostlen; (void)rem_port; + (void)int_client; (void)int_clientlen; (void)int_port; (void)proto; + (void)desc; (void)desclen; (void)timestamp; (void)packets; (void)bytes; + return -1; } + +int get_pinhole_uid_by_index(int index) +{ (void)index; return -1; } + +int clean_pinhole_list(unsigned int *next_timestamp) +{ (void)next_timestamp; return 0; } diff --git a/tests/integration/nathelper.nim b/tests/integration/nathelper.nim new file mode 100644 index 00000000..68f21176 --- /dev/null +++ b/tests/integration/nathelper.nim @@ -0,0 +1,34 @@ +import std/json +import std/sequtils +import pkg/chronos +import pkg/questionable/results + +import ./multinodes +import ./storageclient +import ./storageconfig + +const + RelayTimeout* = 30_000 + PollInterval* = 1_000 + +proc checkNatStatus*( + client: StorageClient, reachability: string, relayRunning: bool, clientMode: bool +) {.async.} = + check eventuallySafe( + block: + let info = (await client.info()).get + let nat = info["nat"] + let addrs = info["addrs"].getElems.mapIt(it.getStr) + nat["reachability"].getStr() == reachability and + nat["clientMode"].getBool() == clientMode and + nat["relayRunning"].getBool() == relayRunning and + addrs.anyIt("p2p-circuit" in it) == relayRunning, + timeout = RelayTimeout, + pollInterval = PollInterval, + ) + +proc checkNatStatus*(client: StorageClient, reachability: string) {.async.} = + let notReachable = reachability == "NotReachable" + await client.checkNatStatus( + reachability, relayRunning = notReachable, clientMode = notReachable + ) diff --git a/tests/integration/storageconfig.nim b/tests/integration/storageconfig.nim index 3508b31e..b509c0dd 100644 --- a/tests/integration/storageconfig.nim +++ b/tests/integration/storageconfig.nim @@ -330,6 +330,30 @@ proc withNatScheduleInterval*( config.addCliOption("--nat-schedule-interval", $scheduleInterval) return startConfig +proc withNatPortMappingDiscoverTimeout*( + self: StorageConfigs, timeout: int +): StorageConfigs {.raises: [StorageConfigError].} = + var startConfig = self + for config in startConfig.configs.mitems: + config.addCliOption("--nat-port-mapping-discover-timeout", $timeout) + return startConfig + +proc withNatPortMappingTimeout*( + self: StorageConfigs, timeout: int +): StorageConfigs {.raises: [StorageConfigError].} = + var startConfig = self + for config in startConfig.configs.mitems: + config.addCliOption("--nat-port-mapping-timeout", $timeout) + return startConfig + +proc withNatPortMappingRecheckPeriod*( + self: StorageConfigs, timeout: int +): StorageConfigs {.raises: [StorageConfigError].} = + var startConfig = self + for config in startConfig.configs.mitems: + config.addCliOption("--nat-port-mapping-recheck-period", $timeout) + return startConfig + proc withExtIp*( self: StorageConfigs, idx: int, ip = "127.0.0.1" ): StorageConfigs {.raises: [StorageConfigError].} = diff --git a/tests/storage/testnat.nim b/tests/storage/testnat.nim index d4c14f23..c1af8eaf 100644 --- a/tests/storage/testnat.nim +++ b/tests/storage/testnat.nim @@ -1,4 +1,4 @@ -import std/[net, importutils] +import std/[net] import pkg/chronos import pkg/libp2p/[multiaddress, multihash, multicodec] import pkg/libp2p/protocols/connectivity/autonat/types @@ -14,54 +14,16 @@ import ../../storage/discovery import ../../storage/rng import ../../storage/utils -privateAccess(NatMapper) - -type MockUpnpDevice = ref object of UpnpDevice - deletedPorts: seq[(Port, NatIpProtocol)] - -method discover*(d: MockUpnpDevice): Result[int, cstring] {.gcsafe.} = - ok(1) - -method selectIGD*(d: MockUpnpDevice): SelectIGDResult {.gcsafe.} = - IGDFound - -method deletePortMapping*( - d: MockUpnpDevice, port: Port, proto: NatIpProtocol -): Result[void, string] {.gcsafe.} = - d.deletedPorts.add((port, proto)) - ok() - type MockNatMapper = ref object of NatMapper - mappedPorts: Option[(Port, Port)] + mappedPorts: Option[(Port, Port, MappingProtocol)] method mapNatPorts*( m: MockNatMapper -): Future[Option[(Port, Port)]] {.async: (raises: [CancelledError]), gcsafe.} = +): Future[Option[(Port, Port, MappingProtocol)]] {. + async: (raises: [CancelledError]), gcsafe +.} = m.mappedPorts -suite "NAT - NatMapper.close": - test "does nothing when no upnp mapping": - let mapper = MockNatMapper( - natConfig: NatConfig(hasExtIp: false, nat: NatAuto), - tcpPort: Port(8080), - discoveryPort: Port(8090), - ) - let device = MockUpnpDevice() - mapper.close(device) - check device.deletedPorts.len == 0 - - test "deletes tcp and udp ports when upnp mapping exists": - let mapper = MockNatMapper( - natConfig: NatConfig(hasExtIp: false, nat: NatAuto), - tcpPort: Port(8080), - discoveryPort: Port(8090), - ) - mapper.portMappingType = UpnpMapping - let device = MockUpnpDevice() - mapper.close(device) - check device.deletedPorts == - @[(Port(8080), NatIpProtocol.Tcp), (Port(8090), NatIpProtocol.Udp)] - asyncchecksuite "NAT - handleNatStatus": var sw: Switch var key: PrivateKey @@ -86,7 +48,8 @@ asyncchecksuite "NAT - handleNatStatus": test "handleNatStatus announces mapped address when NotReachable and UPnP succeeds": let dialBack = MultiAddress.init("/ip4/1.2.3.4/tcp/8080").expect("valid") - let mapper = MockNatMapper(mappedPorts: some((Port(9000), Port(9001)))) + let mapper = + MockNatMapper(mappedPorts: some((Port(9000), Port(9001), MappingProtocol.UPnP))) await mapper.handleNatStatus( NotReachable, Opt.some(dialBack), discoveryPort, disc, sw, autoRelay @@ -98,7 +61,7 @@ asyncchecksuite "NAT - handleNatStatus": check disc.protocol.clientMode test "handleNatStatus starts autoRelay when NotReachable and UPnP failed": - let mapper = MockNatMapper(mappedPorts: none((Port, Port))) + let mapper = MockNatMapper(mappedPorts: none((Port, Port, MappingProtocol))) await mapper.handleNatStatus( NotReachable, Opt.none(MultiAddress), discoveryPort, disc, sw, autoRelay @@ -109,7 +72,7 @@ asyncchecksuite "NAT - handleNatStatus": test "handleNatStatus starts autoRelay when NotReachable and mapping fails": let dialBack = MultiAddress.init("/ip4/1.2.3.4/tcp/8080").expect("valid") - let mapper = MockNatMapper(mappedPorts: none((Port, Port))) + let mapper = MockNatMapper(mappedPorts: none((Port, Port, MappingProtocol))) await mapper.handleNatStatus( NotReachable, Opt.some(dialBack), discoveryPort, disc, sw, autoRelay @@ -120,7 +83,7 @@ asyncchecksuite "NAT - handleNatStatus": check disc.protocol.clientMode test "handleNatStatus does not announce address when Reachable and no dialBackAddr": - let mapper = MockNatMapper(mappedPorts: none((Port, Port))) + let mapper = MockNatMapper(mappedPorts: none((Port, Port, MappingProtocol))) await mapper.handleNatStatus( Reachable, Opt.none(MultiAddress), discoveryPort, disc, sw, autoRelay @@ -132,7 +95,7 @@ asyncchecksuite "NAT - handleNatStatus": test "handleNatStatus stops relay and announces dialBackAddr when Reachable": let dialBack = MultiAddress.init("/ip4/1.2.3.4/tcp/8080").expect("valid") - let mapper = MockNatMapper(mappedPorts: none((Port, Port))) + let mapper = MockNatMapper(mappedPorts: none((Port, Port, MappingProtocol))) discard await autorelayservice.setup(autoRelay, sw) await mapper.handleNatStatus( diff --git a/tests/storage/testnatutils.nim b/tests/storage/testnatutils.nim deleted file mode 100644 index 10de4d18..00000000 --- a/tests/storage/testnatutils.nim +++ /dev/null @@ -1,93 +0,0 @@ -import std/[options, net] -import nat_traversal/[miniupnpc, natpmp] -import pkg/results -import ../asynctest -import ../../storage/utils/natutils - -type MockUpnpDev = ref object of UpnpDevice - discoverOk: bool - igdResult: SelectIGDResult - addPortMappingOk: bool - failOnProto: Option[NatIpProtocol] - -type MockPmpDev = ref object of PmpDevice - addPortMappingOk: bool - mappedPort: Port - -method discover*(d: MockUpnpDev): Result[int, cstring] {.gcsafe.} = - if d.discoverOk: - ok(1) - else: - err(cstring("discover failed")) - -method selectIGD*(d: MockUpnpDev): SelectIGDResult {.gcsafe.} = - d.igdResult - -method addPortMapping*( - d: MockUpnpDev, port: Port, proto: NatIpProtocol -): Result[Port, string] {.gcsafe.} = - if d.failOnProto == some(proto): - err("mapping failed") - elif d.addPortMappingOk: - ok(port) - else: - err("mapping failed") - -method getSpecificPortMapping*( - d: MockUpnpDev, externalPort: string, protocol: UPNPProtocol -): Result[PortMappingRes, cstring] {.gcsafe.} = - ok(PortMappingRes()) - -method addPortMapping*( - d: MockPmpDev, port: Port, proto: NatIpProtocol -): Result[Port, string] {.gcsafe.} = - if d.addPortMappingOk: - ok(d.mappedPort) - else: - err("mapping failed") - -suite "NAT - UpnpDevice.init": - test "returns err when discover fails": - check MockUpnpDev(discoverOk: false).init().isErr - - test "returns err when IGD not found": - check MockUpnpDev(discoverOk: true, igdResult: IGDNotFound).init().isErr - - test "returns ok when IGD found": - check MockUpnpDev(discoverOk: true, igdResult: IGDFound).init().isOk - - test "returns ok when IGD not connected": - check MockUpnpDev(discoverOk: true, igdResult: IGDNotConnected).init().isOk - - test "returns ok when not an IGD": - check MockUpnpDev(discoverOk: true, igdResult: NotAnIGD).init().isOk - - test "returns ok when IP not routable": - check MockUpnpDev(discoverOk: true, igdResult: IGDIpNotRoutable).init().isOk - -suite "NAT - UpnpDevice.mapPorts": - test "returns none when addPortMapping fails": - check MockUpnpDev(addPortMappingOk: false).mapPorts(Port(8080), Port(8090)).isNone - - test "returns mapped ports": - let res = MockUpnpDev(addPortMappingOk: true).mapPorts(Port(8080), Port(8090)) - check res.isSome - check res.get() == (Port(8080), Port(8090)) - - test "returns none when tcp mapping fails": - let d = MockUpnpDev(addPortMappingOk: true, failOnProto: some(NatIpProtocol.Tcp)) - check d.mapPorts(Port(8080), Port(8090)).isNone - - test "returns none when udp mapping fails": - let d = MockUpnpDev(addPortMappingOk: true, failOnProto: some(NatIpProtocol.Udp)) - check d.mapPorts(Port(8080), Port(8090)).isNone - -suite "NAT - PmpDevice.mapPorts": - test "returns none when mapping fails": - check MockPmpDev(addPortMappingOk: false).mapPorts(Port(8080), Port(8090)).isNone - - test "returns assigned external ports": - let d = MockPmpDev(addPortMappingOk: true, mappedPort: Port(9000)) - let res = d.mapPorts(Port(8080), Port(8090)) - check res.isSome - check res.get() == (Port(9000), Port(9000)) From b04165e34d7596255dfbb79e3ff17dc7712eedcf Mon Sep 17 00:00:00 2001 From: Arnaud Date: Tue, 19 May 2026 17:38:52 +0400 Subject: [PATCH 053/167] Fix integration tests --- .github/workflows/ci.yml | 12 ++++++++++++ Makefile | 8 +++++++- build.nims | 4 ++-- tests/{integration/1_minute => nat}/testnatupnp.nim | 8 ++++---- 4 files changed, 25 insertions(+), 7 deletions(-) rename tests/{integration/1_minute => nat}/testnatupnp.nim (90%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ee4f88d7..c1f92183 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,6 +40,18 @@ jobs: matrix: ${{ needs.matrix.outputs.matrix }} cache_nonce: ${{ needs.matrix.outputs.cache_nonce }} + nat-integration: + name: NAT integration tests + runs-on: ubuntu-latest + steps: + - name: Checkout sources + uses: actions/checkout@v4 + with: + submodules: recursive + ref: ${{ github.event.pull_request.head.sha }} + - name: Run NAT integration tests + run: make testNatIntegration + linting: runs-on: ubuntu-latest if: github.event_name == 'pull_request' diff --git a/Makefile b/Makefile index 2a1a05a3..77512966 100644 --- a/Makefile +++ b/Makefile @@ -128,7 +128,13 @@ deps: | libbacktrace endif libplum: - + "$(MAKE)" -C vendor/nim-libplum/vendor/libplum libplum.a CC=$(CC) $(HANDLE_OUTPUT) + cmake -B vendor/nim-libplum/vendor/libplum/build \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_SHARED_LIBS=OFF \ + vendor/nim-libplum/vendor/libplum $(HANDLE_OUTPUT) + + $(MAKE) -C vendor/nim-libplum/vendor/libplum/build $(HANDLE_OUTPUT) + cp vendor/nim-libplum/vendor/libplum/build/libplum.a \ + vendor/nim-libplum/vendor/libplum/libplum.a update: | update-common diff --git a/build.nims b/build.nims index 1ecc0319..484d365a 100644 --- a/build.nims +++ b/build.nims @@ -82,7 +82,7 @@ task testNatPortMapping, "Run UPnP NAT integration test (requires miniupnpd cont buildBinary "storage", outName = "storage", params = "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE" - putEnv("STORAGE_INTEGRATION_TEST_INCLUDES", "integration/1_minute/testnatupnp.nim") + putEnv("STORAGE_INTEGRATION_TEST_INCLUDES", "nat/testnatupnp.nim") test "testIntegration", outName = "testIntegrationNat" # Used to build the testing binarie in Docker @@ -90,7 +90,7 @@ task buildNatPortMappingBinaries, "Build UPnP NAT test binaries without running buildBinary "storage", outName = "storage", params = "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE" - putEnv("STORAGE_INTEGRATION_TEST_INCLUDES", "integration/1_minute/testnatupnp.nim") + putEnv("STORAGE_INTEGRATION_TEST_INCLUDES", "nat/testnatupnp.nim") buildBinary "testIntegration", outName = "testIntegrationNat", srcDir = "tests/" task build, "build Logos Storage binary": diff --git a/tests/integration/1_minute/testnatupnp.nim b/tests/nat/testnatupnp.nim similarity index 90% rename from tests/integration/1_minute/testnatupnp.nim rename to tests/nat/testnatupnp.nim index 747246a9..9552ae71 100644 --- a/tests/integration/1_minute/testnatupnp.nim +++ b/tests/nat/testnatupnp.nim @@ -2,11 +2,11 @@ import std/[json, strutils, sequtils] import pkg/chronos import pkg/questionable/results -import ../multinodes -import ../storageclient -import ../storageconfig +import ../integration/multinodes +import ../integration/storageclient +import ../integration/storageconfig -import ../nathelper +import ../integration/nathelper const DetectionTimeout = 15_000 From 6f24d79f338d41c5de0d90d8f7b3a92920ebbc8c Mon Sep 17 00:00:00 2001 From: Arnaud Date: Tue, 19 May 2026 20:01:07 +0400 Subject: [PATCH 054/167] Fix build for windows --- Makefile | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 77512966..a0292028 100644 --- a/Makefile +++ b/Makefile @@ -127,15 +127,6 @@ ifneq ($(USE_LIBBACKTRACE), 0) deps: | libbacktrace endif -libplum: - cmake -B vendor/nim-libplum/vendor/libplum/build \ - -DCMAKE_BUILD_TYPE=Release \ - -DBUILD_SHARED_LIBS=OFF \ - vendor/nim-libplum/vendor/libplum $(HANDLE_OUTPUT) - + $(MAKE) -C vendor/nim-libplum/vendor/libplum/build $(HANDLE_OUTPUT) - cp vendor/nim-libplum/vendor/libplum/build/libplum.a \ - vendor/nim-libplum/vendor/libplum/libplum.a - update: | update-common # detecting the os @@ -182,6 +173,23 @@ testAll: | build deps $(ENV_SCRIPT) nim testAll $(NIM_PARAMS) build.nims $(MAKE) $(if $(ncpu),-j$(ncpu),) testLibstorage +LIBPLUM_DIR := vendor/nim-libplum/vendor/libplum +LIBPLUM_BUILD_DIR := $(LIBPLUM_DIR)/build +LIBPLUM_CMAKE_FLAGS := -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF + +libplum: +ifeq ($(detected_OS), Windows) +ifneq ($(MSYSTEM),) + cmake -B $(LIBPLUM_BUILD_DIR) $(LIBPLUM_CMAKE_FLAGS) -G"MSYS Makefiles" $(LIBPLUM_DIR) $(HANDLE_OUTPUT) +else + cmake -B $(LIBPLUM_BUILD_DIR) $(LIBPLUM_CMAKE_FLAGS) $(LIBPLUM_DIR) $(HANDLE_OUTPUT) +endif +else + cmake -B $(LIBPLUM_BUILD_DIR) $(LIBPLUM_CMAKE_FLAGS) $(LIBPLUM_DIR) $(HANDLE_OUTPUT) +endif + + $(MAKE) -C $(LIBPLUM_BUILD_DIR) $(HANDLE_OUTPUT) + cp $(LIBPLUM_BUILD_DIR)/libplum.a $(LIBPLUM_DIR)/libplum.a + # nim-libbacktrace LIBBACKTRACE_MAKE_FLAGS := -C vendor/nim-libbacktrace --no-print-directory BUILD_CXX_LIB=0 libbacktrace: From 9c95ae7f057845a2bdcb3ff3f082acb67ae6f45c Mon Sep 17 00:00:00 2001 From: Arnaud Date: Wed, 20 May 2026 09:50:13 +0400 Subject: [PATCH 055/167] Add dockerfile for nat testing --- .gitignore | 1 + tests/integration/nat/Dockerfile | 54 ++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+) create mode 100644 tests/integration/nat/Dockerfile diff --git a/.gitignore b/.gitignore index f689aee1..166a02ff 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ !LICENSE* !Makefile !Jenkinsfile +!Dockerfile nimcache/ diff --git a/tests/integration/nat/Dockerfile b/tests/integration/nat/Dockerfile new file mode 100644 index 00000000..2b7f9075 --- /dev/null +++ b/tests/integration/nat/Dockerfile @@ -0,0 +1,54 @@ +FROM ubuntu:24.04 + +ARG NIM_VERSION=2.2.10 + +RUN apt-get update && apt-get install -y --no-install-recommends \ + gcc g++ make cmake git curl ca-certificates xz-utils \ + libc-dev ccache \ + iproute2 \ + && rm -rf /var/lib/apt/lists/* + +# Build miniupnpd with stub redirector: no iptables/nftables needed, always +# returns success for port mapping requests — safe for isolated test containers. +COPY tests/integration/nat/miniupnpd_stub_rdr.c /tmp/stub_rdr.c +RUN git clone --depth=1 --branch miniupnpd_2_3_9 \ + https://github.com/miniupnp/miniupnp.git /tmp/miniupnp \ + && cd /tmp/miniupnp/miniupnpd \ + && ./configure \ + && cp /tmp/stub_rdr.c . \ + && make NETFILTEROBJS=stub_rdr.o miniupnpd \ + && install -m 755 miniupnpd /usr/local/sbin/miniupnpd \ + && rm -rf /tmp/miniupnp /tmp/stub_rdr.c + +# Install Nim +RUN curl -fsSL "https://nim-lang.org/download/nim-${NIM_VERSION}-linux_x64.tar.xz" \ + | tar -xJ -C /opt && \ + ln -s "/opt/nim-${NIM_VERSION}/bin/nim" /usr/local/bin/nim + +WORKDIR /app + +# Copy project source (build context must be the project root) +COPY vendor/ vendor/ +COPY storage/ storage/ +COPY library/ library/ +COPY tests/ tests/ +COPY build.nims config.nims storage.nim ./ + +# Build libplum C library and Nim binaries. +# ccache caches C compilation across builds; nimcache caches Nim's generated C files. +RUN --mount=type=cache,target=/root/.ccache \ + --mount=type=cache,target=/app/nimcache \ + export PATH="/usr/lib/ccache:$PATH" && \ + rm -rf vendor/nim-libplum/vendor/libplum/build && \ + cmake -B vendor/nim-libplum/vendor/libplum/build \ + -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF \ + vendor/nim-libplum/vendor/libplum && \ + make -j$(nproc) -C vendor/nim-libplum/vendor/libplum/build && \ + cp vendor/nim-libplum/vendor/libplum/build/libplum.a \ + vendor/nim-libplum/vendor/libplum/libplum.a && \ + USE_SYSTEM_NIM=1 vendor/nimbus-build-system/scripts/env.sh \ + nim buildNatPortMappingBinaries -d:debug -d:disable_libbacktrace build.nims + +COPY tests/integration/nat/docker-entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh +ENTRYPOINT ["/entrypoint.sh"] From 080e024cb5d092b40c7ad7e7240c1e403efed413 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Wed, 20 May 2026 10:00:17 +0400 Subject: [PATCH 056/167] Add NAT integration tests in matrix --- .github/workflows/ci-reusable.yml | 5 +++++ .github/workflows/ci.yml | 12 ------------ tools/scripts/ci-job-matrix.sh | 11 +++++++++++ 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci-reusable.yml b/.github/workflows/ci-reusable.yml index fe0c13af..2d775f1b 100644 --- a/.github/workflows/ci-reusable.yml +++ b/.github/workflows/ci-reusable.yml @@ -68,6 +68,11 @@ jobs: if: matrix.tests == 'libstorage' || matrix.tests == 'all' run: make -j${ncpu} testLibstorage + ## Part 4 Tests ## + - name: NAT integration tests + if: matrix.tests == 'nat-integration' + run: make testNatIntegration + status: if: always() needs: [build] diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c1f92183..ee4f88d7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,18 +40,6 @@ jobs: matrix: ${{ needs.matrix.outputs.matrix }} cache_nonce: ${{ needs.matrix.outputs.cache_nonce }} - nat-integration: - name: NAT integration tests - runs-on: ubuntu-latest - steps: - - name: Checkout sources - uses: actions/checkout@v4 - with: - submodules: recursive - ref: ${{ github.event.pull_request.head.sha }} - - name: Run NAT integration tests - run: make testNatIntegration - linting: runs-on: ubuntu-latest if: github.event_name == 'pull_request' diff --git a/tools/scripts/ci-job-matrix.sh b/tools/scripts/ci-job-matrix.sh index e55968bf..455234c4 100755 --- a/tools/scripts/ci-job-matrix.sh +++ b/tools/scripts/ci-job-matrix.sh @@ -117,11 +117,22 @@ libstorage_test () { job } +# outputs a NAT integration test job +# Linux-only: miniupnpd is a Linux daemon, network namespace manipulation requires Linux +nat_integration_test () { + job_tests="nat-integration" + job_includes="" + job +} + # outputs jobs for all test types all_tests () { unit_test integration_test libstorage_test + if [ "$job_os" = "linux" ]; then + nat_integration_test + fi } # outputs jobs for the specified operating systems and all test types From f020de1387396c614314cb9c57283dd09843862f Mon Sep 17 00:00:00 2001 From: Arnaud Date: Thu, 4 Jun 2026 14:37:47 +0400 Subject: [PATCH 057/167] Refactactoring --- storage/conf.nim | 11 +++- storage/nat.nim | 73 +++++++++++++++-------- storage/rest/api.nim | 4 +- storage/storage.nim | 115 ++++++++++++++++++++++-------------- storage/utils/addrutils.nim | 10 ++++ 5 files changed, 141 insertions(+), 72 deletions(-) diff --git a/storage/conf.nim b/storage/conf.nim index 54b89d41..9b08d3e2 100644 --- a/storage/conf.nim +++ b/storage/conf.nim @@ -367,10 +367,17 @@ type hidden .}: Option[string] - relay* {. + autonatServer* {. + desc: "Enable AutoNAT server to help other nodes check their reachability", + defaultValue: false, + name: "autonat-server", + hidden + .}: bool + + isRelayServer* {. desc: "Enable circuit relay server (hop) - use on publicly reachable nodes only", defaultValue: false, - name: "relay" + name: "relay-server" .}: bool func defaultAddress*(conf: StorageConf): IpAddress = diff --git a/storage/nat.nim b/storage/nat.nim index ce3fea5a..9842b838 100644 --- a/storage/nat.nim +++ b/storage/nat.nim @@ -8,7 +8,7 @@ {.push raises: [].} -import std/[options, net] +import std/[options, net, os] import results import pkg/chronos @@ -30,7 +30,7 @@ type NatConfig* = object of true: extIp*: IpAddress of false: nat*: NatStrategy -type NatMapper* = ref object of RootObj +type NatPortMapper* = ref object of RootObj natConfig*: NatConfig tcpPort*: Port discoveryPort*: Port @@ -45,7 +45,7 @@ type NatMapper* = ref object of RootObj plumInitialized: bool method mapNatPorts*( - m: NatMapper + m: NatPortMapper ): Future[Option[(Port, Port, MappingProtocol)]] {. async: (raises: [CancelledError]), base, gcsafe .} = @@ -59,7 +59,11 @@ method mapNatPorts*( if not m.plumInitialized: # 5s matches the old NatPortMappingTimeout used with miniupnpc/libnatpmp. + let plumLogLevel = + if getEnv("DEBUG") == "1": PLUM_LOG_LEVEL_VERBOSE + else: PLUM_LOG_LEVEL_NONE let res = init( + logLevel = plumLogLevel, discoverTimeout = m.discoverTimeout, mappingTimeout = m.mappingTimeout, recheckPeriod = m.recheckPeriod, @@ -83,12 +87,12 @@ method mapNatPorts*( m.activeTcpPort = none(Port) m.activeUdpPort = none(Port) - let tcpRes = await createMapping(TCP, m.tcpPort.uint16) + let tcpRes = await createMapping(TCP, m.tcpPort.uint16, m.tcpPort.uint16) if tcpRes.isErr: warn "TCP port mapping failed", msg = tcpRes.error return none((Port, Port, MappingProtocol)) - let udpRes = await createMapping(UDP, m.discoveryPort.uint16) + let udpRes = await createMapping(UDP, m.discoveryPort.uint16, m.discoveryPort.uint16) if udpRes.isErr: warn "UDP port mapping failed", msg = udpRes.error destroyMapping(tcpRes.value.id) @@ -102,8 +106,28 @@ method mapNatPorts*( some((m.activeTcpPort.get, m.activeUdpPort.get, m.activeMappingProtocol.get)) +proc close*(m: NatPortMapper) = + if m.tcpMappingId.isSome: + destroyMapping(m.tcpMappingId.get) + m.tcpMappingId = none(cint) + + if m.udpMappingId.isSome: + destroyMapping(m.udpMappingId.get) + m.udpMappingId = none(cint) + + m.activeMappingProtocol = none(MappingProtocol) + m.activeTcpPort = none(Port) + m.activeUdpPort = none(Port) + + if m.plumInitialized: + discard cleanup() + m.plumInitialized = false + +proc isPortMapped*(m: NatPortMapper, port: Port): bool = + m.activeTcpPort.isSome and m.activeTcpPort.get == port + method handleNatStatus*( - m: NatMapper, + m: NatPortMapper, networkReachability: NetworkReachability, dialBackAddr: Opt[MultiAddress], discoveryPort: Port, @@ -134,12 +158,20 @@ method handleNatStatus*( if dialBackAddr.isNone: warn "Got empty dialback address in AutoNat when node is NotReachable" + elif m.tcpMappingId.isSome and m.udpMappingId.isSome: + warn "Not Reachable with active port mapping. The port mapping will be deleted and relay will start." + + # The mapping was created the the node is still not reachable. + # In that case, we delete the mapping and relay will start. + # We will keep retrying on the next iteration + m.close() + + # We remove the announced records. + # Eventually, it will we updated by the relay when it started + discovery.updateRecords(@[], udpPort = discoveryPort) else: debug "Node is not reachable trying port mapping now" - # Here we should check first that a mapping exists. - # If it does exist but Autonat still report as Not Reachable - # we should fallback to relay. let maybePorts = await m.mapNatPorts() if maybePorts.isSome: @@ -152,7 +184,7 @@ method handleNatStatus*( if autoRelayService.isRunning: # Here we stop the relay because the node *should* be reachable if not await autoRelayService.stop(switch): - debug "AutoRelayService stop method returned false" + debug "AutoRelayService returned an issue when trying to stop" else: debug "AutoRelayService stopped" @@ -160,36 +192,29 @@ method handleNatStatus*( # to false because we are not sure the node is reachable. # The client mode will be updated on the next iteration of autonat. # Trying to check manually that the node is reachable is not trivial, - # this is exactly what Autonat does. + # this is exactly what Autonat is for. discovery.updateRecords(@[announceAddress], udpPort = udpPort) hasPortMapping = true + else: + # In case of failure, close the port mapping in order to rerun discover + # on the next iteration + m.close() if not hasPortMapping and not autoRelayService.isRunning: debug "No port mapping found let's start autorelay" if not await autoRelayService.setup(switch): - warn "Cannot start autorelay service" + warn "Unable to start autorelay service" else: debug "AutoRelayService started" -proc close*(m: NatMapper) = - if m.tcpMappingId.isSome: - destroyMapping(m.tcpMappingId.get) - m.tcpMappingId = none(cint) - if m.udpMappingId.isSome: - destroyMapping(m.udpMappingId.get) - m.udpMappingId = none(cint) - if m.plumInitialized: - discard cleanup() - m.plumInitialized = false - proc reachabilityStr*(autonat: Option[AutonatV2Service]): string = if autonat.isSome: $autonat.get.networkReachability else: "unknown" -proc portMappingStr*(natMapper: Option[NatMapper]): string = +proc portMappingStr*(natMapper: Option[NatPortMapper]): string = if natMapper.isNone or natMapper.get.activeMappingProtocol.isNone: return "none" case natMapper.get.activeMappingProtocol.get diff --git a/storage/rest/api.nim b/storage/rest/api.nim index c125c7fb..d53d79c8 100644 --- a/storage/rest/api.nim +++ b/storage/rest/api.nim @@ -567,7 +567,7 @@ proc initDebugApi( conf: StorageConf, autonat: Option[AutonatV2Service], autoRelay: Option[AutoRelayService], - natMapper: Option[NatMapper], + natMapper: Option[NatPortMapper], natRouter: Option[NatRouter], router: var RestRouter, ) = @@ -679,7 +679,7 @@ proc initRestApi*( repoStore: RepoStore, autonat: Option[AutonatV2Service], autoRelay: Option[AutoRelayService], - natMapper: Option[NatMapper], + natMapper: Option[NatPortMapper], natRouter: Option[NatRouter], corsAllowedOrigin: ?string, ): RestRouter = diff --git a/storage/storage.nim b/storage/storage.nim index 0c975b5d..450587cc 100644 --- a/storage/storage.nim +++ b/storage/storage.nim @@ -58,7 +58,7 @@ type # Expose to make reachability accessible from rest api autonatService*: Option[AutonatV2Service] autoRelayService*: Option[AutoRelayService] - natMapper*: Option[NatMapper] + natMapper*: Option[NatPortMapper] natRouter*: Option[NatRouter] isStarted: bool @@ -85,6 +85,13 @@ proc start*(s: StorageServer) {.async.} = await s.storageNode.switch.start() + if s.natMapper.isSome and s.config.listenPort == Port(0): + for listenAddr in s.storageNode.switch.peerInfo.listenAddrs: + let maybePort = getTcpPort(listenAddr) + if maybePort.isSome: + s.natMapper.get.tcpPort = maybePort.get + break + let announceAddrs = if s.config.nat.hasExtIp: # extip means that we assume the IP is reachable @@ -100,7 +107,7 @@ proc start*(s: StorageServer) {.async.} = else: # Don't announce address and wait for AutoNat @[] - + info "info ", addrs = $s.storageNode.switch.peerInfo.addrs[0] if not s.config.nat.hasExtIp: # Nodes with autonat start with client mode. # It will be updated if reachable. @@ -201,35 +208,26 @@ proc new*( logFile: Option[IoHandle] = IoHandle.none, ): StorageServer = ## create StorageServer including setting up datastore, repostore, etc - let listenMultiAddr = getMultiAddrWithIpAndTcpPort(config.listenIp, config.listenPort) - let autonatClient = AutonatV2Client.new(random.Rng.instance()) - let autonatService = - if config.nat.hasExtIp: - none(AutonatV2Service) - else: - some( - AutonatV2Service.new( - rng = random.Rng.instance(), - client = autonatClient, - config = AutonatV2ServiceConfig.new( - scheduleInterval = Opt.some(config.natScheduleInterval), - askNewConnectedPeers = true, - numPeersToAsk = config.natNumPeersToAsk, - maxQueueSize = config.natMaxQueueSize, - minConfidence = config.natMinConfidence, - ), - ) - ) + # Guards + if config.autonatServer and not config.nat.hasExtIp: + raise newException(StorageError, "--autonat-server requires --extip") + + if config.isRelayServer and not config.autonatServer: + raise + newException(StorageError, "--relay-server is not compatible with autonat client") + + # Switch + let listenMultiAddr = getMultiAddrWithIpAndTcpPort(config.listenIp, config.listenPort) let relayClient = RelayClient.new() let relay: Relay = - if config.relay: + if config.isRelayServer: Relay.new() else: relayClient - let switchBuilder = SwitchBuilder + var switchBuilder = SwitchBuilder .new() .withPrivateKey(privateKey) .withAddresses(@[listenMultiAddr], enableWildcardResolver = true) @@ -240,18 +238,33 @@ proc new*( .withMaxConnections(config.maxPeers) .withAgentVersion(config.agentString) .withSignedPeerRecord(true) - .withAutonatV2Server() .withCircuitRelay(relay) - .withServices( - if autonatService.isSome: - @[Service(autonatService.get)] - else: - @[] + + if config.autonatServer: + info "AutoNAT server enabled" + switchBuilder = switchBuilder.withAutonatV2Server() + elif not config.nat.hasExtIp: + info "AutoNAT client enabled", + scheduleInterval = config.natScheduleInterval, + numPeersToAsk = config.natNumPeersToAsk, + maxQueueSize = config.natMaxQueueSize, + minConfidence = config.natMinConfidence + switchBuilder = switchBuilder.withAutonatV2( + AutonatV2ServiceConfig.new( + scheduleInterval = Opt.some(config.natScheduleInterval), + askNewConnectedPeers = true, + numPeersToAsk = config.natNumPeersToAsk, + maxQueueSize = config.natMaxQueueSize, + minConfidence = config.natMinConfidence, + ) ) + else: + info "AutoNAT disabled (extip configured)" var natRouter: Option[NatRouter] let switch = if config.natSimulation.isSome: + # Provide a NAT simulation useful for testing NAT Traversal let filtering = FilteringBehavior.fromString(config.natSimulation.get).valueOr( AddressAndPortDependent ) @@ -269,6 +282,14 @@ proc new*( autonatClient.setup(switch) switch.mount(autonatClient) + let autonatService: Option[AutonatV2Service] = + if switchBuilder.autonatV2Service.isSome: + some(switchBuilder.autonatV2Service.value) + else: + none(AutonatV2Service) + + # Storage infrastructure + try: if config.numThreads == ThreadCount(0): taskPool = Taskpool.new(numThreads = min(countProcessors(), 16)) @@ -383,12 +404,28 @@ proc new*( taskPool = taskPool, ) - var natMapper: Option[NatMapper] + switch.mount(network) + switch.mount(manifestProto) + + # NAT services + var natMapper: Option[NatPortMapper] var autoRelayService: Option[AutoRelayService] if autonatService.isSome: + let relayService = AutoRelayService.new( + maxNumRelays = config.natMaxRelays, + client = relayClient, + onReservation = proc(addresses: seq[MultiAddress]) {.gcsafe, raises: [].} = + info "Relay reservation updated", addresses + # relay addresses are for download traffic only, not DHT routing + discovery.updateAnnounceRecord(addresses), + rng = random.Rng.instance(), + ) + + autoRelayService = some(relayService) + natMapper = some( - NatMapper( + NatPortMapper( natConfig: config.nat, tcpPort: config.listenPort, discoveryPort: config.discoveryPort, @@ -397,17 +434,9 @@ proc new*( recheckPeriod: config.natPortMappingRecheckPeriod, ) ) - let relayService = AutoRelayService.new( - maxNumRelays = config.natMaxRelays, - client = relayClient, - onReservation = proc(addresses: seq[MultiAddress]) {.gcsafe, raises: [].} = - debug "Relay reservation updated", addresses - # relay addresses are for download traffic only, not DHT routing - discovery.updateAnnounceRecord(addresses), - rng = random.Rng.instance(), - ) - autoRelayService = some(relayService) + if natRouter.isSome: + natRouter.get.natMapper = natMapper autonatService.get.setStatusAndConfidenceHandler( proc( @@ -422,9 +451,7 @@ proc new*( ) ) - switch.mount(network) - switch.mount(manifestProto) - + # REST server var restServer: RestServerRef = nil if config.apiBindAddress.isSome: diff --git a/storage/utils/addrutils.nim b/storage/utils/addrutils.nim index 47204889..ae5441fa 100644 --- a/storage/utils/addrutils.nim +++ b/storage/utils/addrutils.nim @@ -59,6 +59,16 @@ proc getMultiAddrWithIPAndUDPPort*(ip: IpAddress, port: Port): MultiAddress = let ipFamily = if ip.family == IpAddressFamily.IPv4: "/ip4/" else: "/ip6/" return MultiAddress.init(ipFamily & $ip & "/udp/" & $port).expect("valid multiaddr") +func getTcpPort*(ma: MultiAddress): Option[Port] = + let parts = ($ma).split("/") + for i, part in parts: + if part == "tcp" and i + 1 < parts.len: + try: + return some(Port(parseInt(parts[i + 1]))) + except ValueError: + return Port.none + Port.none + proc getMultiAddrWithIpAndTcpPort*(ip: IpAddress, port: Port): MultiAddress = ## Creates a MultiAddress with the specified IP address and TCP port ## From 71dc55c78dce3f33066c9daa711c8c88ca2594c3 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Fri, 22 May 2026 21:14:06 +0400 Subject: [PATCH 058/167] Add double nat for simulation --- storage/utils/natsimulation.nim | 44 ++++++++++++++++++++++++++------- 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/storage/utils/natsimulation.nim b/storage/utils/natsimulation.nim index bce08a2a..0dd05f8a 100644 --- a/storage/utils/natsimulation.nim +++ b/storage/utils/natsimulation.nim @@ -1,6 +1,6 @@ {.push raises: [].} -import std/sequtils +import std/[options, sequtils] import pkg/chronos import pkg/results import pkg/libp2p @@ -8,14 +8,19 @@ import pkg/libp2p/transports/tcptransport import pkg/libp2p/transports/transport import pkg/libp2p/wire +import ../nat + type FilteringBehavior* = enum EndpointIndependent AddressDependent AddressAndPortDependent + DoubleNat type NatRouter* = ref object filtering*: FilteringBehavior conntrack: seq[TransportAddress] + natMapper*: Option[NatPortMapper] + dropTimeout*: Duration type NatTransport* = ref object of Transport tcp: TcpTransport @@ -31,20 +36,33 @@ proc fromString*( ok(AddressDependent) of "address-and-port-dependent": ok(AddressAndPortDependent) + of "double-nat": + ok(DoubleNat) else: err("Unknown filtering behavior: " & s) -proc new*(T: type NatRouter, filtering: FilteringBehavior): T = - T(filtering: filtering) +proc new*( + T: type NatRouter, filtering: FilteringBehavior, dropTimeout = 20.seconds +): T = + T(filtering: filtering, dropTimeout: dropTimeout) proc setFiltering*(r: NatRouter, filtering: FilteringBehavior) = r.filtering = filtering r.conntrack = @[] -proc allowInbound(r: NatRouter, remote: TransportAddress): bool = +proc allowInbound(r: NatRouter, remote: TransportAddress, localPort: Port): bool = case r.filtering + of DoubleNat: + return false of EndpointIndependent: - true + return true + else: + discard + + if r.natMapper.isSome and r.natMapper.get.isPortMapped(localPort): + return true + + case r.filtering of AddressDependent: r.conntrack.anyIt( try: @@ -54,6 +72,8 @@ proc allowInbound(r: NatRouter, remote: TransportAddress): bool = ) of AddressAndPortDependent: remote in r.conntrack + else: + false proc new*( T: type NatTransport, @@ -95,11 +115,11 @@ method dial*( return conn -proc dropAfterTimeout(conn: Connection) {.async: (raises: []).} = +proc dropAfterTimeout(conn: Connection, timeout: Duration) {.async: (raises: []).} = # Hold the connection open long enough for the remote's dial to time out, # then close it. This simulates a NAT that drops packets rather than RSTs # them, which is what AutoNAT needs to detect NotReachable. - await noCancel sleepAsync(20.seconds) + await noCancel sleepAsync(timeout) await noCancel conn.close() method accept*( @@ -121,11 +141,17 @@ method accept*( await conn.close() continue - if not self.router.allowInbound(transportAddr.get): + var localPort = Port(0) + if self.addrs.len > 0: + let localAddr = initTAddress(self.addrs[0]) + if localAddr.isOk: + localPort = localAddr.get.port + + if not self.router.allowInbound(transportAddr.get, localPort): # Do not close immediately: let the remote's dial time out naturally, # then clean up. Returning a fast RST would produce EDialRefused (Unknown) # instead of EDialError (NotReachable) in AutoNAT. - asyncSpawn dropAfterTimeout(conn) + asyncSpawn dropAfterTimeout(conn, self.router.dropTimeout) continue return conn From 283fea80a2180281f50f3aa9a9e8afbc0b525e4e Mon Sep 17 00:00:00 2001 From: Arnaud Date: Fri, 22 May 2026 21:16:57 +0400 Subject: [PATCH 059/167] Provide better testing and add separate upnp and pcp test on miniupnpd --- .github/workflows/ci-reusable.yml | 10 ++- Makefile | 13 +++- build.nims | 13 +++- tests/integration/1_minute/testnat.nim | 33 +++++--- tests/integration/multinodes.nim | 2 +- tests/integration/nat/Dockerfile | 2 +- tests/integration/nat/docker-entrypoint.sh | 31 ++++++-- tests/integration/nathelper.nim | 29 +++++-- tests/integration/storageconfig.nim | 64 ++------------- tests/nat/testnatpcp.nim | 90 ++++++++++++++++++++++ tests/nat/testnatupnp.nim | 72 +++++++++++++---- tests/storage/testaddrutils.nim | 17 ++++ tests/storage/testnat.nim | 14 ++-- tests/storage/testnatsimulation.nim | 63 ++++++++++++++- tools/scripts/ci-job-matrix.sh | 11 ++- 15 files changed, 341 insertions(+), 123 deletions(-) create mode 100644 tests/nat/testnatpcp.nim diff --git a/.github/workflows/ci-reusable.yml b/.github/workflows/ci-reusable.yml index 2d775f1b..89d42b00 100644 --- a/.github/workflows/ci-reusable.yml +++ b/.github/workflows/ci-reusable.yml @@ -69,9 +69,13 @@ jobs: run: make -j${ncpu} testLibstorage ## Part 4 Tests ## - - name: NAT integration tests - if: matrix.tests == 'nat-integration' - run: make testNatIntegration + - name: NAT UPnP integration tests + if: matrix.tests == 'nat-upnp-integration' + run: make testNatUpnpIntegration + + - name: NAT PCP integration tests + if: matrix.tests == 'nat-pcp-integration' + run: make testNatPcpIntegration status: if: always() diff --git a/Makefile b/Makefile index a0292028..71319a00 100644 --- a/Makefile +++ b/Makefile @@ -87,7 +87,8 @@ endif testAll \ testIntegration \ testLibstorage \ - testNatIntegration \ + testNatUpnpIntegration \ + testNatPcpIntegration \ update ifeq ($(NIM_PARAMS),) @@ -149,11 +150,15 @@ testIntegration: | build deps echo -e $(BUILD_MSG) "build/$@" && \ $(ENV_SCRIPT) nim testIntegration $(TEST_PARAMS) $(NIM_PARAMS) build.nims -# Builds and runs the UPnP NAT integration test inside a miniupnpd container DOCKER := $(or $(shell which podman 2>/dev/null), $(shell which docker 2>/dev/null)) -testNatIntegration: + +testNatUpnpIntegration: $(DOCKER) build -t miniupnpd-test -f tests/integration/nat/Dockerfile . - $(DOCKER) run --rm --cap-add NET_ADMIN miniupnpd-test + $(DOCKER) run --rm --cap-add NET_ADMIN -e DEBUG=$(DEBUG) miniupnpd-test + +testNatPcpIntegration: + $(DOCKER) build -t miniupnpd-test -f tests/integration/nat/Dockerfile . + $(DOCKER) run --rm --cap-add NET_ADMIN -e DEBUG=$(DEBUG) -e TEST_PCP=1 miniupnpd-test # Builds a C example that uses the libstorage C library and runs it testLibstorage: | build deps diff --git a/build.nims b/build.nims index 484d365a..d694206d 100644 --- a/build.nims +++ b/build.nims @@ -85,13 +85,22 @@ task testNatPortMapping, "Run UPnP NAT integration test (requires miniupnpd cont putEnv("STORAGE_INTEGRATION_TEST_INCLUDES", "nat/testnatupnp.nim") test "testIntegration", outName = "testIntegrationNat" -# Used to build the testing binarie in Docker -task buildNatPortMappingBinaries, "Build UPnP NAT test binaries without running them": +task testNatPcpMapping, "Run PCP NAT integration test (requires miniupnpd container)": + buildBinary "storage", + outName = "storage", + params = "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE" + putEnv("STORAGE_INTEGRATION_TEST_INCLUDES", "nat/testnatpcp.nim") + test "testIntegration", outName = "testIntegrationNatPcp" + +# Used to build the testing binaries in Docker +task buildNatPortMappingBinaries, "Build UPnP and PCP NAT test binaries without running them": buildBinary "storage", outName = "storage", params = "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE" putEnv("STORAGE_INTEGRATION_TEST_INCLUDES", "nat/testnatupnp.nim") buildBinary "testIntegration", outName = "testIntegrationNat", srcDir = "tests/" + putEnv("STORAGE_INTEGRATION_TEST_INCLUDES", "nat/testnatpcp.nim") + buildBinary "testIntegration", outName = "testIntegrationNatPcp", srcDir = "tests/" task build, "build Logos Storage binary": storageTask() diff --git a/tests/integration/1_minute/testnat.nim b/tests/integration/1_minute/testnat.nim index abf40928..b21d8c64 100644 --- a/tests/integration/1_minute/testnat.nim +++ b/tests/integration/1_minute/testnat.nim @@ -24,7 +24,7 @@ multinodesuite "AutoNAT detection": ) test "node is reachable when using bootstrap node on same network", natConfig: let node2 = clients()[1] - await node2.client.checkNatStatus("Reachable") + await node2.client.checkReachable() let endpointIndependentConfig = NodeConfigs( clients: StorageConfigs @@ -39,7 +39,7 @@ multinodesuite "AutoNAT detection": # EIF = Endpoint Independent Filtering test "node with simulated EIF nat is detected as reachable", endpointIndependentConfig: let node2 = clients()[1] - await node2.client.checkNatStatus("Reachable") + await node2.client.checkReachable() let autonatConfig = NodeConfigs( clients: StorageConfigs @@ -55,7 +55,7 @@ multinodesuite "AutoNAT detection": test "node with simulated APDF nat is detected as not reachable and starts relay", autonatConfig: let node2 = clients()[1] - await node2.client.checkNatStatus("NotReachable") + await node2.client.checkNotReachable() let transitionConfig = NodeConfigs( clients: StorageConfigs @@ -73,11 +73,11 @@ multinodesuite "AutoNAT detection": transitionConfig: let node2 = clients()[1] - await node2.client.checkNatStatus("NotReachable") + await node2.client.checkNotReachable() check (await node2.client.setNatFiltering("endpoint-independent")).isOk - await node2.client.checkNatStatus("Reachable") + await node2.client.checkReachable() let natToSimConfig = NodeConfigs( clients: StorageConfigs @@ -94,11 +94,26 @@ multinodesuite "AutoNAT detection": natToSimConfig: let node2 = clients()[1] - await node2.client.checkNatStatus("Reachable") + await node2.client.checkReachable() check (await node2.client.setNatFiltering("address-and-port-dependent")).isOk - await node2.client.checkNatStatus("NotReachable") + await node2.client.checkNotReachable() + + let doubleNatConfig = NodeConfigs( + clients: StorageConfigs + .init(nodes = 2) + .withRelay(0) + .withNatSimulation(idx = 1, "double-nat") + .withNatNumPeersToAsk(1) + .withNatMinConfidence(0.5) + .withNatScheduleInterval(5.seconds) + .withNatMaxQueueSize(1).some + ) + test "node behind double NAT is detected as not reachable and starts relay", + doubleNatConfig: + let node2 = clients()[1] + await node2.client.checkNotReachable() let multiNatConfig = NodeConfigs( clients: StorageConfigs @@ -117,5 +132,5 @@ multinodesuite "AutoNAT detection": let node2 = clients()[1] let node3 = clients()[2] - await node2.client.checkNatStatus("NotReachable") - await node3.client.checkNatStatus("NotReachable") + await node2.client.checkNotReachable() + await node3.client.checkNotReachable() diff --git a/tests/integration/multinodes.nim b/tests/integration/multinodes.nim index a0c70cc9..75fdb9d1 100644 --- a/tests/integration/multinodes.nim +++ b/tests/integration/multinodes.nim @@ -215,7 +215,7 @@ template multinodesuite*(suiteName: string, body: untyped) = failAndTeardownOnError "failed to start client nodes": # Only the first node (bootstrap) gets a known extip. Other nodes use # nat=auto so AutoNAT can run and determine their reachability. - clients = clients.withExtIp(0) + clients = clients.withExtIp(0).withAutonatServer(0) for config in clients.configs: let node = await startClientNode(config) running.add RunningNode(role: Role.Client, node: node) diff --git a/tests/integration/nat/Dockerfile b/tests/integration/nat/Dockerfile index 2b7f9075..503e86cc 100644 --- a/tests/integration/nat/Dockerfile +++ b/tests/integration/nat/Dockerfile @@ -9,7 +9,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && rm -rf /var/lib/apt/lists/* # Build miniupnpd with stub redirector: no iptables/nftables needed, always -# returns success for port mapping requests — safe for isolated test containers. +# returns success for port mapping requests. COPY tests/integration/nat/miniupnpd_stub_rdr.c /tmp/stub_rdr.c RUN git clone --depth=1 --branch miniupnpd_2_3_9 \ https://github.com/miniupnp/miniupnp.git /tmp/miniupnp \ diff --git a/tests/integration/nat/docker-entrypoint.sh b/tests/integration/nat/docker-entrypoint.sh index 1be6b923..b71e64d3 100644 --- a/tests/integration/nat/docker-entrypoint.sh +++ b/tests/integration/nat/docker-entrypoint.sh @@ -5,24 +5,41 @@ RUNDIR=/tmp/miniupnpd mkdir -p "$RUNDIR" LAN_IF=$(ip route show default | awk '/default/{print $5; exit}') +LAN_IP=$(ip -4 addr show "$LAN_IF" | awk '/inet /{print $2; exit}' | cut -d/ -f1) ip link add plum-wan type dummy ip addr add 1.2.3.4/24 dev plum-wan ip link set plum-wan up -cat > "$RUNDIR/miniupnpd.conf" << EOF +start_miniupnpd() { + local enable_pcp_pmp=$1 + cat > "$RUNDIR/miniupnpd.conf" << EOF ext_ifname=plum-wan listening_ip=$LAN_IF -enable_pcp_pmp=no +enable_pcp_pmp=$enable_pcp_pmp port=0 allow 1024-65535 0.0.0.0/0 1024-65535 EOF + miniupnpd -d -f "$RUNDIR/miniupnpd.conf" > "$RUNDIR/miniupnpd.log" 2>&1 & + sleep 1 +} + +if [[ "${TEST_PCP:-0}" == "1" ]]; then + # PCP requires the UDP source IP to match the client_address in the MAP request. + # Point the default route at LAN_IP so libplum uses it as both gateway and PCP target. + ip route replace default via "$LAN_IP" dev "$LAN_IF" + start_miniupnpd yes + failed=0 + DEBUG=${DEBUG:-0} /app/build/testIntegrationNatPcp || failed=1 +else + start_miniupnpd no + failed=0 + DEBUG=${DEBUG:-0} /app/build/testIntegrationNat || failed=1 +fi if [[ "${DEBUG:-0}" == "1" ]]; then - miniupnpd -d -f "$RUNDIR/miniupnpd.conf" & -else - miniupnpd -d -f "$RUNDIR/miniupnpd.conf" > /dev/null 2>&1 & + echo "--- miniupnpd log ---" + cat "$RUNDIR/miniupnpd.log" 2>/dev/null || true fi -sleep 1 -/app/build/testIntegrationNat +[ $failed -eq 0 ] || exit 1 diff --git a/tests/integration/nathelper.nim b/tests/integration/nathelper.nim index 68f21176..82caadd4 100644 --- a/tests/integration/nathelper.nim +++ b/tests/integration/nathelper.nim @@ -19,16 +19,31 @@ proc checkNatStatus*( let info = (await client.info()).get let nat = info["nat"] let addrs = info["addrs"].getElems.mapIt(it.getStr) - nat["reachability"].getStr() == reachability and - nat["clientMode"].getBool() == clientMode and - nat["relayRunning"].getBool() == relayRunning and - addrs.anyIt("p2p-circuit" in it) == relayRunning, + let r = nat["reachability"].getStr() + let cm = nat["clientMode"].getBool() + let rr = nat["relayRunning"].getBool() + let ha = addrs.anyIt("p2p-circuit" in it) + let pm = nat["portMapping"].getStr() + + # It is important to check all the conditions together to avoid race + # (new autonat iteration) + # So we add a checkoint for better debug in case of failures + checkpoint( + "reachability=" & r & " (want " & reachability & ")" & " clientMode=" & $cm & + " (want " & $clientMode & ")" & " relayRunning=" & $rr & " (want " & + $relayRunning & ")" & " p2p-circuit=" & $ha & " (want " & $relayRunning & ")" & + " portMapping=" & pm + ) + r == reachability and cm == clientMode and rr == relayRunning and + ha == relayRunning, timeout = RelayTimeout, pollInterval = PollInterval, ) -proc checkNatStatus*(client: StorageClient, reachability: string) {.async.} = - let notReachable = reachability == "NotReachable" +proc checkReachable*(client: StorageClient) {.async.} = + await client.checkNatStatus("Reachable", relayRunning = false, clientMode = false) + +proc checkNotReachable*(client: StorageClient, relayRunning = true) {.async.} = await client.checkNatStatus( - reachability, relayRunning = notReachable, clientMode = notReachable + "NotReachable", relayRunning = relayRunning, clientMode = true ) diff --git a/tests/integration/storageconfig.nim b/tests/integration/storageconfig.nim index b509c0dd..c3ab9ea7 100644 --- a/tests/integration/storageconfig.nim +++ b/tests/integration/storageconfig.nim @@ -282,22 +282,6 @@ proc withStorageQuota*( config.addCliOption("--storage-quota", $quota) return startConfig -proc withListenIp*( - self: StorageConfigs, ip: string -): StorageConfigs {.raises: [StorageConfigError].} = - var startConfig = self - for config in startConfig.configs.mitems: - config.addCliOption("--listen-ip", ip) - return startConfig - -proc withListenPort*( - self: StorageConfigs, idx: int, port: int -): StorageConfigs {.raises: [StorageConfigError].} = - self.checkBounds idx - var startConfig = self - startConfig.configs[idx].addCliOption("--listen-port", $port) - return startConfig - proc withNatNumPeersToAsk*( self: StorageConfigs, numPeersToAsk: int ): StorageConfigs {.raises: [StorageConfigError].} = @@ -330,30 +314,6 @@ proc withNatScheduleInterval*( config.addCliOption("--nat-schedule-interval", $scheduleInterval) return startConfig -proc withNatPortMappingDiscoverTimeout*( - self: StorageConfigs, timeout: int -): StorageConfigs {.raises: [StorageConfigError].} = - var startConfig = self - for config in startConfig.configs.mitems: - config.addCliOption("--nat-port-mapping-discover-timeout", $timeout) - return startConfig - -proc withNatPortMappingTimeout*( - self: StorageConfigs, timeout: int -): StorageConfigs {.raises: [StorageConfigError].} = - var startConfig = self - for config in startConfig.configs.mitems: - config.addCliOption("--nat-port-mapping-timeout", $timeout) - return startConfig - -proc withNatPortMappingRecheckPeriod*( - self: StorageConfigs, timeout: int -): StorageConfigs {.raises: [StorageConfigError].} = - var startConfig = self - for config in startConfig.configs.mitems: - config.addCliOption("--nat-port-mapping-recheck-period", $timeout) - return startConfig - proc withExtIp*( self: StorageConfigs, idx: int, ip = "127.0.0.1" ): StorageConfigs {.raises: [StorageConfigError].} = @@ -363,27 +323,13 @@ proc withExtIp*( startConfig.configs[idx].addCliOption("--nat", "extip:" & ip) return startConfig -proc withExtIp*( - self: StorageConfigs, ip = "127.0.0.1" -): StorageConfigs {.raises: [StorageConfigError].} = - var startConfig = self - for config in startConfig.configs.mitems: - config.addCliOption("--nat", "extip:" & ip) - return startConfig - proc withRelay*( self: StorageConfigs, idx: int ): StorageConfigs {.raises: [StorageConfigError].} = self.checkBounds idx var startConfig = self - startConfig.configs[idx].addCliOption("--relay") - return startConfig - -proc withRelay*(self: StorageConfigs): StorageConfigs {.raises: [StorageConfigError].} = - var startConfig = self - for config in startConfig.configs.mitems: - config.addCliOption("--relay") + startConfig.configs[idx].addCliOption("--relay-server") return startConfig # For testing, a node with extip (not behind nat) is a bootstrap node @@ -408,10 +354,10 @@ proc withNatSimulation*( startConfig.configs[idx].addCliOption("--nat-simulation", filtering) return startConfig -proc withNatSimulation*( - self: StorageConfigs, filtering: string +proc withAutonatServer*( + self: StorageConfigs, idx: int ): StorageConfigs {.raises: [StorageConfigError].} = + self.checkBounds idx var startConfig = self - for config in startConfig.configs.mitems: - config.addCliOption("--nat-simulation", filtering) + startConfig.configs[idx].addCliOption("--autonat-server") return startConfig diff --git a/tests/nat/testnatpcp.nim b/tests/nat/testnatpcp.nim new file mode 100644 index 00000000..02deecd2 --- /dev/null +++ b/tests/nat/testnatpcp.nim @@ -0,0 +1,90 @@ +import std/[json, strutils, sequtils] +import pkg/chronos +import pkg/questionable/results + +import ../integration/multinodes +import ../integration/storageclient +import ../integration/storageconfig + +import ../integration/nathelper + +multinodesuite "AutoNAT PCP port mapping": + let pcpConfig = NodeConfigs( + clients: StorageConfigs + .init(nodes = 2) + .withRelay(0) + .withNatSimulation(idx = 1, "address-and-port-dependent") + .withNatNumPeersToAsk(1) + .withNatMinConfidence(0.5) + .withNatScheduleInterval(10.seconds) + .withNatMaxQueueSize(1).some + ) + + test "node behind NAT maps ports via PCP and exposes mapping in debug info", pcpConfig: + let node2 = clients()[1] + + await node2.client.checkNotReachable(relayRunning = false) + + check eventuallySafe( + block: + let res = await node2.client.natPortMapping() + res.isOk and res.get == "pcp", + timeout = RelayTimeout, + pollInterval = PollInterval, + ) + + await node2.client.checkReachable() + + await node2.stop() + + let relayFallbackConfig = NodeConfigs( + clients: StorageConfigs + .init(nodes = 2) + .withRelay(0) + .withNatSimulation(idx = 1, "double-nat") + .withNatNumPeersToAsk(1) + .withNatMinConfidence(0.5) + .withNatScheduleInterval(10.seconds) + # Increase the max queue to trigger the AutoNat 2 times + .withNatMaxQueueSize(2).some + ) + + test "node behind double NAT falls back to relay after PCP mapping does not help", + relayFallbackConfig: + let node2 = clients()[1] + + await node2.client.checkNotReachable(relayRunning = false) + + check eventuallySafe( + block: + let res = await node2.client.natPortMapping() + res.isOk and res.get == "pcp", + timeout = RelayTimeout, + pollInterval = PollInterval, + ) + + await node2.client.checkNotReachable() + + test "reachable node downloads content uploaded by node behind NAT after PCP mapping", + pcpConfig: + let node1 = clients()[0] + let node2 = clients()[1] + + check eventuallySafe( + block: + let res = await node2.client.natPortMapping() + res.isOk and res.get == "pcp", + timeout = RelayTimeout, + pollInterval = PollInterval, + ) + + let content = "content uploaded by nat node" + let cid = (await node2.client.upload(content)).get + + check eventuallySafe( + (await node1.client.download(cid)).isOk, + timeout = RelayTimeout, + pollInterval = PollInterval, + ) + + check (await node1.client.download(cid)).get == content diff --git a/tests/nat/testnatupnp.nim b/tests/nat/testnatupnp.nim index 9552ae71..e36def22 100644 --- a/tests/nat/testnatupnp.nim +++ b/tests/nat/testnatupnp.nim @@ -8,30 +8,23 @@ import ../integration/storageconfig import ../integration/nathelper -const DetectionTimeout = 15_000 - multinodesuite "AutoNAT UPnP port mapping": let upnpConfig = NodeConfigs( clients: StorageConfigs .init(nodes = 2) .withRelay(0) .withNatSimulation(idx = 1, "address-and-port-dependent") - .withListenPort(idx = 1, 8102) .withNatNumPeersToAsk(1) .withNatMinConfidence(0.5) .withNatScheduleInterval(10.seconds) - .withNatMaxQueueSize(1) - .withLogFile() - .withLogLevel(idx = 1, LogLevel.DEBUG).some + .withNatMaxQueueSize(1).some ) test "node behind NAT maps ports via UPnP and exposes mapping in debug info", upnpConfig: let node2 = clients()[1] - await node2.client.checkNatStatus( - "NotReachable", relayRunning = false, clientMode = true - ) + await node2.client.checkNotReachable(relayRunning = false) check eventuallySafe( block: @@ -41,13 +34,58 @@ multinodesuite "AutoNAT UPnP port mapping": pollInterval = PollInterval, ) - await node2.client.checkNatStatus( - "NotReachable", relayRunning = false, clientMode = true - ) - - let announceAddrs = - (await node2.client.info()).get["announceAddresses"].getElems.mapIt(it.getStr) - let tcpAddr = announceAddrs.filterIt(it.startsWith("/ip4/") and "/tcp/" in it) - check tcpAddr.len > 0 + await node2.client.checkReachable() await node2.stop() + + let relayFallbackConfig = NodeConfigs( + clients: StorageConfigs + .init(nodes = 2) + .withRelay(0) + .withNatSimulation(idx = 1, "double-nat") + .withNatNumPeersToAsk(1) + .withNatMinConfidence(0.5) + .withNatScheduleInterval(10.seconds) + # Increase the max queue to trigger the AutoNat 2 times + .withNatMaxQueueSize(2).some + ) + + test "node behind double NAT falls back to relay after UPnP mapping does not help", + relayFallbackConfig: + let node2 = clients()[1] + + await node2.client.checkNotReachable(relayRunning = false) + + check eventuallySafe( + block: + let res = await node2.client.natPortMapping() + res.isOk and res.get == "upnp", + timeout = RelayTimeout, + pollInterval = PollInterval, + ) + + await node2.client.checkNotReachable() + + test "reachable node downloads content uploaded by node behind NAT after UPnP mapping", + upnpConfig: + let node1 = clients()[0] + let node2 = clients()[1] + + check eventuallySafe( + block: + let res = await node2.client.natPortMapping() + res.isOk and res.get == "upnp", + timeout = RelayTimeout, + pollInterval = PollInterval, + ) + + let content = "content uploaded by nat node" + let cid = (await node2.client.upload(content)).get + + check eventuallySafe( + (await node1.client.download(cid)).isOk, + timeout = RelayTimeout, + pollInterval = PollInterval, + ) + + check (await node1.client.download(cid)).get == content diff --git a/tests/storage/testaddrutils.nim b/tests/storage/testaddrutils.nim index 3444f55a..b336fb1a 100644 --- a/tests/storage/testaddrutils.nim +++ b/tests/storage/testaddrutils.nim @@ -3,6 +3,23 @@ import pkg/libp2p/multiaddress import ../asynctest import ../../storage/utils/addrutils +suite "addrutils - getTcpPort": + test "extracts port from ipv4 tcp address": + let ma = MultiAddress.init("/ip4/1.2.3.4/tcp/5000").expect("valid") + check getTcpPort(ma) == some(Port(5000)) + + test "extracts port from ipv6 tcp address": + let ma = MultiAddress.init("/ip6/::1/tcp/8080").expect("valid") + check getTcpPort(ma) == some(Port(8080)) + + test "returns none for udp address": + let ma = MultiAddress.init("/ip4/1.2.3.4/udp/5000").expect("valid") + check getTcpPort(ma) == Port.none + + test "extracts port 0": + let ma = MultiAddress.init("/ip4/0.0.0.0/tcp/0").expect("valid") + check getTcpPort(ma) == some(Port(0)) + suite "addrutils - remapAddr": test "replaces protocol tcp with udp": let ma = MultiAddress.init("/ip4/1.2.3.4/tcp/5000").expect("valid") diff --git a/tests/storage/testnat.nim b/tests/storage/testnat.nim index c1af8eaf..0808cf2d 100644 --- a/tests/storage/testnat.nim +++ b/tests/storage/testnat.nim @@ -14,11 +14,11 @@ import ../../storage/discovery import ../../storage/rng import ../../storage/utils -type MockNatMapper = ref object of NatMapper +type MockNatPortMapper = ref object of NatPortMapper mappedPorts: Option[(Port, Port, MappingProtocol)] method mapNatPorts*( - m: MockNatMapper + m: MockNatPortMapper ): Future[Option[(Port, Port, MappingProtocol)]] {. async: (raises: [CancelledError]), gcsafe .} = @@ -49,7 +49,7 @@ asyncchecksuite "NAT - handleNatStatus": test "handleNatStatus announces mapped address when NotReachable and UPnP succeeds": let dialBack = MultiAddress.init("/ip4/1.2.3.4/tcp/8080").expect("valid") let mapper = - MockNatMapper(mappedPorts: some((Port(9000), Port(9001), MappingProtocol.UPnP))) + MockNatPortMapper(mappedPorts: some((Port(9000), Port(9001), MappingProtocol.UPnP))) await mapper.handleNatStatus( NotReachable, Opt.some(dialBack), discoveryPort, disc, sw, autoRelay @@ -61,7 +61,7 @@ asyncchecksuite "NAT - handleNatStatus": check disc.protocol.clientMode test "handleNatStatus starts autoRelay when NotReachable and UPnP failed": - let mapper = MockNatMapper(mappedPorts: none((Port, Port, MappingProtocol))) + let mapper = MockNatPortMapper(mappedPorts: none((Port, Port, MappingProtocol))) await mapper.handleNatStatus( NotReachable, Opt.none(MultiAddress), discoveryPort, disc, sw, autoRelay @@ -72,7 +72,7 @@ asyncchecksuite "NAT - handleNatStatus": test "handleNatStatus starts autoRelay when NotReachable and mapping fails": let dialBack = MultiAddress.init("/ip4/1.2.3.4/tcp/8080").expect("valid") - let mapper = MockNatMapper(mappedPorts: none((Port, Port, MappingProtocol))) + let mapper = MockNatPortMapper(mappedPorts: none((Port, Port, MappingProtocol))) await mapper.handleNatStatus( NotReachable, Opt.some(dialBack), discoveryPort, disc, sw, autoRelay @@ -83,7 +83,7 @@ asyncchecksuite "NAT - handleNatStatus": check disc.protocol.clientMode test "handleNatStatus does not announce address when Reachable and no dialBackAddr": - let mapper = MockNatMapper(mappedPorts: none((Port, Port, MappingProtocol))) + let mapper = MockNatPortMapper(mappedPorts: none((Port, Port, MappingProtocol))) await mapper.handleNatStatus( Reachable, Opt.none(MultiAddress), discoveryPort, disc, sw, autoRelay @@ -95,7 +95,7 @@ asyncchecksuite "NAT - handleNatStatus": test "handleNatStatus stops relay and announces dialBackAddr when Reachable": let dialBack = MultiAddress.init("/ip4/1.2.3.4/tcp/8080").expect("valid") - let mapper = MockNatMapper(mappedPorts: none((Port, Port, MappingProtocol))) + let mapper = MockNatPortMapper(mappedPorts: none((Port, Port, MappingProtocol))) discard await autorelayservice.setup(autoRelay, sw) await mapper.handleNatStatus( diff --git a/tests/storage/testnatsimulation.nim b/tests/storage/testnatsimulation.nim index 1563446a..81c51d4b 100644 --- a/tests/storage/testnatsimulation.nim +++ b/tests/storage/testnatsimulation.nim @@ -1,8 +1,11 @@ +import std/net import pkg/chronos +import pkg/libp2p/wire import ./helpers import ../asynctest import ../../storage/rng +import ../../storage/nat import ../../storage/utils/natsimulation const flags = {ServerFlags.ReuseAddr} @@ -52,7 +55,7 @@ asyncchecksuite "NatTransport - Address-Dependent Filtering": var bootstrap, thirdNode, natNode: Switch setup: - let router = NatRouter.new(AddressDependent) + let router = NatRouter.new(AddressDependent, dropTimeout = 1.seconds) bootstrap = newSwitch(Rng.instance()) thirdNode = newSwitch(Rng.instance()) natNode = newNatSwitch(router, Rng.instance()) @@ -85,7 +88,7 @@ asyncchecksuite "NatTransport - Address-and-Port-Dependent Filtering": var bootstrap, thirdNode, natNode: Switch setup: - let router = NatRouter.new(AddressAndPortDependent) + let router = NatRouter.new(AddressAndPortDependent, dropTimeout = 1.seconds) bootstrap = newSwitch(Rng.instance()) thirdNode = newSwitch(Rng.instance()) natNode = newNatSwitch(router, Rng.instance()) @@ -113,3 +116,59 @@ asyncchecksuite "NatTransport - Address-and-Port-Dependent Filtering": await natNode.connect(bootstrap.peerInfo.peerId, bootstrap.peerInfo.addrs) expect(LPError): await thirdNode.connect(natNode.peerInfo.peerId, natNode.peerInfo.addrs) + +asyncchecksuite "NatTransport - Double NAT": + var bootstrap, natNode: Switch + var router: NatRouter + + setup: + router = NatRouter.new(DoubleNat, dropTimeout = 1.seconds) + bootstrap = newSwitch(Rng.instance()) + natNode = newNatSwitch(router, Rng.instance()) + await bootstrap.start() + await natNode.start() + + teardown: + await bootstrap.stop() + await natNode.stop() + + test "bootstrap cannot connect to nat node regardless of port mapping": + let actualPort = initTAddress(natNode.peerInfo.addrs[0]).get().port + let natMapper = NatPortMapper() + natMapper.activeTcpPort = some(actualPort) + router.natMapper = some(natMapper) + + expect(LPError): + await bootstrap.connect(natNode.peerInfo.peerId, natNode.peerInfo.addrs) + +asyncchecksuite "NatTransport - Port Mapping": + var bootstrap, natNode: Switch + var router: NatRouter + + setup: + router = NatRouter.new(AddressAndPortDependent, dropTimeout = 1.seconds) + bootstrap = newSwitch(Rng.instance()) + natNode = newNatSwitch(router, Rng.instance()) + await bootstrap.start() + await natNode.start() + + teardown: + await bootstrap.stop() + await natNode.stop() + + test "bootstrap can connect to nat node when port mapping matches listen port": + let actualPort = initTAddress(natNode.peerInfo.addrs[0]).get().port + let natMapper = NatPortMapper() + natMapper.activeTcpPort = some(actualPort) + router.natMapper = some(natMapper) + + await bootstrap.connect(natNode.peerInfo.peerId, natNode.peerInfo.addrs) + check bootstrap.isConnected(natNode.peerInfo.peerId) + + test "bootstrap cannot connect to nat node when port mapping does not match": + let natMapper = NatPortMapper() + natMapper.activeTcpPort = some(Port(1)) + router.natMapper = some(natMapper) + + expect(LPError): + await bootstrap.connect(natNode.peerInfo.peerId, natNode.peerInfo.addrs) diff --git a/tools/scripts/ci-job-matrix.sh b/tools/scripts/ci-job-matrix.sh index 455234c4..15b5db96 100755 --- a/tools/scripts/ci-job-matrix.sh +++ b/tools/scripts/ci-job-matrix.sh @@ -117,10 +117,13 @@ libstorage_test () { job } -# outputs a NAT integration test job +# outputs NAT integration test jobs # Linux-only: miniupnpd is a Linux daemon, network namespace manipulation requires Linux -nat_integration_test () { - job_tests="nat-integration" +nat_integration_tests () { + job_tests="nat-upnp-integration" + job_includes="" + job + job_tests="nat-pcp-integration" job_includes="" job } @@ -131,7 +134,7 @@ all_tests () { integration_test libstorage_test if [ "$job_os" = "linux" ]; then - nat_integration_test + nat_integration_tests fi } From 0ffee3d14f65c0fe952269ff05323d80e2618546 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Fri, 22 May 2026 21:33:06 +0400 Subject: [PATCH 060/167] Export port --- storage/nat.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/storage/nat.nim b/storage/nat.nim index 9842b838..fcf07234 100644 --- a/storage/nat.nim +++ b/storage/nat.nim @@ -40,8 +40,8 @@ type NatPortMapper* = ref object of RootObj tcpMappingId: Option[cint] udpMappingId: Option[cint] activeMappingProtocol*: Option[MappingProtocol] - activeTcpPort: Option[Port] - activeUdpPort: Option[Port] + activeTcpPort*: Option[Port] + activeUdpPort*: Option[Port] plumInitialized: bool method mapNatPorts*( From 03933c2b3a838e6ee8f80b81886e91092355584e Mon Sep 17 00:00:00 2001 From: Arnaud Date: Fri, 22 May 2026 22:54:07 +0400 Subject: [PATCH 061/167] Add hole punching --- storage/nat.nim | 80 ++++++++++++++++++++++++++++++++++++++- storage/storage.nim | 2 + tests/storage/testnat.nim | 35 ++++++++++++++++- 3 files changed, 113 insertions(+), 4 deletions(-) diff --git a/storage/nat.nim b/storage/nat.nim index fcf07234..b1729713 100644 --- a/storage/nat.nim +++ b/storage/nat.nim @@ -16,6 +16,10 @@ import pkg/chronicles import pkg/libp2p import pkg/libp2p/services/autorelayservice import pkg/libp2p/protocols/connectivity/autonatv2/service +import pkg/libp2p/protocols/connectivity/relay/relay as relayProtocol +import pkg/libp2p/protocols/connectivity/dcutr/client as dcutrClientModule +import pkg/libp2p/protocols/connectivity/dcutr/server as dcutrServerModule +import pkg/libp2p/wire import ./utils import ./utils/natutils @@ -60,8 +64,7 @@ method mapNatPorts*( if not m.plumInitialized: # 5s matches the old NatPortMappingTimeout used with miniupnpc/libnatpmp. let plumLogLevel = - if getEnv("DEBUG") == "1": PLUM_LOG_LEVEL_VERBOSE - else: PLUM_LOG_LEVEL_NONE + if getEnv("DEBUG") == "1": PLUM_LOG_LEVEL_VERBOSE else: PLUM_LOG_LEVEL_NONE let res = init( logLevel = plumLogLevel, discoverTimeout = m.discoverTimeout, @@ -229,3 +232,76 @@ proc findReachableNodes*(bootstrapNodes: seq[SignedPeerRecord]): seq[SignedPeerR ## Currently returns bootstrap nodes. In the future, any network participant ## confirmed reachable by AutoNAT could be included. bootstrapNodes + +# Hole punching logic below is adapted from libp2p's HPService +# (libp2p/services/hpservice.nim). HPService cannot be used directly because it +# depends on AutoNAT v1 and starts the relay immediately on NotReachable, +# bypassing the UPnP step. + +proc tryStartingDirectConn( + switch: Switch, peerId: PeerId +): Future[bool] {.async: (raises: [CancelledError]).} = + proc tryConnect( + address: MultiAddress + ): Future[bool] {.async: (raises: [DialFailedError, CancelledError]).} = + debug "Trying to create direct connection", peerId, address + await switch.connect(peerId, @[address], true, false) + debug "Direct connection created." + return true + + await sleepAsync(500.milliseconds) # wait for AddressBook to be populated + for address in switch.peerStore[AddressBook][peerId]: + try: + let isRelayedAddr = address.contains(multiCodec("p2p-circuit")) + if not isRelayedAddr.get(false) and address.isPublicMA(): + return await tryConnect(address) + except CatchableError as err: + debug "Failed to create direct connection.", description = err.msg + continue + return false + +proc closeRelayConn(relayedConn: Connection) {.async: (raises: [CancelledError]).} = + await sleepAsync(2000.milliseconds) # grace period before closing relayed connection + await relayedConn.close() + +proc holePunchIfRelayed*( + switch: Switch, peerId: PeerId +) {.async: (raises: [CancelledError]).} = + ## Attempts to establish a direct connection when a peer connected via relay. + ## First tries a direct TCP connect (if the peer's address is known and public), + ## then falls back to dcutr simultaneous-open hole punching. + ## Closes the relay connection once a direct path is established. + let connections = + switch.connManager.getConnections().getOrDefault(peerId).mapIt(it.connection) + if connections.anyIt(not isRelayed(it)): + return + let incomingRelays = connections.filterIt(it.transportDir == Direction.In) + if incomingRelays.len == 0: + return + + let relayedConn = incomingRelays[0] + + if await tryStartingDirectConn(switch, peerId): + await closeRelayConn(relayedConn) + return + + var natAddrs = switch.peerStore.getMostObservedProtosAndPorts() + if natAddrs.len == 0: + natAddrs = switch.peerInfo.listenAddrs.mapIt(switch.peerStore.guessDialableAddr(it)) + try: + await DcutrClient.new().startSync(switch, peerId, natAddrs) + await closeRelayConn(relayedConn) + except DcutrError as err: + debug "Hole punching failed during dcutr", description = err.msg + +proc setupHolePunching*(switch: Switch) = + try: + switch.mount(Dcutr.new(switch)) + except LPError as err: + error "Failed to mount Dcutr protocol", description = err.msg + + let handler = proc( + peerId: PeerId, event: PeerEvent + ) {.async: (raises: [CancelledError]).} = + await holePunchIfRelayed(switch, peerId) + switch.addPeerEventHandler(handler, PeerEventKind.Joined) diff --git a/storage/storage.nim b/storage/storage.nim index 450587cc..15165fcc 100644 --- a/storage/storage.nim +++ b/storage/storage.nim @@ -451,6 +451,8 @@ proc new*( ) ) + setupHolePunching(switch) + # REST server var restServer: RestServerRef = nil diff --git a/tests/storage/testnat.nim b/tests/storage/testnat.nim index 0808cf2d..e4ac92d3 100644 --- a/tests/storage/testnat.nim +++ b/tests/storage/testnat.nim @@ -3,6 +3,8 @@ import pkg/chronos import pkg/libp2p/[multiaddress, multihash, multicodec] import pkg/libp2p/protocols/connectivity/autonat/types import pkg/libp2p/protocols/connectivity/relay/client as relayClientModule +import pkg/libp2p/protocols/connectivity/dcutr/core as dcutrCore +import pkg/libp2p/multistream import pkg/libp2p/services/autorelayservice except setup import pkg/results @@ -48,8 +50,9 @@ asyncchecksuite "NAT - handleNatStatus": test "handleNatStatus announces mapped address when NotReachable and UPnP succeeds": let dialBack = MultiAddress.init("/ip4/1.2.3.4/tcp/8080").expect("valid") - let mapper = - MockNatPortMapper(mappedPorts: some((Port(9000), Port(9001), MappingProtocol.UPnP))) + let mapper = MockNatPortMapper( + mappedPorts: some((Port(9000), Port(9001), MappingProtocol.UPnP)) + ) await mapper.handleNatStatus( NotReachable, Opt.some(dialBack), discoveryPort, disc, sw, autoRelay @@ -105,3 +108,31 @@ asyncchecksuite "NAT - handleNatStatus": check not autoRelay.isRunning check disc.announceAddrs == @[dialBack] check not disc.protocol.clientMode + +asyncchecksuite "NAT - Hole punching": + test "setupHolePunching mounts the dcutr protocol on the switch": + let sw = newStandardSwitch() + setupHolePunching(sw) + check sw.ms.handlers.anyIt(dcutrCore.DcutrCodec in it.protos) + + test "holePunchIfRelayed returns early when the peer has no connections": + let sw1 = newStandardSwitch() + let sw2 = newStandardSwitch() + await allFutures(sw1.start(), sw2.start()) + + await holePunchIfRelayed(sw1, sw2.peerInfo.peerId) + + await allFutures(sw1.stop(), sw2.stop()) + + test "holePunchIfRelayed returns early when a direct connection already exists": + let sw1 = newStandardSwitch() + let sw2 = newStandardSwitch() + await allFutures(sw1.start(), sw2.start()) + + await sw1.connect(sw2.peerInfo.peerId, sw2.peerInfo.addrs) + check sw1.isConnected(sw2.peerInfo.peerId) + + await holePunchIfRelayed(sw1, sw2.peerInfo.peerId) + + check sw1.isConnected(sw2.peerInfo.peerId) + await allFutures(sw1.stop(), sw2.stop()) From 4db7cb64693bb00e45f80ed8c2e0d3a0aa11eabc Mon Sep 17 00:00:00 2001 From: Arnaud Date: Mon, 25 May 2026 10:34:47 +0400 Subject: [PATCH 062/167] Use local record as source of truth for the spr --- storage/discovery.nim | 42 +++++++++++------------------ storage/nat.nim | 6 ++--- storage/storage.nim | 6 +++-- tests/storage/helpers/nodeutils.nim | 4 ++- 4 files changed, 25 insertions(+), 33 deletions(-) diff --git a/storage/discovery.nim b/storage/discovery.nim index 70e8d1b2..66da8c32 100644 --- a/storage/discovery.nim +++ b/storage/discovery.nim @@ -177,27 +177,10 @@ method removeProvider*( raiseAssert("Unexpected Exception in removeProvider") proc getSpr*(d: Discovery): ?SignedPeerRecord = - ## Combined TCP+UDP record for bootstrap use by connecting nodes. - without providerRecord =? d.providerRecord: - return none(SignedPeerRecord) + ## Returns the node's current Signed Peer Record as registered in the DHT. + some(d.protocol.getRecord()) - without dhtRecord =? d.dhtRecord: - return none(SignedPeerRecord) - - let tcpAddrs = providerRecord.data.addresses.mapIt(it.address) - let udpAddrs = dhtRecord.data.addresses.mapIt(it.address) - - SignedPeerRecord - .init(d.key, PeerRecord.init(d.peerId, tcpAddrs & udpAddrs)) - .expect("Should construct signed record").some - -proc updateSpr(d: Discovery) = - if not d.protocol.isNil: - let spr = d.getSpr() - if spr.isSome: - d.protocol.updateRecord(spr).expect("Should update SPR") - -proc updateRecords*( +proc updateRecordsAndSpr*( d: Discovery, announceAddrs: openArray[MultiAddress], udpPort: Port ) = # UDP addresses are derived from TCP announce addresses by remapping protocol and port. @@ -215,7 +198,11 @@ proc updateRecords*( .init(d.key, PeerRecord.init(d.peerId, udpAddrs)) .expect("Should construct signed record").some - d.updateSpr() + if not d.protocol.isNil: + let spr = SignedPeerRecord + .init(d.key, PeerRecord.init(d.peerId, tcpAddrs & udpAddrs)) + .expect("Should construct signed record").some + d.protocol.updateRecord(spr).expect("Should update SPR") proc updateAnnounceRecord*(d: Discovery, addrs: openArray[MultiAddress]) = # Updates announce addresses only, not the DHT routing record. @@ -226,18 +213,14 @@ proc updateAnnounceRecord*(d: Discovery, addrs: openArray[MultiAddress]) = .init(d.key, PeerRecord.init(d.peerId, d.announceAddrs)) .expect("Should construct signed record").some - d.updateSpr() - proc updateDhtRecord*( d: Discovery, addrs: openArray[MultiAddress] -) {.deprecated: "use updateRecords instead".} = +) {.deprecated: "use updateRecordsAndSpr instead".} = info "Updating Dht record", addrs = addrs d.dhtRecord = SignedPeerRecord .init(d.key, PeerRecord.init(d.peerId, @addrs)) .expect("Should construct signed record").some - d.updateSpr() - proc start*(d: Discovery) {.async: (raises: []).} = try: d.protocol.open() @@ -287,7 +270,9 @@ proc new*( key: key, peerId: PeerId.init(key).expect("Should construct PeerId"), store: store ) - self.updateRecords(announceAddrs, udpPort = discoveryPort) + # Called even when announceAddrs is empty: newProtocol below requires + # providerRecord to be set, and it will be updated with real addresses in start(). + self.updateRecordsAndSpr(announceAddrs, udpPort = discoveryPort) let discoveryConfig = DiscoveryConfig(tableIpLimits: tableIpLimits, bitsPerHop: DefaultBitsPerHop) @@ -303,4 +288,7 @@ proc new*( config = discoveryConfig, ) + # Protocol now exists: call again so the SPR is synced into the protocol's local record. + self.updateRecordsAndSpr(announceAddrs, udpPort = discoveryPort) + self diff --git a/storage/nat.nim b/storage/nat.nim index b1729713..07c56b2d 100644 --- a/storage/nat.nim +++ b/storage/nat.nim @@ -152,7 +152,7 @@ method handleNatStatus*( else: debug "AutoRelayService stopped" - discovery.updateRecords(@[dialBackAddr.get], udpPort = discoveryPort) + discovery.updateRecordsAndSpr(@[dialBackAddr.get], udpPort = discoveryPort) discovery.protocol.clientMode = false of NotReachable: var hasPortMapping = false @@ -171,7 +171,7 @@ method handleNatStatus*( # We remove the announced records. # Eventually, it will we updated by the relay when it started - discovery.updateRecords(@[], udpPort = discoveryPort) + discovery.updateRecordsAndSpr(@[], udpPort = discoveryPort) else: debug "Node is not reachable trying port mapping now" @@ -196,7 +196,7 @@ method handleNatStatus*( # The client mode will be updated on the next iteration of autonat. # Trying to check manually that the node is reachable is not trivial, # this is exactly what Autonat is for. - discovery.updateRecords(@[announceAddress], udpPort = udpPort) + discovery.updateRecordsAndSpr(@[announceAddress], udpPort = udpPort) hasPortMapping = true else: # In case of failure, close the port mapping in order to rerun discover diff --git a/storage/storage.nim b/storage/storage.nim index 15165fcc..9374bcb8 100644 --- a/storage/storage.nim +++ b/storage/storage.nim @@ -107,13 +107,15 @@ proc start*(s: StorageServer) {.async.} = else: # Don't announce address and wait for AutoNat @[] - info "info ", addrs = $s.storageNode.switch.peerInfo.addrs[0] + if not s.config.nat.hasExtIp: # Nodes with autonat start with client mode. # It will be updated if reachable. s.storageNode.discovery.protocol.clientMode = true - s.storageNode.discovery.updateRecords(announceAddrs, udpPort = s.config.discoveryPort) + s.storageNode.discovery.updateRecordsAndSpr( + announceAddrs, udpPort = s.config.discoveryPort + ) await s.storageNode.start() diff --git a/tests/storage/helpers/nodeutils.nim b/tests/storage/helpers/nodeutils.nim index fd9e658f..108635a0 100644 --- a/tests/storage/helpers/nodeutils.nim +++ b/tests/storage/helpers/nodeutils.nim @@ -224,7 +224,9 @@ proc generateNodes*( if config.enableBootstrap: waitFor switch.peerInfo.update() - blockDiscovery.updateRecords(switch.peerInfo.addrs, udpPort = bindPort.Port) + blockDiscovery.updateRecordsAndSpr( + switch.peerInfo.addrs, udpPort = bindPort.Port + ) if blockDiscovery.getSpr().isSome: bootstrapNodes.add !blockDiscovery.getSpr() From bffa085dae6073eccfdd1e197a84e02d2c7dcdcd Mon Sep 17 00:00:00 2001 From: Arnaud Date: Mon, 25 May 2026 10:38:32 +0400 Subject: [PATCH 063/167] Close the mapping when the dial back is none. --- storage/nat.nim | 2 ++ 1 file changed, 2 insertions(+) diff --git a/storage/nat.nim b/storage/nat.nim index 07c56b2d..3ddccd27 100644 --- a/storage/nat.nim +++ b/storage/nat.nim @@ -161,6 +161,8 @@ method handleNatStatus*( if dialBackAddr.isNone: warn "Got empty dialback address in AutoNat when node is NotReachable" + if m.tcpMappingId.isSome and m.udpMappingId.isSome: + m.close() elif m.tcpMappingId.isSome and m.udpMappingId.isSome: warn "Not Reachable with active port mapping. The port mapping will be deleted and relay will start." From 84761db205d1e534f3e6b57c4b71fa3187304d56 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Mon, 25 May 2026 10:38:48 +0400 Subject: [PATCH 064/167] Add discovery tests --- tests/storage/testdiscovery.nim | 50 +++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 tests/storage/testdiscovery.nim diff --git a/tests/storage/testdiscovery.nim b/tests/storage/testdiscovery.nim new file mode 100644 index 00000000..965959d4 --- /dev/null +++ b/tests/storage/testdiscovery.nim @@ -0,0 +1,50 @@ +import std/[net, options, sequtils] +import pkg/libp2p/[multiaddress, routing_record] + +import ../asynctest +import ./helpers +import ../../storage/discovery +import ../../storage/rng + +suite "Discovery - SPR record logic": + var key: PrivateKey + var disc: Discovery + + let + directAddr = MultiAddress.init("/ip4/1.2.3.4/tcp/4001").expect("valid") + relayAddr = MultiAddress + .init( + "/ip4/5.6.7.8/tcp/4002/p2p/16Uiu2HAmQu456Ae52JqPuqog6wCex47LLvNY8oHMBC4GRRtaStHs/p2p-circuit" + ) + .expect("valid") + udpPort = Port(8090) + + setup: + key = PrivateKey.random(Rng.instance[]).get() + disc = Discovery.new(key, announceAddrs = @[]) + + test "updateRecordsAndSpr sets the SPR with both TCP and UDP addresses": + disc.updateRecordsAndSpr(@[directAddr], udpPort) + + let spr = disc.getSpr() + check spr.isSome + let addrs = spr.get.data.addresses.mapIt($it.address) + check addrs.anyIt(it.contains("/tcp/")) + check addrs.anyIt(it.contains("/udp/")) + + test "updateAnnounceRecord does not update the SPR": + disc.updateRecordsAndSpr(@[directAddr], udpPort) + let sprBefore = disc.getSpr() + + disc.updateAnnounceRecord(@[relayAddr]) + + check disc.getSpr() == sprBefore + + test "updateDhtRecord deprecated does not update the SPR": + disc.updateRecordsAndSpr(@[directAddr], udpPort) + let sprBefore = disc.getSpr() + + let otherUdpAddr = MultiAddress.init("/ip4/9.9.9.9/udp/9999").expect("valid") + disc.updateDhtRecord(@[otherUdpAddr]) + + check disc.getSpr() == sprBefore From bb64130614c76734f5ff19aff75187710f6fec67 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Mon, 25 May 2026 11:22:46 +0400 Subject: [PATCH 065/167] Update dht records and spr when dial back is empty --- storage/nat.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/storage/nat.nim b/storage/nat.nim index 3ddccd27..387c2168 100644 --- a/storage/nat.nim +++ b/storage/nat.nim @@ -163,6 +163,7 @@ method handleNatStatus*( warn "Got empty dialback address in AutoNat when node is NotReachable" if m.tcpMappingId.isSome and m.udpMappingId.isSome: m.close() + discovery.updateRecordsAndSpr(@[], udpPort = discoveryPort) elif m.tcpMappingId.isSome and m.udpMappingId.isSome: warn "Not Reachable with active port mapping. The port mapping will be deleted and relay will start." From b89fa72840d16a5e796aa3db8b35ecbb51d37fa0 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Mon, 25 May 2026 11:23:28 +0400 Subject: [PATCH 066/167] Cleanup the nat simulation --- storage/utils/natsimulation.nim | 30 ++++++++++++------------------ 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/storage/utils/natsimulation.nim b/storage/utils/natsimulation.nim index 0dd05f8a..14c93c13 100644 --- a/storage/utils/natsimulation.nim +++ b/storage/utils/natsimulation.nim @@ -18,9 +18,8 @@ type FilteringBehavior* = enum type NatRouter* = ref object filtering*: FilteringBehavior - conntrack: seq[TransportAddress] + conntrack: seq[TransportAddress] # remote addrs we dialed; allows them to connect back natMapper*: Option[NatPortMapper] - dropTimeout*: Duration type NatTransport* = ref object of Transport tcp: TcpTransport @@ -41,10 +40,8 @@ proc fromString*( else: err("Unknown filtering behavior: " & s) -proc new*( - T: type NatRouter, filtering: FilteringBehavior, dropTimeout = 20.seconds -): T = - T(filtering: filtering, dropTimeout: dropTimeout) +proc new*(T: type NatRouter, filtering: FilteringBehavior): T = + T(filtering: filtering) proc setFiltering*(r: NatRouter, filtering: FilteringBehavior) = r.filtering = filtering @@ -111,17 +108,16 @@ method dial*( if conn.observedAddr.isSome: let transportAddr = initTAddress(conn.observedAddr.get) if transportAddr.isOk: - self.router.conntrack.add(transportAddr.get) + let remote = transportAddr.get + self.router.conntrack.add(remote) + proc cleanupConntrack() {.async: (raises: []).} = + await noCancel conn.closeEvent.wait() + self.router.conntrack.keepItIf(it != remote) + + asyncSpawn cleanupConntrack() return conn -proc dropAfterTimeout(conn: Connection, timeout: Duration) {.async: (raises: []).} = - # Hold the connection open long enough for the remote's dial to time out, - # then close it. This simulates a NAT that drops packets rather than RSTs - # them, which is what AutoNAT needs to detect NotReachable. - await noCancel sleepAsync(timeout) - await noCancel conn.close() - method accept*( self: NatTransport ): Future[Connection] {.async: (raises: [transport.TransportError, CancelledError]).} = @@ -148,10 +144,8 @@ method accept*( localPort = localAddr.get.port if not self.router.allowInbound(transportAddr.get, localPort): - # Do not close immediately: let the remote's dial time out naturally, - # then clean up. Returning a fast RST would produce EDialRefused (Unknown) - # instead of EDialError (NotReachable) in AutoNAT. - asyncSpawn dropAfterTimeout(conn, self.router.dropTimeout) + # The rejected connection is not closed here: tcp.stop() closes all + # accepted TCP connections on teardown. continue return conn From 8d8f19c6ada23f4337d4bf8f65d30b02b6a7eecc Mon Sep 17 00:00:00 2001 From: Arnaud Date: Mon, 25 May 2026 11:33:04 +0400 Subject: [PATCH 067/167] Cleanup --- storage/discovery.nim | 8 -------- tests/storage/testdiscovery.nim | 9 --------- tests/storage/testnatsimulation.nim | 8 ++++---- 3 files changed, 4 insertions(+), 21 deletions(-) diff --git a/storage/discovery.nim b/storage/discovery.nim index 66da8c32..4c9a5ef2 100644 --- a/storage/discovery.nim +++ b/storage/discovery.nim @@ -213,14 +213,6 @@ proc updateAnnounceRecord*(d: Discovery, addrs: openArray[MultiAddress]) = .init(d.key, PeerRecord.init(d.peerId, d.announceAddrs)) .expect("Should construct signed record").some -proc updateDhtRecord*( - d: Discovery, addrs: openArray[MultiAddress] -) {.deprecated: "use updateRecordsAndSpr instead".} = - info "Updating Dht record", addrs = addrs - d.dhtRecord = SignedPeerRecord - .init(d.key, PeerRecord.init(d.peerId, @addrs)) - .expect("Should construct signed record").some - proc start*(d: Discovery) {.async: (raises: []).} = try: d.protocol.open() diff --git a/tests/storage/testdiscovery.nim b/tests/storage/testdiscovery.nim index 965959d4..863f40a6 100644 --- a/tests/storage/testdiscovery.nim +++ b/tests/storage/testdiscovery.nim @@ -39,12 +39,3 @@ suite "Discovery - SPR record logic": disc.updateAnnounceRecord(@[relayAddr]) check disc.getSpr() == sprBefore - - test "updateDhtRecord deprecated does not update the SPR": - disc.updateRecordsAndSpr(@[directAddr], udpPort) - let sprBefore = disc.getSpr() - - let otherUdpAddr = MultiAddress.init("/ip4/9.9.9.9/udp/9999").expect("valid") - disc.updateDhtRecord(@[otherUdpAddr]) - - check disc.getSpr() == sprBefore diff --git a/tests/storage/testnatsimulation.nim b/tests/storage/testnatsimulation.nim index 81c51d4b..0bd630f0 100644 --- a/tests/storage/testnatsimulation.nim +++ b/tests/storage/testnatsimulation.nim @@ -55,7 +55,7 @@ asyncchecksuite "NatTransport - Address-Dependent Filtering": var bootstrap, thirdNode, natNode: Switch setup: - let router = NatRouter.new(AddressDependent, dropTimeout = 1.seconds) + let router = NatRouter.new(AddressDependent) bootstrap = newSwitch(Rng.instance()) thirdNode = newSwitch(Rng.instance()) natNode = newNatSwitch(router, Rng.instance()) @@ -88,7 +88,7 @@ asyncchecksuite "NatTransport - Address-and-Port-Dependent Filtering": var bootstrap, thirdNode, natNode: Switch setup: - let router = NatRouter.new(AddressAndPortDependent, dropTimeout = 1.seconds) + let router = NatRouter.new(AddressAndPortDependent) bootstrap = newSwitch(Rng.instance()) thirdNode = newSwitch(Rng.instance()) natNode = newNatSwitch(router, Rng.instance()) @@ -122,7 +122,7 @@ asyncchecksuite "NatTransport - Double NAT": var router: NatRouter setup: - router = NatRouter.new(DoubleNat, dropTimeout = 1.seconds) + router = NatRouter.new(DoubleNat) bootstrap = newSwitch(Rng.instance()) natNode = newNatSwitch(router, Rng.instance()) await bootstrap.start() @@ -146,7 +146,7 @@ asyncchecksuite "NatTransport - Port Mapping": var router: NatRouter setup: - router = NatRouter.new(AddressAndPortDependent, dropTimeout = 1.seconds) + router = NatRouter.new(AddressAndPortDependent) bootstrap = newSwitch(Rng.instance()) natNode = newNatSwitch(router, Rng.instance()) await bootstrap.start() From 4457c4856d1add4d833b58971f2e533ce1307d28 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Mon, 25 May 2026 11:55:40 +0400 Subject: [PATCH 068/167] Cleanup nat simulation --- storage/utils/natsimulation.nim | 5 +++-- tests/storage/testnatsimulation.nim | 26 ++++++++++++++++---------- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/storage/utils/natsimulation.nim b/storage/utils/natsimulation.nim index 14c93c13..889cd04e 100644 --- a/storage/utils/natsimulation.nim +++ b/storage/utils/natsimulation.nim @@ -138,8 +138,9 @@ method accept*( continue var localPort = Port(0) - if self.addrs.len > 0: - let localAddr = initTAddress(self.addrs[0]) + if conn.localAddr.isSome: + # Local address read from the accepted socket. + let localAddr = initTAddress(conn.localAddr.get) if localAddr.isOk: localPort = localAddr.get.port diff --git a/tests/storage/testnatsimulation.nim b/tests/storage/testnatsimulation.nim index 0bd630f0..b1de0b38 100644 --- a/tests/storage/testnatsimulation.nim +++ b/tests/storage/testnatsimulation.nim @@ -10,6 +10,17 @@ import ../../storage/utils/natsimulation const flags = {ServerFlags.ReuseAddr} const listenAddr = "/ip4/127.0.0.1/tcp/0" +const filterTimeout = 500.millis + +proc cannotConnect(a, b: Switch): Future[bool] {.async.} = + let completed = + try: + await a.connect(b.peerInfo.peerId, b.peerInfo.addrs).withTimeout(filterTimeout) + except LPError: + false + if completed: + return false + return not a.isConnected(b.peerInfo.peerId) proc newSwitch(rng: Rng): Switch = SwitchBuilder @@ -81,8 +92,7 @@ asyncchecksuite "NatTransport - Address-Dependent Filtering": check thirdNode.isConnected(natNode.peerInfo.peerId) test "bootstrap cannot connect to nat node without a pre-existing connection": - expect(LPError): - await bootstrap.connect(natNode.peerInfo.peerId, natNode.peerInfo.addrs) + check await cannotConnect(bootstrap, natNode) asyncchecksuite "NatTransport - Address-and-Port-Dependent Filtering": var bootstrap, thirdNode, natNode: Switch @@ -109,13 +119,11 @@ asyncchecksuite "NatTransport - Address-and-Port-Dependent Filtering": check bootstrap.isConnected(natNode.peerInfo.peerId) test "bootstrap cannot connect to nat node without a pre-existing connection": - expect(LPError): - await bootstrap.connect(natNode.peerInfo.peerId, natNode.peerInfo.addrs) + check await cannotConnect(bootstrap, natNode) test "third node cannot connect to nat node even after nat node connected to bootstrap": await natNode.connect(bootstrap.peerInfo.peerId, bootstrap.peerInfo.addrs) - expect(LPError): - await thirdNode.connect(natNode.peerInfo.peerId, natNode.peerInfo.addrs) + check await cannotConnect(thirdNode, natNode) asyncchecksuite "NatTransport - Double NAT": var bootstrap, natNode: Switch @@ -138,8 +146,7 @@ asyncchecksuite "NatTransport - Double NAT": natMapper.activeTcpPort = some(actualPort) router.natMapper = some(natMapper) - expect(LPError): - await bootstrap.connect(natNode.peerInfo.peerId, natNode.peerInfo.addrs) + check await cannotConnect(bootstrap, natNode) asyncchecksuite "NatTransport - Port Mapping": var bootstrap, natNode: Switch @@ -170,5 +177,4 @@ asyncchecksuite "NatTransport - Port Mapping": natMapper.activeTcpPort = some(Port(1)) router.natMapper = some(natMapper) - expect(LPError): - await bootstrap.connect(natNode.peerInfo.peerId, natNode.peerInfo.addrs) + check await cannotConnect(bootstrap, natNode) From f68dd0b037aeb0a8e6fd9c46c3637d33493653d9 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Mon, 25 May 2026 12:06:55 +0400 Subject: [PATCH 069/167] Add comment for double nat --- storage/utils/natsimulation.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/utils/natsimulation.nim b/storage/utils/natsimulation.nim index 889cd04e..04dc370b 100644 --- a/storage/utils/natsimulation.nim +++ b/storage/utils/natsimulation.nim @@ -50,7 +50,7 @@ proc setFiltering*(r: NatRouter, filtering: FilteringBehavior) = proc allowInbound(r: NatRouter, remote: TransportAddress, localPort: Port): bool = case r.filtering of DoubleNat: - return false + return false # always blocks: simulates a scenario where inbound connections are never possible of EndpointIndependent: return true else: From ede9bb1bfe216f1e21c5e05fc1c97b115b2e29a5 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Mon, 25 May 2026 12:07:15 +0400 Subject: [PATCH 070/167] Add comment for double nat --- storage/utils/natsimulation.nim | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/storage/utils/natsimulation.nim b/storage/utils/natsimulation.nim index 04dc370b..79c965ec 100644 --- a/storage/utils/natsimulation.nim +++ b/storage/utils/natsimulation.nim @@ -50,7 +50,9 @@ proc setFiltering*(r: NatRouter, filtering: FilteringBehavior) = proc allowInbound(r: NatRouter, remote: TransportAddress, localPort: Port): bool = case r.filtering of DoubleNat: - return false # always blocks: simulates a scenario where inbound connections are never possible + return + false + # always blocks: simulates a scenario where inbound connections are never possible of EndpointIndependent: return true else: From 3362e1a343f30cc3b5d110c6cf2b96194dcc832f Mon Sep 17 00:00:00 2001 From: Arnaud Date: Mon, 25 May 2026 12:07:30 +0400 Subject: [PATCH 071/167] Close peer event handler --- storage/nat.nim | 3 ++- storage/storage.nim | 11 ++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/storage/nat.nim b/storage/nat.nim index 387c2168..363e8307 100644 --- a/storage/nat.nim +++ b/storage/nat.nim @@ -297,7 +297,7 @@ proc holePunchIfRelayed*( except DcutrError as err: debug "Hole punching failed during dcutr", description = err.msg -proc setupHolePunching*(switch: Switch) = +proc setupHolePunching*(switch: Switch): PeerEventHandler = try: switch.mount(Dcutr.new(switch)) except LPError as err: @@ -308,3 +308,4 @@ proc setupHolePunching*(switch: Switch) = ) {.async: (raises: [CancelledError]).} = await holePunchIfRelayed(switch, peerId) switch.addPeerEventHandler(handler, PeerEventKind.Joined) + handler diff --git a/storage/storage.nim b/storage/storage.nim index 9374bcb8..009bc9dc 100644 --- a/storage/storage.nim +++ b/storage/storage.nim @@ -17,6 +17,7 @@ import pkg/chronos import pkg/taskpools import pkg/presto import pkg/libp2p +import pkg/libp2p/connmanager import pkg/libp2p/protocols/connectivity/autonatv2/[service, client] import pkg/libp2p/protocols/connectivity/relay/client as relayClientModule import pkg/libp2p/protocols/connectivity/relay/relay as relayModule @@ -60,6 +61,7 @@ type autoRelayService*: Option[AutoRelayService] natMapper*: Option[NatPortMapper] natRouter*: Option[NatRouter] + holePunchHandler: Option[connmanager.PeerEventHandler] isStarted: bool StoragePrivateKey* = libp2p.PrivateKey # alias @@ -142,6 +144,11 @@ proc stop*(s: StorageServer) {.async.} = if s.natMapper.isSome: s.natMapper.get.close() + if s.holePunchHandler.isSome: + s.storageNode.switch.removePeerEventHandler( + s.holePunchHandler.get, PeerEventKind.Joined + ) + var futures = @[ s.storageNode.switch.stop(), s.storageNode.stop(), @@ -412,6 +419,7 @@ proc new*( # NAT services var natMapper: Option[NatPortMapper] var autoRelayService: Option[AutoRelayService] + var holePunchHandler: Option[connmanager.PeerEventHandler] if autonatService.isSome: let relayService = AutoRelayService.new( @@ -453,7 +461,7 @@ proc new*( ) ) - setupHolePunching(switch) + holePunchHandler = some(setupHolePunching(switch)) # REST server var restServer: RestServerRef = nil @@ -483,4 +491,5 @@ proc new*( autoRelayService: autoRelayService, natMapper: natMapper, natRouter: natRouter, + holePunchHandler: holePunchHandler, ) From da955907a5ced9a491cb0b7d40a4cd504443cf8e Mon Sep 17 00:00:00 2001 From: Arnaud Date: Mon, 25 May 2026 12:08:49 +0400 Subject: [PATCH 072/167] Add debug log --- storage/utils/natsimulation.nim | 3 +++ 1 file changed, 3 insertions(+) diff --git a/storage/utils/natsimulation.nim b/storage/utils/natsimulation.nim index 79c965ec..c30eb357 100644 --- a/storage/utils/natsimulation.nim +++ b/storage/utils/natsimulation.nim @@ -2,6 +2,7 @@ import std/[options, sequtils] import pkg/chronos +import pkg/chronicles import pkg/results import pkg/libp2p import pkg/libp2p/transports/tcptransport @@ -136,6 +137,8 @@ method accept*( let transportAddr = initTAddress(conn.observedAddr.get) if transportAddr.isErr: + debug "Dropping inbound connection: invalid observed address", + address = conn.observedAddr.get await conn.close() continue From 15187dd935ce17ee0e606c08feda992b30ef9b72 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Mon, 25 May 2026 16:59:43 +0400 Subject: [PATCH 073/167] Remove option usage for spr --- .../requests/node_debug_request.nim | 2 +- .../requests/node_info_request.nim | 6 +----- storage/discovery.nim | 9 +++------ storage/rest/api.nim | 7 ++----- 4 files changed, 7 insertions(+), 17 deletions(-) diff --git a/library/storage_thread_requests/requests/node_debug_request.nim b/library/storage_thread_requests/requests/node_debug_request.nim index a72d7143..2bbead69 100644 --- a/library/storage_thread_requests/requests/node_debug_request.nim +++ b/library/storage_thread_requests/requests/node_debug_request.nim @@ -59,7 +59,7 @@ proc getDebug( let json = %*{ "id": $node.switch.peerInfo.peerId, "addrs": node.switch.peerInfo.addrs.mapIt($it), - "spr": if nodeSpr.isSome: nodeSpr.get.toURI else: "", + "spr": nodeSpr.toURI, "announceAddresses": node.discovery.announceAddrs, "table": table, "nat": { diff --git a/library/storage_thread_requests/requests/node_info_request.nim b/library/storage_thread_requests/requests/node_info_request.nim index f6c8c93c..7c0d818e 100644 --- a/library/storage_thread_requests/requests/node_info_request.nim +++ b/library/storage_thread_requests/requests/node_info_request.nim @@ -39,11 +39,7 @@ proc getRepo( proc getSpr( storage: ptr StorageServer ): Future[Result[string, string]] {.async: (raises: []).} = - let spr = storage[].node.discovery.getSpr() - if spr.isNone: - return err("Failed to get SPR: no SPR record found.") - - return ok(spr.get.toURI) + return ok(storage[].node.discovery.getSpr().toURI) proc getPeerId( storage: ptr StorageServer diff --git a/storage/discovery.nim b/storage/discovery.nim index 4c9a5ef2..919ff237 100644 --- a/storage/discovery.nim +++ b/storage/discovery.nim @@ -176,9 +176,9 @@ method removeProvider*( warn "Error removing provider", peerId = peerId, exc = exc.msg raiseAssert("Unexpected Exception in removeProvider") -proc getSpr*(d: Discovery): ?SignedPeerRecord = +proc getSpr*(d: Discovery): SignedPeerRecord = ## Returns the node's current Signed Peer Record as registered in the DHT. - some(d.protocol.getRecord()) + d.protocol.getRecord() proc updateRecordsAndSpr*( d: Discovery, announceAddrs: openArray[MultiAddress], udpPort: Port @@ -188,7 +188,7 @@ proc updateRecordsAndSpr*( let udpAddrs = tcpAddrs.mapIt(it.remapAddr(protocol = some("udp"), port = some(udpPort))) - debug "Updating addresses", tcpAddrs, udpAddrs + info "Updating announce and DHT records", tcpAddrs, udpAddrs d.announceAddrs = tcpAddrs d.providerRecord = SignedPeerRecord @@ -280,7 +280,4 @@ proc new*( config = discoveryConfig, ) - # Protocol now exists: call again so the SPR is synced into the protocol's local record. - self.updateRecordsAndSpr(announceAddrs, udpPort = discoveryPort) - self diff --git a/storage/rest/api.nim b/storage/rest/api.nim index d53d79c8..e9a14007 100644 --- a/storage/rest/api.nim +++ b/storage/rest/api.nim @@ -489,10 +489,7 @@ proc initNodeApi(node: StorageNodeRef, conf: StorageConf, router: var RestRouter var headers = buildCorsHeaders("GET", allowedOrigin) try: - without spr =? node.discovery.getSpr(): - return RestApiResponse.response( - "", status = Http503, contentType = "application/json", headers = headers - ) + let spr = node.discovery.getSpr() if $preferredContentType().get() == "text/plain": return RestApiResponse.response( @@ -586,7 +583,7 @@ proc initDebugApi( "id": $node.switch.peerInfo.peerId, "addrs": node.switch.peerInfo.addrs.mapIt($it), "repo": $conf.dataDir, - "spr": if nodeSpr.isSome: nodeSpr.get.toURI else: "", + "spr": nodeSpr.toURI, "announceAddresses": node.discovery.announceAddrs, "table": table, "storage": {"version": $storageVersion, "revision": $storageRevision}, From 5cc057f04bc4a897c523b7c3b76236c77be6156c Mon Sep 17 00:00:00 2001 From: Arnaud Date: Mon, 25 May 2026 16:59:52 +0400 Subject: [PATCH 074/167] Update openapi doc --- openapi.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openapi.yaml b/openapi.yaml index eed45eba..b736ec30 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -151,7 +151,7 @@ components: description: Whether the AutoRelay service is currently running portMapping: type: string - enum: [none, upnp, pmp] + enum: [none, upnp, pmp, pcp] description: Active NAT port mapping type DataList: @@ -572,7 +572,7 @@ paths: required: true schema: type: string - enum: [endpoint-independent, address-dependent, address-and-port-dependent] + enum: [endpoint-independent, address-dependent, address-and-port-dependent, double-nat] responses: "200": From b7dcf87d3fa7d4e448b725882b599f5f2f5b81a4 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Mon, 25 May 2026 17:00:14 +0400 Subject: [PATCH 075/167] Update records and spr even if the mapping does not exist when dial back is none --- storage/nat.nim | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/storage/nat.nim b/storage/nat.nim index 363e8307..6c8bd7b1 100644 --- a/storage/nat.nim +++ b/storage/nat.nim @@ -161,9 +161,11 @@ method handleNatStatus*( if dialBackAddr.isNone: warn "Got empty dialback address in AutoNat when node is NotReachable" + if m.tcpMappingId.isSome and m.udpMappingId.isSome: m.close() - discovery.updateRecordsAndSpr(@[], udpPort = discoveryPort) + + discovery.updateRecordsAndSpr(@[], udpPort = discoveryPort) elif m.tcpMappingId.isSome and m.udpMappingId.isSome: warn "Not Reachable with active port mapping. The port mapping will be deleted and relay will start." From 54d418ebc46dfb604d30cc10a3f98a2443e6a819 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Mon, 25 May 2026 17:00:34 +0400 Subject: [PATCH 076/167] Add more checks during start --- storage/storage.nim | 65 ++++++++++++++++++++++++++------------------- 1 file changed, 37 insertions(+), 28 deletions(-) diff --git a/storage/storage.nim b/storage/storage.nim index 009bc9dc..1c97a1a5 100644 --- a/storage/storage.nim +++ b/storage/storage.nim @@ -87,6 +87,10 @@ proc start*(s: StorageServer) {.async.} = await s.storageNode.switch.start() + # When listenPort is 0 the OS assigns a random port at bind time. + # We read it back here so the natMapper can create a port mapping for the + # actual port. The switch is configured with a single listen address so + # there is at most one TCP port. if s.natMapper.isSome and s.config.listenPort == Port(0): for listenAddr in s.storageNode.switch.peerInfo.listenAddrs: let maybePort = getTcpPort(listenAddr) @@ -94,33 +98,35 @@ proc start*(s: StorageServer) {.async.} = s.natMapper.get.tcpPort = maybePort.get break - let announceAddrs = - if s.config.nat.hasExtIp: - # extip means that we assume the IP is reachable - # So we just take the first peer addr and remap it with extip to keep the port only - if s.storageNode.switch.peerInfo.addrs.len == 0: - raise - newException(StorageError, "extip is set but switch has no listen addresses") - @[ - s.storageNode.switch.peerInfo.addrs[0].remapAddr( - ip = some(s.config.nat.extIp), port = none(Port) - ) - ] - else: - # Don't announce address and wait for AutoNat - @[] + # The addresses are announced during the start process + # only with extIp because they should be Reachable. + # For other nodes, wait for AutoNat to announce addresses and update SPR. + if s.config.nat.hasExtIp: + if s.storageNode.switch.peerInfo.addrs.len == 0: + raise + newException(StorageError, "extip is set but switch has no listen addresses") - if not s.config.nat.hasExtIp: - # Nodes with autonat start with client mode. + # extip means that we assume the IP is reachable + # So we just take the first peer addr and remap it with extip to keep the port only + let announceAddresses = @[ + s.storageNode.switch.peerInfo.addrs[0].remapAddr( + ip = some(s.config.nat.extIp), port = none(Port) + ) + ] + s.storageNode.discovery.updateRecordsAndSpr( + announceAddresses, udpPort = s.config.discoveryPort + ) + else: + # Other nodes wait for Autonat to announce addresses and update SPR. + # Nodes with autonat start with client mode in order to + # not pollute DHT tables with NotReachable records. # It will be updated if reachable. s.storageNode.discovery.protocol.clientMode = true - s.storageNode.discovery.updateRecordsAndSpr( - announceAddrs, udpPort = s.config.discoveryPort - ) - await s.storageNode.start() + # Connect to the bootstrap nodes in order to have connected peers + # for Autonat. for spr in findReachableNodes(s.config.bootstrapNodes): try: let addrs = spr.data.addresses.mapIt(it.address) @@ -218,13 +224,17 @@ proc new*( ): StorageServer = ## create StorageServer including setting up datastore, repostore, etc - # Guards + # Ensure that you can run an autonat server if the node is Reachable, assumed + # with extIp. + # In other words, a node cannot have autonat server AND autonat client. + # Currently, only bootstrap node should be autonat server. if config.autonatServer and not config.nat.hasExtIp: raise newException(StorageError, "--autonat-server requires --extip") - if config.isRelayServer and not config.autonatServer: - raise - newException(StorageError, "--relay-server is not compatible with autonat client") + # Same for relay server. The node has to be Reachable, assumed by extIp + # but not is the node runs Autonat. + if config.isRelayServer and not config.nat.hasExtIp: + raise newException(StorageError, "--relay-server requires --extip") # Switch let listenMultiAddr = getMultiAddrWithIpAndTcpPort(config.listenIp, config.listenPort) @@ -267,8 +277,6 @@ proc new*( minConfidence = config.natMinConfidence, ) ) - else: - info "AutoNAT disabled (extip configured)" var natRouter: Option[NatRouter] let switch = @@ -292,7 +300,7 @@ proc new*( switch.mount(autonatClient) let autonatService: Option[AutonatV2Service] = - if switchBuilder.autonatV2Service.isSome: + if not config.autonatServer and switchBuilder.autonatV2Service.isSome: some(switchBuilder.autonatV2Service.value) else: none(AutonatV2Service) @@ -445,6 +453,7 @@ proc new*( ) ) + # natRouter is some only when using nat simulation if natRouter.isSome: natRouter.get.natMapper = natMapper From 858fae7154f222914cd8059f87c3c7963fcf928c Mon Sep 17 00:00:00 2001 From: Arnaud Date: Mon, 25 May 2026 17:00:49 +0400 Subject: [PATCH 077/167] Add debug and docs for nat simulation --- storage/utils/natsimulation.nim | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/storage/utils/natsimulation.nim b/storage/utils/natsimulation.nim index c30eb357..514684e7 100644 --- a/storage/utils/natsimulation.nim +++ b/storage/utils/natsimulation.nim @@ -1,3 +1,12 @@ +# NAT simulation for integration testing. +# +# Testing NAT traversal in CI requires controlling inbound/outbound filtering +# rules, which is not possible with real network interfaces. This module wraps +# the TCP transport to enforce configurable filtering behaviors (endpoint- +# independent, address-dependent, address-and-port-dependent, double NAT) at +# the connection level, so the full AutoNAT detection and relay +# stack can be exercised without actual NAT hardware. + {.push raises: [].} import std/[options, sequtils] @@ -11,6 +20,9 @@ import pkg/libp2p/wire import ../nat +logScope: + topics = "nat simulation" + type FilteringBehavior* = enum EndpointIndependent AddressDependent @@ -45,6 +57,8 @@ proc new*(T: type NatRouter, filtering: FilteringBehavior): T = T(filtering: filtering) proc setFiltering*(r: NatRouter, filtering: FilteringBehavior) = + debug "NAT filtering changed", previous = r.filtering, next = filtering + r.filtering = filtering r.conntrack = @[] @@ -154,6 +168,8 @@ method accept*( # accepted TCP connections on teardown. continue + debug "Inbound connection accepted", + remote = transportAddr.get, filtering = self.router.filtering return conn method handles*( From c95d48ee2412a13cfb778dc02d33fe716b9211a7 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Mon, 25 May 2026 17:01:33 +0400 Subject: [PATCH 078/167] Define autonat interval --- tests/integration/1_minute/testnat.nim | 18 +++++++----------- .../integration/5_minutes/testnatdownload.nim | 12 ++++++++++-- tests/integration/nathelper.nim | 3 +++ tests/nat/testnatpcp.nim | 7 +++++-- tests/nat/testnatupnp.nim | 7 +++++-- 5 files changed, 30 insertions(+), 17 deletions(-) diff --git a/tests/integration/1_minute/testnat.nim b/tests/integration/1_minute/testnat.nim index b21d8c64..f7981030 100644 --- a/tests/integration/1_minute/testnat.nim +++ b/tests/integration/1_minute/testnat.nim @@ -19,7 +19,7 @@ multinodesuite "AutoNAT detection": .withRelay(0) .withNatNumPeersToAsk(1) .withNatMinConfidence(0.5) - .withNatScheduleInterval(10.seconds) + .withNatScheduleInterval(NatScheduleInterval) .withNatMaxQueueSize(1).some ) test "node is reachable when using bootstrap node on same network", natConfig: @@ -33,7 +33,7 @@ multinodesuite "AutoNAT detection": .withNatSimulation(idx = 1, "endpoint-independent") .withNatNumPeersToAsk(1) .withNatMinConfidence(0.5) - .withNatScheduleInterval(10.seconds) + .withNatScheduleInterval(NatScheduleInterval) .withNatMaxQueueSize(1).some ) # EIF = Endpoint Independent Filtering @@ -48,7 +48,7 @@ multinodesuite "AutoNAT detection": .withNatSimulation(idx = 1, "address-and-port-dependent") .withNatNumPeersToAsk(1) .withNatMinConfidence(0.5) - .withNatScheduleInterval(10.seconds) + .withNatScheduleInterval(NatScheduleInterval) .withNatMaxQueueSize(1).some ) # APDF = Address and Port-Dependent Filtering @@ -64,7 +64,7 @@ multinodesuite "AutoNAT detection": .withNatSimulation(idx = 1, "address-and-port-dependent") .withNatNumPeersToAsk(1) .withNatMinConfidence(0.5) - .withNatScheduleInterval(5.seconds) + .withNatScheduleInterval(NatScheduleInterval) .withNatMaxQueueSize(1).some ) # APDF = Address and Port-Dependent Filtering @@ -74,9 +74,7 @@ multinodesuite "AutoNAT detection": let node2 = clients()[1] await node2.client.checkNotReachable() - check (await node2.client.setNatFiltering("endpoint-independent")).isOk - await node2.client.checkReachable() let natToSimConfig = NodeConfigs( @@ -86,7 +84,7 @@ multinodesuite "AutoNAT detection": .withNatSimulation(idx = 1, "endpoint-independent") .withNatNumPeersToAsk(1) .withNatMinConfidence(0.5) - .withNatScheduleInterval(5.seconds) + .withNatScheduleInterval(NatScheduleInterval) .withNatMaxQueueSize(1).some ) # APDF = Address and Port-Dependent Filtering @@ -95,9 +93,7 @@ multinodesuite "AutoNAT detection": let node2 = clients()[1] await node2.client.checkReachable() - check (await node2.client.setNatFiltering("address-and-port-dependent")).isOk - await node2.client.checkNotReachable() let doubleNatConfig = NodeConfigs( @@ -107,7 +103,7 @@ multinodesuite "AutoNAT detection": .withNatSimulation(idx = 1, "double-nat") .withNatNumPeersToAsk(1) .withNatMinConfidence(0.5) - .withNatScheduleInterval(5.seconds) + .withNatScheduleInterval(NatScheduleInterval) .withNatMaxQueueSize(1).some ) test "node behind double NAT is detected as not reachable and starts relay", @@ -123,7 +119,7 @@ multinodesuite "AutoNAT detection": .withNatSimulation(idx = 2, "address-and-port-dependent") .withNatNumPeersToAsk(1) .withNatMinConfidence(0.5) - .withNatScheduleInterval(5.seconds) + .withNatScheduleInterval(NatScheduleInterval) .withNatMaxQueueSize(1).some ) # APDF = Address and Port-Dependent Filtering diff --git a/tests/integration/5_minutes/testnatdownload.nim b/tests/integration/5_minutes/testnatdownload.nim index 0216c000..95554326 100644 --- a/tests/integration/5_minutes/testnatdownload.nim +++ b/tests/integration/5_minutes/testnatdownload.nim @@ -1,10 +1,11 @@ -import std/json +import std/[json, sequtils] import pkg/chronos import pkg/questionable/results import ../multinodes import ../storageclient import ../storageconfig +import ../nathelper const RelayTimeout = 30_000 @@ -18,7 +19,7 @@ multinodesuite "NAT download": .withNatSimulation(idx = 2, "address-and-port-dependent") .withNatNumPeersToAsk(1) .withNatMinConfidence(0.5) - .withNatScheduleInterval(5.seconds) + .withNatScheduleInterval(NatScheduleInterval) .withNatMaxQueueSize(1).some ) # APDF = Address and Port-Dependent Filtering @@ -50,6 +51,13 @@ multinodesuite "NAT download": pollInterval = PollInterval, ) + # Verify natNode advertises a relay circuit address. seed has never dialed + # natNode, so APDF blocks any direct inbound connection from seed — the + # only reachable address is the p2p-circuit one. + let info = (await natNode.client.info()).get + let addrs = info["addrs"].getElems.mapIt(it.getStr) + check addrs.anyIt("p2p-circuit" in it) + let content = "content seeded from nat node" let cid = (await natNode.client.upload(content)).get diff --git a/tests/integration/nathelper.nim b/tests/integration/nathelper.nim index 82caadd4..95b3da17 100644 --- a/tests/integration/nathelper.nim +++ b/tests/integration/nathelper.nim @@ -10,6 +10,7 @@ import ./storageconfig const RelayTimeout* = 30_000 PollInterval* = 1_000 + NatScheduleInterval* = 5.seconds proc checkNatStatus*( client: StorageClient, reachability: string, relayRunning: bool, clientMode: bool @@ -43,6 +44,8 @@ proc checkNatStatus*( proc checkReachable*(client: StorageClient) {.async.} = await client.checkNatStatus("Reachable", relayRunning = false, clientMode = false) +# Relay might be false when the mapping has been created for UPnP / TCP but +# Autonat didn't detect yet Reachable proc checkNotReachable*(client: StorageClient, relayRunning = true) {.async.} = await client.checkNatStatus( "NotReachable", relayRunning = relayRunning, clientMode = true diff --git a/tests/nat/testnatpcp.nim b/tests/nat/testnatpcp.nim index 02deecd2..25f5902b 100644 --- a/tests/nat/testnatpcp.nim +++ b/tests/nat/testnatpcp.nim @@ -16,7 +16,7 @@ multinodesuite "AutoNAT PCP port mapping": .withNatSimulation(idx = 1, "address-and-port-dependent") .withNatNumPeersToAsk(1) .withNatMinConfidence(0.5) - .withNatScheduleInterval(10.seconds) + .withNatScheduleInterval(NatScheduleInterval) .withNatMaxQueueSize(1).some ) @@ -44,7 +44,7 @@ multinodesuite "AutoNAT PCP port mapping": .withNatSimulation(idx = 1, "double-nat") .withNatNumPeersToAsk(1) .withNatMinConfidence(0.5) - .withNatScheduleInterval(10.seconds) + .withNatScheduleInterval(NatScheduleInterval) # Increase the max queue to trigger the AutoNat 2 times .withNatMaxQueueSize(2).some ) @@ -63,6 +63,9 @@ multinodesuite "AutoNAT PCP port mapping": pollInterval = PollInterval, ) + # Wait for next Autonat iteration + await sleepAsync(6.seconds) + await node2.client.checkNotReachable() test "reachable node downloads content uploaded by node behind NAT after PCP mapping", diff --git a/tests/nat/testnatupnp.nim b/tests/nat/testnatupnp.nim index e36def22..9f7c29bf 100644 --- a/tests/nat/testnatupnp.nim +++ b/tests/nat/testnatupnp.nim @@ -16,7 +16,7 @@ multinodesuite "AutoNAT UPnP port mapping": .withNatSimulation(idx = 1, "address-and-port-dependent") .withNatNumPeersToAsk(1) .withNatMinConfidence(0.5) - .withNatScheduleInterval(10.seconds) + .withNatScheduleInterval(NatScheduleInterval) .withNatMaxQueueSize(1).some ) @@ -45,7 +45,7 @@ multinodesuite "AutoNAT UPnP port mapping": .withNatSimulation(idx = 1, "double-nat") .withNatNumPeersToAsk(1) .withNatMinConfidence(0.5) - .withNatScheduleInterval(10.seconds) + .withNatScheduleInterval(NatScheduleInterval) # Increase the max queue to trigger the AutoNat 2 times .withNatMaxQueueSize(2).some ) @@ -64,6 +64,9 @@ multinodesuite "AutoNAT UPnP port mapping": pollInterval = PollInterval, ) + # Wait for next Autonat iteration + await sleepAsync(6.seconds) + await node2.client.checkNotReachable() test "reachable node downloads content uploaded by node behind NAT after UPnP mapping", From c2d00a9502a31a196caf6b268ce3f70e80334265 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Mon, 25 May 2026 17:01:51 +0400 Subject: [PATCH 079/167] Add tests for nat filtering api --- .../5_minutes/testrestapivalidation.nim | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/integration/5_minutes/testrestapivalidation.nim b/tests/integration/5_minutes/testrestapivalidation.nim index 20a3ad40..ab0a1b99 100644 --- a/tests/integration/5_minutes/testrestapivalidation.nim +++ b/tests/integration/5_minutes/testrestapivalidation.nim @@ -44,3 +44,25 @@ multinodesuite "Rest API validation": check: response.status == 400 (await response.body) == "Incorrect Cid" + + test "nat/filtering returns 400 when nat simulation not active", config: + let response = await client.post( + client.buildUrl("/debug/nat/filtering?filtering=endpoint-independent") + ) + check response.status == 400 + + let natSimConfig = NodeConfigs( + clients: StorageConfigs + .init(nodes = 1) + .withNatSimulation(idx = 0, "address-and-port-dependent").some + ) + + test "nat/filtering returns 400 for invalid filtering value", natSimConfig: + let response = await client.post( + client.buildUrl("/debug/nat/filtering?filtering=not-a-valid-value") + ) + check response.status == 400 + + test "nat/filtering returns 400 when filtering param is missing", natSimConfig: + let response = await client.post(client.buildUrl("/debug/nat/filtering")) + check response.status == 400 From e2f8220e8c66011c27306b29dff1e97bd6355913 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Mon, 25 May 2026 17:02:12 +0400 Subject: [PATCH 080/167] Add docs for docker setup --- tests/integration/nat/Dockerfile | 8 ++++++-- tests/integration/nat/docker-entrypoint.sh | 21 +++++++++++++++++++++ tests/integration/nat/miniupnpd_stub_rdr.c | 13 ++++++++++--- 3 files changed, 37 insertions(+), 5 deletions(-) diff --git a/tests/integration/nat/Dockerfile b/tests/integration/nat/Dockerfile index 503e86cc..56c29ea9 100644 --- a/tests/integration/nat/Dockerfile +++ b/tests/integration/nat/Dockerfile @@ -8,8 +8,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ iproute2 \ && rm -rf /var/lib/apt/lists/* -# Build miniupnpd with stub redirector: no iptables/nftables needed, always -# returns success for port mapping requests. +# Build miniupnpd with a stub redirector. miniupnpd normally calls iptables/nftables +# to install the actual port forwarding rules when it receives a mapping request. +# In Docker, those calls fail because the container lacks the required kernel +# capabilities, causing every mapping request to return an error to the client. +# The stub replaces the firewall backend with no-ops that always return success, +# so mapping requests complete normally without touching the kernel. COPY tests/integration/nat/miniupnpd_stub_rdr.c /tmp/stub_rdr.c RUN git clone --depth=1 --branch miniupnpd_2_3_9 \ https://github.com/miniupnp/miniupnp.git /tmp/miniupnp \ diff --git a/tests/integration/nat/docker-entrypoint.sh b/tests/integration/nat/docker-entrypoint.sh index b71e64d3..1d5b9df9 100644 --- a/tests/integration/nat/docker-entrypoint.sh +++ b/tests/integration/nat/docker-entrypoint.sh @@ -4,9 +4,24 @@ set -euo pipefail RUNDIR=/tmp/miniupnpd mkdir -p "$RUNDIR" +# miniupnpd must listen on the same interface as the test node. +# We get the default route interface (e.g. eth0) and its IP. LAN_IF=$(ip route show default | awk '/default/{print $5; exit}') LAN_IP=$(ip -4 addr show "$LAN_IF" | awk '/inet /{print $2; exit}' | cut -d/ -f1) +if [[ -z "$LAN_IF" ]]; then + echo "ERROR: could not determine LAN interface" >&2 + exit 1 +fi + +if [[ -z "$LAN_IP" ]]; then + echo "ERROR: could not determine LAN IP on $LAN_IF" >&2 + exit 1 +fi + +# We use a public WAN IP (1.2.3.4) on a dummy interface because miniupnpd +# disables port forwarding when the external interface has a private/RFC1918 +# address (treats it as double-NAT). ip link add plum-wan type dummy ip addr add 1.2.3.4/24 dev plum-wan ip link set plum-wan up @@ -17,11 +32,17 @@ start_miniupnpd() { ext_ifname=plum-wan listening_ip=$LAN_IF enable_pcp_pmp=$enable_pcp_pmp +# port=0: pick a random HTTP port to avoid conflicts with host services. port=0 +# Without an allow rule miniupnpd denies all mapping requests by default. allow 1024-65535 0.0.0.0/0 1024-65535 EOF miniupnpd -d -f "$RUNDIR/miniupnpd.conf" > "$RUNDIR/miniupnpd.log" 2>&1 & + MINIUPNPD_PID=$! sleep 1 + kill -0 "$MINIUPNPD_PID" 2>/dev/null \ + || { echo "ERROR: miniupnpd failed to start" >&2; cat "$RUNDIR/miniupnpd.log" >&2; exit 1; } + echo "miniupnpd started (pid $MINIUPNPD_PID)" } if [[ "${TEST_PCP:-0}" == "1" ]]; then diff --git a/tests/integration/nat/miniupnpd_stub_rdr.c b/tests/integration/nat/miniupnpd_stub_rdr.c index 15e45b0a..9caf44b8 100644 --- a/tests/integration/nat/miniupnpd_stub_rdr.c +++ b/tests/integration/nat/miniupnpd_stub_rdr.c @@ -1,6 +1,13 @@ -/* Stub firewall backend for miniupnpd. - * Replaces iptcrdr.o + iptpinhole.o + nfct_get.o. - * All mapping operations succeed without touching the kernel. */ +/* Stub firewall backend for miniupnpd used in Docker-based tests. + * + * miniupnpd normally calls iptables/nftables to install port forwarding rules + * when it processes a UPnP/PCP/NAT-PMP mapping request. In a Docker container + * those calls fail because the container lacks the required kernel capabilities, + * causing every mapping request to return an error to the client. + * + * This file replaces iptcrdr.o + iptpinhole.o + nfct_get.o with no-ops that + * always return success, so miniupnpd responds correctly to mapping requests + * without touching the kernel. */ #include #include From 6a454490b1ee3f2ff4210be53379c6afcacaa137 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Mon, 25 May 2026 17:02:32 +0400 Subject: [PATCH 081/167] Cleanup --- tests/integration/storageclient.nim | 9 --------- tests/storage/helpers/nodeutils.nim | 3 +-- tests/storage/testdiscovery.nim | 5 ++--- tests/storage/testnat.nim | 2 +- 4 files changed, 4 insertions(+), 15 deletions(-) diff --git a/tests/integration/storageclient.nim b/tests/integration/storageclient.nim index e1bf04a8..dc852533 100644 --- a/tests/integration/storageclient.nim +++ b/tests/integration/storageclient.nim @@ -262,15 +262,6 @@ proc hasBlockRaw*( let url = client.baseurl & "/data/" & cid & "/exists" return client.get(url) -proc connectPeer*( - client: StorageClient, peerId: string, addrs: seq[string] -): Future[void] {.async: (raises: [CancelledError, HttpError]).} = - var url = client.baseurl & "/connect/" & peerId - if addrs.len > 0: - url &= "?" & addrs.mapIt("addrs=" & it).join("&") - let response = await client.get(url) - assert response.status == 200 - proc natRelayRunning*( client: StorageClient ): Future[?!bool] {.async: (raises: [CancelledError, HttpError]).} = diff --git a/tests/storage/helpers/nodeutils.nim b/tests/storage/helpers/nodeutils.nim index 108635a0..f34daa21 100644 --- a/tests/storage/helpers/nodeutils.nim +++ b/tests/storage/helpers/nodeutils.nim @@ -227,8 +227,7 @@ proc generateNodes*( blockDiscovery.updateRecordsAndSpr( switch.peerInfo.addrs, udpPort = bindPort.Port ) - if blockDiscovery.getSpr().isSome: - bootstrapNodes.add !blockDiscovery.getSpr() + bootstrapNodes.add blockDiscovery.getSpr() fullNode else: diff --git a/tests/storage/testdiscovery.nim b/tests/storage/testdiscovery.nim index 863f40a6..bba63d64 100644 --- a/tests/storage/testdiscovery.nim +++ b/tests/storage/testdiscovery.nim @@ -1,4 +1,4 @@ -import std/[net, options, sequtils] +import std/[net, sequtils] import pkg/libp2p/[multiaddress, routing_record] import ../asynctest @@ -27,8 +27,7 @@ suite "Discovery - SPR record logic": disc.updateRecordsAndSpr(@[directAddr], udpPort) let spr = disc.getSpr() - check spr.isSome - let addrs = spr.get.data.addresses.mapIt($it.address) + let addrs = spr.data.addresses.mapIt($it.address) check addrs.anyIt(it.contains("/tcp/")) check addrs.anyIt(it.contains("/udp/")) diff --git a/tests/storage/testnat.nim b/tests/storage/testnat.nim index e4ac92d3..ef5599e1 100644 --- a/tests/storage/testnat.nim +++ b/tests/storage/testnat.nim @@ -112,7 +112,7 @@ asyncchecksuite "NAT - handleNatStatus": asyncchecksuite "NAT - Hole punching": test "setupHolePunching mounts the dcutr protocol on the switch": let sw = newStandardSwitch() - setupHolePunching(sw) + discard setupHolePunching(sw) check sw.ms.handlers.anyIt(dcutrCore.DcutrCodec in it.protos) test "holePunchIfRelayed returns early when the peer has no connections": From 690b08650865c41b898d30ba43c278c07eaf3915 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Mon, 25 May 2026 17:02:48 +0400 Subject: [PATCH 082/167] Add autonat server check for bootstrap nodes in tests --- tests/integration/storageconfig.nim | 5 +++-- tests/integration/twonodes.nim | 5 ++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/integration/storageconfig.nim b/tests/integration/storageconfig.nim index c3ab9ea7..32d5b536 100644 --- a/tests/integration/storageconfig.nim +++ b/tests/integration/storageconfig.nim @@ -332,12 +332,13 @@ proc withRelay*( startConfig.configs[idx].addCliOption("--relay-server") return startConfig -# For testing, a node with extip (not behind nat) is a bootstrap node +# For testing, a node with extip (not behind nat) with autonat server +# enabled is a bootstrap node proc isBootstrapNode*(config: StorageConfig): bool {.raises: [].} = let opts = config.cliOptions.getOrDefault(StartUpCmd.noCmd) try: - if "--nat" in opts and "extip" in opts["--nat"].value: + if "--nat" in opts and "extip" in opts["--nat"].value and "--autonat-server" in opts: return true except KeyError: warn "Failed to look at the extip config" diff --git a/tests/integration/twonodes.nim b/tests/integration/twonodes.nim index 2398873a..e2553aa6 100644 --- a/tests/integration/twonodes.nim +++ b/tests/integration/twonodes.nim @@ -12,7 +12,10 @@ export multinodes template twonodessuite*(name: string, body: untyped) = multinodesuite name: let twoNodesConfig {.inject, used.} = - NodeConfigs(clients: StorageConfigs.init(nodes = 2).withExtIp(1).some) + # Disable Autonat for this suite + NodeConfigs( + clients: StorageConfigs.init(nodes = 2).withExtIp(1).withAutonatServer(0).some + ) var node1 {.inject, used.}: StorageProcess var node2 {.inject, used.}: StorageProcess From 7260cd6fe601b40839ddaea02b3e537c50b6aff0 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Mon, 25 May 2026 17:43:39 +0400 Subject: [PATCH 083/167] Simplify docker tests --- build.nims | 10 ---------- tests/integration/nat/Dockerfile | 9 +++------ tests/integration/nat/docker-entrypoint.sh | 8 ++++++-- 3 files changed, 9 insertions(+), 18 deletions(-) diff --git a/build.nims b/build.nims index d694206d..6318e459 100644 --- a/build.nims +++ b/build.nims @@ -92,16 +92,6 @@ task testNatPcpMapping, "Run PCP NAT integration test (requires miniupnpd contai putEnv("STORAGE_INTEGRATION_TEST_INCLUDES", "nat/testnatpcp.nim") test "testIntegration", outName = "testIntegrationNatPcp" -# Used to build the testing binaries in Docker -task buildNatPortMappingBinaries, "Build UPnP and PCP NAT test binaries without running them": - buildBinary "storage", - outName = "storage", - params = "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE" - putEnv("STORAGE_INTEGRATION_TEST_INCLUDES", "nat/testnatupnp.nim") - buildBinary "testIntegration", outName = "testIntegrationNat", srcDir = "tests/" - putEnv("STORAGE_INTEGRATION_TEST_INCLUDES", "nat/testnatpcp.nim") - buildBinary "testIntegration", outName = "testIntegrationNatPcp", srcDir = "tests/" - task build, "build Logos Storage binary": storageTask() diff --git a/tests/integration/nat/Dockerfile b/tests/integration/nat/Dockerfile index 56c29ea9..1795e1e6 100644 --- a/tests/integration/nat/Dockerfile +++ b/tests/integration/nat/Dockerfile @@ -38,10 +38,9 @@ COPY library/ library/ COPY tests/ tests/ COPY build.nims config.nims storage.nim ./ -# Build libplum C library and Nim binaries. -# ccache caches C compilation across builds; nimcache caches Nim's generated C files. +# Build libplum C library. Nim binaries are compiled at test runtime. +# ccache caches C compilation across builds. RUN --mount=type=cache,target=/root/.ccache \ - --mount=type=cache,target=/app/nimcache \ export PATH="/usr/lib/ccache:$PATH" && \ rm -rf vendor/nim-libplum/vendor/libplum/build && \ cmake -B vendor/nim-libplum/vendor/libplum/build \ @@ -49,9 +48,7 @@ RUN --mount=type=cache,target=/root/.ccache \ vendor/nim-libplum/vendor/libplum && \ make -j$(nproc) -C vendor/nim-libplum/vendor/libplum/build && \ cp vendor/nim-libplum/vendor/libplum/build/libplum.a \ - vendor/nim-libplum/vendor/libplum/libplum.a && \ - USE_SYSTEM_NIM=1 vendor/nimbus-build-system/scripts/env.sh \ - nim buildNatPortMappingBinaries -d:debug -d:disable_libbacktrace build.nims + vendor/nim-libplum/vendor/libplum/libplum.a COPY tests/integration/nat/docker-entrypoint.sh /entrypoint.sh RUN chmod +x /entrypoint.sh diff --git a/tests/integration/nat/docker-entrypoint.sh b/tests/integration/nat/docker-entrypoint.sh index 1d5b9df9..921a740e 100644 --- a/tests/integration/nat/docker-entrypoint.sh +++ b/tests/integration/nat/docker-entrypoint.sh @@ -45,17 +45,21 @@ EOF echo "miniupnpd started (pid $MINIUPNPD_PID)" } +export DEBUG=${DEBUG:-0} + if [[ "${TEST_PCP:-0}" == "1" ]]; then # PCP requires the UDP source IP to match the client_address in the MAP request. # Point the default route at LAN_IP so libplum uses it as both gateway and PCP target. ip route replace default via "$LAN_IP" dev "$LAN_IF" start_miniupnpd yes failed=0 - DEBUG=${DEBUG:-0} /app/build/testIntegrationNatPcp || failed=1 + USE_SYSTEM_NIM=1 vendor/nimbus-build-system/scripts/env.sh \ + nim testNatPcpMapping -d:debug -d:disable_libbacktrace build.nims || failed=1 else start_miniupnpd no failed=0 - DEBUG=${DEBUG:-0} /app/build/testIntegrationNat || failed=1 + USE_SYSTEM_NIM=1 vendor/nimbus-build-system/scripts/env.sh \ + nim testNatPortMapping -d:debug -d:disable_libbacktrace build.nims || failed=1 fi if [[ "${DEBUG:-0}" == "1" ]]; then From 1c6ae98948af69cbc7bd4aa7fc18dc6304c0c0bd Mon Sep 17 00:00:00 2001 From: Arnaud Date: Mon, 25 May 2026 17:44:01 +0400 Subject: [PATCH 084/167] Improve handleNatStatus case when dial back is none --- storage/nat.nim | 4 +++- tests/storage/testnat.nim | 14 +++++++++----- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/storage/nat.nim b/storage/nat.nim index 6c8bd7b1..d4e0fa37 100644 --- a/storage/nat.nim +++ b/storage/nat.nim @@ -144,6 +144,8 @@ method handleNatStatus*( of Reachable: if dialBackAddr.isNone: warn "Got empty dialback address in AutoNat when node is Reachable" + # Reachable but no address to announce: incomplete information, do nothing + # and wait for the next AutoNAT cycle. return if autoRelayService.isRunning: @@ -152,8 +154,8 @@ method handleNatStatus*( else: debug "AutoRelayService stopped" - discovery.updateRecordsAndSpr(@[dialBackAddr.get], udpPort = discoveryPort) discovery.protocol.clientMode = false + discovery.updateRecordsAndSpr(@[dialBackAddr.get], udpPort = discoveryPort) of NotReachable: var hasPortMapping = false diff --git a/tests/storage/testnat.nim b/tests/storage/testnat.nim index ef5599e1..888d0b2f 100644 --- a/tests/storage/testnat.nim +++ b/tests/storage/testnat.nim @@ -54,6 +54,7 @@ asyncchecksuite "NAT - handleNatStatus": mappedPorts: some((Port(9000), Port(9001), MappingProtocol.UPnP)) ) + discard await autorelayservice.setup(autoRelay, sw) await mapper.handleNatStatus( NotReachable, Opt.some(dialBack), discoveryPort, disc, sw, autoRelay ) @@ -63,7 +64,7 @@ asyncchecksuite "NAT - handleNatStatus": check not autoRelay.isRunning check disc.protocol.clientMode - test "handleNatStatus starts autoRelay when NotReachable and UPnP failed": + test "handleNatStatus starts autoRelay when NotReachable and no dialBackAddr": let mapper = MockNatPortMapper(mappedPorts: none((Port, Port, MappingProtocol))) await mapper.handleNatStatus( @@ -73,7 +74,7 @@ asyncchecksuite "NAT - handleNatStatus": check autoRelay.isRunning check disc.protocol.clientMode - test "handleNatStatus starts autoRelay when NotReachable and mapping fails": + test "handleNatStatus starts autoRelay when NotReachable and dialBackAddr but no mapped ports": let dialBack = MultiAddress.init("/ip4/1.2.3.4/tcp/8080").expect("valid") let mapper = MockNatPortMapper(mappedPorts: none((Port, Port, MappingProtocol))) @@ -85,21 +86,24 @@ asyncchecksuite "NAT - handleNatStatus": check disc.announceAddrs == newSeq[MultiAddress]() check disc.protocol.clientMode - test "handleNatStatus does not announce address when Reachable and no dialBackAddr": + test "handleNatStatus does nothing when Reachable and no dialBackAddr": let mapper = MockNatPortMapper(mappedPorts: none((Port, Port, MappingProtocol))) + discard await autorelayservice.setup(autoRelay, sw) + disc.protocol.clientMode = true await mapper.handleNatStatus( Reachable, Opt.none(MultiAddress), discoveryPort, disc, sw, autoRelay ) + check autoRelay.isRunning check disc.announceAddrs == newSeq[MultiAddress]() - check not autoRelay.isRunning - check not disc.protocol.clientMode + check disc.protocol.clientMode test "handleNatStatus stops relay and announces dialBackAddr when Reachable": let dialBack = MultiAddress.init("/ip4/1.2.3.4/tcp/8080").expect("valid") let mapper = MockNatPortMapper(mappedPorts: none((Port, Port, MappingProtocol))) + disc.protocol.clientMode = true discard await autorelayservice.setup(autoRelay, sw) await mapper.handleNatStatus( Reachable, Opt.some(dialBack), discoveryPort, disc, sw, autoRelay From acd6c5f47f8e56ed160d95b6d3f6b9410e26becc Mon Sep 17 00:00:00 2001 From: Arnaud Date: Mon, 25 May 2026 17:45:37 +0400 Subject: [PATCH 085/167] Update submodules url --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index d986bff6..7e178360 100644 --- a/.gitmodules +++ b/.gitmodules @@ -193,4 +193,4 @@ branch = main [submodule "vendor/nim-libplum"] path = vendor/nim-libplum - url = git@github.com:2-towns/nim-libplum.git + url = https://github.com/2-towns/nim-libplum.git From 893514d005590fac6b68c495401c6018d77c965b Mon Sep 17 00:00:00 2001 From: Arnaud Date: Mon, 25 May 2026 19:41:36 +0400 Subject: [PATCH 086/167] Use bootstrap nodes from preset --- storage/storage.nim | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/storage/storage.nim b/storage/storage.nim index 1c97a1a5..c5873dbf 100644 --- a/storage/storage.nim +++ b/storage/storage.nim @@ -127,7 +127,12 @@ proc start*(s: StorageServer) {.async.} = # Connect to the bootstrap nodes in order to have connected peers # for Autonat. - for spr in findReachableNodes(s.config.bootstrapNodes): + let bootstrapNodes = + if s.config.bootstrapNodes.len > 0: + s.config.bootstrapNodes + else: + s.config.network.bootstrapNodes + for spr in findReachableNodes(bootstrapNodes): try: let addrs = spr.data.addresses.mapIt(it.address) await s.storageNode.switch.connect(spr.data.peerId, addrs) From ca46521a63d281f1a30704a15a72c056e62c3458 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Thu, 4 Jun 2026 14:39:17 +0400 Subject: [PATCH 087/167] Update Nix configuration --- nix/nimble.nix | 1 - 1 file changed, 1 deletion(-) diff --git a/nix/nimble.nix b/nix/nimble.nix index 7dde65c5..86262b7c 100644 --- a/nix/nimble.nix +++ b/nix/nimble.nix @@ -4,7 +4,6 @@ let tools = pkgs.callPackage ./tools.nix {}; nbsVersion = tools.findKeyValue "^[[:space:]]+NIMBLE_COMMIT='([a-f0-9]+)'.*$" ../vendor/nimbus-build-system/scripts/build_nim.sh; nimVersion = tools.findKeyValue "^ +NimbleStableCommit = \"([a-f0-9]+)\".+" ../vendor/nimbus-build-system/vendor/Nim/koch.nim; - sourceFile = ../vendor/nimbus-build-system/vendor/Nim/koch.nim; in pkgs.fetchFromGitHub { owner = "nim-lang"; repo = "nimble"; From cd6ba705a67c98938278c4a9ba6890275991b5c6 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Tue, 26 May 2026 16:24:39 +0400 Subject: [PATCH 088/167] Add preset none --- storage/presets.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/storage/presets.nim b/storage/presets.nim index 81d441a6..a8600ac2 100644 --- a/storage/presets.nim +++ b/storage/presets.nim @@ -53,6 +53,7 @@ proc `bootstrapNodes`*(self: NetworkPreset): seq[SignedPeerRecord] = result.add(parse(SignedPeerRecord, record).tryGet()) const NetworkPresets* = [ + NetworkPreset.init("none", "No bootstrap nodes", @[]), NetworkPreset.init( "logos.test", "Logos testnet", From 1193a2c29d2ec21913fde7d73c34cd84b99f5bd0 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Thu, 28 May 2026 09:39:54 +0400 Subject: [PATCH 089/167] Add Identity mapper callback --- storage/conf.nim | 8 ++++++ storage/storage.nim | 62 ++++++++++++++++++++++++++++++++------------- 2 files changed, 52 insertions(+), 18 deletions(-) diff --git a/storage/conf.nim b/storage/conf.nim index 9b08d3e2..c45b62b4 100644 --- a/storage/conf.nim +++ b/storage/conf.nim @@ -335,6 +335,14 @@ type name: "nat-min-confidence" .}: float + natObservedAddrMinCount* {. + desc: + "Number of identify observations of the same external address required " & + "before it is used as the node's dialable address", + defaultValue: 1, + name: "nat-observed-addr-min-count" + .}: int + natMaxRelays* {. desc: "Maximum number of relay servers to reserve slots on simultaneously", defaultValue: 2, diff --git a/storage/storage.nim b/storage/storage.nim index c5873dbf..0d381920 100644 --- a/storage/storage.nim +++ b/storage/storage.nim @@ -42,6 +42,7 @@ import ./namespaces import ./storagetypes import ./logutils import ./nat +import ./utils/natutils import ./utils/natsimulation logScope: @@ -106,8 +107,8 @@ proc start*(s: StorageServer) {.async.} = raise newException(StorageError, "extip is set but switch has no listen addresses") - # extip means that we assume the IP is reachable - # So we just take the first peer addr and remap it with extip to keep the port only + # extip means that we assume the IP is reachable. + # So we just take the first peer addr and remap it with extip to keep the port only. let announceAddresses = @[ s.storageNode.switch.peerInfo.addrs[0].remapAddr( ip = some(s.config.nat.extIp), port = none(Port) @@ -117,10 +118,9 @@ proc start*(s: StorageServer) {.async.} = announceAddresses, udpPort = s.config.discoveryPort ) else: - # Other nodes wait for Autonat to announce addresses and update SPR. - # Nodes with autonat start with client mode in order to - # not pollute DHT tables with NotReachable records. - # It will be updated if reachable. + # Other nodes wait for AutoNAT to announce addresses and update SPR. + # They start in client mode to avoid polluting DHT with NotReachable records; + # it will be flipped off once AutoNAT confirms reachability. s.storageNode.discovery.protocol.clientMode = true await s.storageNode.start() @@ -132,6 +132,7 @@ proc start*(s: StorageServer) {.async.} = s.config.bootstrapNodes else: s.config.network.bootstrapNodes + for spr in findReachableNodes(bootstrapNodes): try: let addrs = spr.data.addresses.mapIt(it.address) @@ -140,6 +141,16 @@ proc start*(s: StorageServer) {.async.} = warn "Cannot connect to bootstrap node", error = e.msg discard + # Refresh peerInfo.addrs so the observed address collected during the + # bootstrap Identify exchange is applied to peerInfo via the address mapper + # before AutoNAT issues its first DialRequest. AutonatV2Service hooks on the + # Joined event which fires before Identify completes, so the very first + # askPeer captures stale (private) addrs; we re-run it manually here with the + # now-updated peerInfo to avoid waiting a full scheduleInterval. + await s.storageNode.switch.peerInfo.update() + if s.autonatService.isSome: + await s.autonatService.get.run(s.storageNode.switch) + if s.restServer != nil: s.restServer.start() @@ -234,12 +245,10 @@ proc new*( # In other words, a node cannot have autonat server AND autonat client. # Currently, only bootstrap node should be autonat server. if config.autonatServer and not config.nat.hasExtIp: - raise newException(StorageError, "--autonat-server requires --extip") + raise newException(StorageError, "--autonat-server requires --nat=extip:") - # Same for relay server. The node has to be Reachable, assumed by extIp - # but not is the node runs Autonat. if config.isRelayServer and not config.nat.hasExtIp: - raise newException(StorageError, "--relay-server requires --extip") + raise newException(StorageError, "--relay-server requires --nat=extip:") # Switch let listenMultiAddr = getMultiAddrWithIpAndTcpPort(config.listenIp, config.listenPort) @@ -273,15 +282,19 @@ proc new*( numPeersToAsk = config.natNumPeersToAsk, maxQueueSize = config.natMaxQueueSize, minConfidence = config.natMinConfidence - switchBuilder = switchBuilder.withAutonatV2( - AutonatV2ServiceConfig.new( - scheduleInterval = Opt.some(config.natScheduleInterval), - askNewConnectedPeers = true, - numPeersToAsk = config.natNumPeersToAsk, - maxQueueSize = config.natMaxQueueSize, - minConfidence = config.natMinConfidence, + switchBuilder = switchBuilder + .withAutonatV2( + AutonatV2ServiceConfig.new( + scheduleInterval = Opt.some(config.natScheduleInterval), + askNewConnectedPeers = false, + numPeersToAsk = config.natNumPeersToAsk, + maxQueueSize = config.natMaxQueueSize, + minConfidence = config.natMinConfidence, + ) + ) + .withObservedAddrManager( + ObservedAddrManager.new(minCount = config.natObservedAddrMinCount) ) - ) var natRouter: Option[NatRouter] let switch = @@ -310,6 +323,19 @@ proc new*( else: none(AutonatV2Service) + # Inject observed addresses into peerInfo.addrs so AutoNAT advertises a + # dialable (public) address. nim-libp2p collects observations via Identify + # but does not wire them into peerInfo automatically; without this, the + # AutoNAT DialRequest carries only private listen addresses and the server + # responds EDialRefused. + if not config.autonatServer: + switch.peerInfo.addressMappers.add( + proc( + addrs: seq[MultiAddress] + ): Future[seq[MultiAddress]] {.async: (raises: [CancelledError]).} = + addrs.mapIt(switch.peerStore.guessDialableAddr(it)) + ) + # Storage infrastructure try: From 00e655705213eb488d058a38cd5332ae81992adf Mon Sep 17 00:00:00 2001 From: Arnaud Date: Mon, 1 Jun 2026 17:44:11 +0400 Subject: [PATCH 090/167] Fix clientMode order and avoid retry port mapping after Not Reachable --- storage/nat.nim | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/storage/nat.nim b/storage/nat.nim index d4e0fa37..a31adc07 100644 --- a/storage/nat.nim +++ b/storage/nat.nim @@ -154,8 +154,10 @@ method handleNatStatus*( else: debug "AutoRelayService stopped" - discovery.protocol.clientMode = false + # Update the record first, then flip to server mode: otherwise the node + # briefly serves DHT queries with the previous (possibly empty) record. discovery.updateRecordsAndSpr(@[dialBackAddr.get], udpPort = discoveryPort) + discovery.protocol.clientMode = false of NotReachable: var hasPortMapping = false @@ -173,12 +175,18 @@ method handleNatStatus*( # The mapping was created the the node is still not reachable. # In that case, we delete the mapping and relay will start. - # We will keep retrying on the next iteration m.close() # We remove the announced records. # Eventually, it will we updated by the relay when it started discovery.updateRecordsAndSpr(@[], udpPort = discoveryPort) + elif autoRelayService.isRunning: + # The mapping was already tried and did not make the node reachable. + # If the relay is running, there is nothing to do. + # We do not want to retry the port mapping if it failed already, + # it would stop the relay service while there is little chance to have + # a Reachable status after it was detected Not Reachable the first time. + discard else: debug "Node is not reachable trying port mapping now" From af21d3c4329effeba20094616aec97419779c915 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Mon, 1 Jun 2026 17:51:03 +0400 Subject: [PATCH 091/167] Define custom AddressMapper --- storage/storage.nim | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/storage/storage.nim b/storage/storage.nim index 0d381920..a1767c77 100644 --- a/storage/storage.nim +++ b/storage/storage.nim @@ -63,6 +63,7 @@ type natMapper*: Option[NatPortMapper] natRouter*: Option[NatRouter] holePunchHandler: Option[connmanager.PeerEventHandler] + observedAddrMapper: Option[AddressMapper] isStarted: bool StoragePrivateKey* = libp2p.PrivateKey # alias @@ -171,6 +172,11 @@ proc stop*(s: StorageServer) {.async.} = s.holePunchHandler.get, PeerEventKind.Joined ) + if s.observedAddrMapper.isSome: + s.storageNode.switch.peerInfo.addressMappers.keepItIf( + it != s.observedAddrMapper.get + ) + var futures = @[ s.storageNode.switch.stop(), s.storageNode.stop(), @@ -290,6 +296,11 @@ proc new*( numPeersToAsk = config.natNumPeersToAsk, maxQueueSize = config.natMaxQueueSize, minConfidence = config.natMinConfidence, + # The AddressMapper in libp2p injects the observed address + # only when the node is detected Reachable. + # We need it before, so we define our custom mapper below, + # and disable this one to avoid having 2 mappers. + enableAddressMapper = false, ) ) .withObservedAddrManager( @@ -328,13 +339,14 @@ proc new*( # but does not wire them into peerInfo automatically; without this, the # AutoNAT DialRequest carries only private listen addresses and the server # responds EDialRefused. - if not config.autonatServer: - switch.peerInfo.addressMappers.add( - proc( - addrs: seq[MultiAddress] - ): Future[seq[MultiAddress]] {.async: (raises: [CancelledError]).} = - addrs.mapIt(switch.peerStore.guessDialableAddr(it)) - ) + var observedAddrMapper: Option[AddressMapper] + if not config.autonatServer and not config.nat.hasExtIp: + let mapper: AddressMapper = proc( + addrs: seq[MultiAddress] + ): Future[seq[MultiAddress]] {.async: (raises: [CancelledError]).} = + addrs.mapIt(switch.peerStore.guessDialableAddr(it)) + switch.peerInfo.addressMappers.add(mapper) + observedAddrMapper = some(mapper) # Storage infrastructure @@ -532,4 +544,5 @@ proc new*( natMapper: natMapper, natRouter: natRouter, holePunchHandler: holePunchHandler, + observedAddrMapper: observedAddrMapper, ) From 8548ecebf3b21b3cb4cce072064c0ce95a8a76b9 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Mon, 1 Jun 2026 17:53:33 +0400 Subject: [PATCH 092/167] Fix openapi --- openapi.yaml | 2 +- storage/nat.nim | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/openapi.yaml b/openapi.yaml index b736ec30..3608cc75 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -151,7 +151,7 @@ components: description: Whether the AutoRelay service is currently running portMapping: type: string - enum: [none, upnp, pmp, pcp] + enum: [none, upnp, pmp, pcp, direct] description: Active NAT port mapping type DataList: diff --git a/storage/nat.nim b/storage/nat.nim index a31adc07..a0b175b0 100644 --- a/storage/nat.nim +++ b/storage/nat.nim @@ -230,7 +230,7 @@ proc reachabilityStr*(autonat: Option[AutonatV2Service]): string = if autonat.isSome: $autonat.get.networkReachability else: - "unknown" + "Unknown" proc portMappingStr*(natMapper: Option[NatPortMapper]): string = if natMapper.isNone or natMapper.get.activeMappingProtocol.isNone: From d5c4461b31e0ae05d60222c9bebe742b04413131 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Mon, 1 Jun 2026 18:09:29 +0400 Subject: [PATCH 093/167] Add comment --- storage/storage.nim | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/storage/storage.nim b/storage/storage.nim index a1767c77..c9aafa4b 100644 --- a/storage/storage.nim +++ b/storage/storage.nim @@ -89,10 +89,8 @@ proc start*(s: StorageServer) {.async.} = await s.storageNode.switch.start() - # When listenPort is 0 the OS assigns a random port at bind time. - # We read it back here so the natMapper can create a port mapping for the - # actual port. The switch is configured with a single listen address so - # there is at most one TCP port. + # When listenPort is 0 the OS assigns a random port. For UDP, the port + # doesn't change so there is no need to update it. if s.natMapper.isSome and s.config.listenPort == Port(0): for listenAddr in s.storageNode.switch.peerInfo.listenAddrs: let maybePort = getTcpPort(listenAddr) @@ -140,7 +138,6 @@ proc start*(s: StorageServer) {.async.} = await s.storageNode.switch.connect(spr.data.peerId, addrs) except CatchableError as e: warn "Cannot connect to bootstrap node", error = e.msg - discard # Refresh peerInfo.addrs so the observed address collected during the # bootstrap Identify exchange is applied to peerInfo via the address mapper From 6a555dc1a7a54f6f7f38bc35c2dc29b08ca91469 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Thu, 4 Jun 2026 14:39:56 +0400 Subject: [PATCH 094/167] Update autonat api --- storage/storage.nim | 63 ++++++++++++++++++++++++++------------------- 1 file changed, 37 insertions(+), 26 deletions(-) diff --git a/storage/storage.nim b/storage/storage.nim index c9aafa4b..5632d94e 100644 --- a/storage/storage.nim +++ b/storage/storage.nim @@ -139,15 +139,13 @@ proc start*(s: StorageServer) {.async.} = except CatchableError as e: warn "Cannot connect to bootstrap node", error = e.msg - # Refresh peerInfo.addrs so the observed address collected during the - # bootstrap Identify exchange is applied to peerInfo via the address mapper - # before AutoNAT issues its first DialRequest. AutonatV2Service hooks on the - # Joined event which fires before Identify completes, so the very first - # askPeer captures stale (private) addrs; we re-run it manually here with the - # now-updated peerInfo to avoid waiting a full scheduleInterval. + # Refresh peerInfo.addrs so the observed address collected during the bootstrap + # Identify exchange is applied to peerInfo via the address mapper, then start + # AutoNAT here (we own it, it is not in switch.services) so its first probe targets + # the now-connected bootstrap peers instead of firing at switch.start on no peers. await s.storageNode.switch.peerInfo.update() if s.autonatService.isSome: - await s.autonatService.get.run(s.storageNode.switch) + await s.autonatService.get.start(s.storageNode.switch) if s.restServer != nil: s.restServer.start() @@ -187,6 +185,9 @@ proc stop*(s: StorageServer) {.async.} = futures.add(stopAutoRelay()) + if s.autonatService.isSome: + futures.add(s.autonatService.get.stop(s.storageNode.switch)) + if s.restServer != nil: futures.add(s.restServer.stop()) @@ -276,6 +277,7 @@ proc new*( .withSignedPeerRecord(true) .withCircuitRelay(relay) + var autonatConfig = none(AutonatV2ServiceConfig) if config.autonatServer: info "AutoNAT server enabled" switchBuilder = switchBuilder.withAutonatV2Server() @@ -285,24 +287,23 @@ proc new*( numPeersToAsk = config.natNumPeersToAsk, maxQueueSize = config.natMaxQueueSize, minConfidence = config.natMinConfidence - switchBuilder = switchBuilder - .withAutonatV2( - AutonatV2ServiceConfig.new( - scheduleInterval = Opt.some(config.natScheduleInterval), - askNewConnectedPeers = false, - numPeersToAsk = config.natNumPeersToAsk, - maxQueueSize = config.natMaxQueueSize, - minConfidence = config.natMinConfidence, - # The AddressMapper in libp2p injects the observed address - # only when the node is detected Reachable. - # We need it before, so we define our custom mapper below, - # and disable this one to avoid having 2 mappers. - enableAddressMapper = false, - ) - ) - .withObservedAddrManager( - ObservedAddrManager.new(minCount = config.natObservedAddrMinCount) + autonatConfig = some( + AutonatV2ServiceConfig.new( + scheduleInterval = Opt.some(config.natScheduleInterval), + askNewConnectedPeers = false, + numPeersToAsk = config.natNumPeersToAsk, + maxQueueSize = config.natMaxQueueSize, + minConfidence = config.natMinConfidence, + # The AddressMapper in libp2p injects the observed address + # only when the node is detected Reachable. + # We need it before, so we define our custom mapper below, + # and disable this one to avoid having 2 mappers. + enableAddressMapper = false, ) + ) + switchBuilder = switchBuilder.withObservedAddrManager( + ObservedAddrManager.new(minCount = config.natObservedAddrMinCount) + ) var natRouter: Option[NatRouter] let switch = @@ -325,9 +326,19 @@ proc new*( autonatClient.setup(switch) switch.mount(autonatClient) + # AutoNAT's first reachability probe fires immediately on start. + # Wired via withAutonatV2 it lands in switch.services and runs at switch.start, + # before bootstrap, on an empty peer set. + # We build and own it here so we can start it ourselves after bootstrap, + # with the bootstrap peers connected. let autonatService: Option[AutonatV2Service] = - if not config.autonatServer and switchBuilder.autonatV2Service.isSome: - some(switchBuilder.autonatV2Service.value) + if autonatConfig.isSome: + let client = AutonatV2Client.new(switch.rng) + client.setup(switch) + switch.mount(client) + let service = AutonatV2Service.new(switch.rng, client, autonatConfig.get) + service.setup(switch) + some(service) else: none(AutonatV2Service) From f0a7e4b4258d3bf7216cd95f157dd28156d46dbf Mon Sep 17 00:00:00 2001 From: Arnaud Date: Thu, 4 Jun 2026 18:51:00 +0400 Subject: [PATCH 095/167] Update nim libplum and provide multiple fixes --- storage/nat.nim | 26 ++++------ storage/presets.nim | 1 - storage/storage.nim | 52 ++++++++++--------- .../integration/5_minutes/testnatdownload.nim | 16 +++--- tests/integration/multinodes.nim | 9 +++- tests/storage/testdiscovery.nim | 2 +- tests/storage/testnat.nim | 10 ++-- tests/storage/testnatsimulation.nim | 4 +- vendor/nim-libplum | 2 +- 9 files changed, 64 insertions(+), 58 deletions(-) diff --git a/storage/nat.nim b/storage/nat.nim index a0b175b0..6abcea63 100644 --- a/storage/nat.nim +++ b/storage/nat.nim @@ -64,12 +64,12 @@ method mapNatPorts*( if not m.plumInitialized: # 5s matches the old NatPortMappingTimeout used with miniupnpc/libnatpmp. let plumLogLevel = - if getEnv("DEBUG") == "1": PLUM_LOG_LEVEL_VERBOSE else: PLUM_LOG_LEVEL_NONE + if getEnv("DEBUG") == "1": PlumLogLevel.Verbose else: PlumLogLevel.None let res = init( logLevel = plumLogLevel, - discoverTimeout = m.discoverTimeout, - mappingTimeout = m.mappingTimeout, - recheckPeriod = m.recheckPeriod, + discoverTimeout = m.discoverTimeout.int32, + mappingTimeout = m.mappingTimeout.int32, + recheckPeriod = m.recheckPeriod.int32, ) if res.isErr: warn "Failed to initialize plum", msg = res.error @@ -149,10 +149,8 @@ method handleNatStatus*( return if autoRelayService.isRunning: - if not await autoRelayService.stop(switch): - debug "AutoRelayService stop method returned false" - else: - debug "AutoRelayService stopped" + await autoRelayService.stop(switch) + debug "AutoRelayService stopped" # Update the record first, then flip to server mode: otherwise the node # briefly serves DHT queries with the previous (possibly empty) record. @@ -201,10 +199,8 @@ method handleNatStatus*( if autoRelayService.isRunning: # Here we stop the relay because the node *should* be reachable - if not await autoRelayService.stop(switch): - debug "AutoRelayService returned an issue when trying to stop" - else: - debug "AutoRelayService stopped" + await autoRelayService.stop(switch) + debug "AutoRelayService stopped" # Note that we update the DHT records but we don't set the client mode # to false because we are not sure the node is reachable. @@ -221,10 +217,8 @@ method handleNatStatus*( if not hasPortMapping and not autoRelayService.isRunning: debug "No port mapping found let's start autorelay" - if not await autoRelayService.setup(switch): - warn "Unable to start autorelay service" - else: - debug "AutoRelayService started" + await autoRelayService.start(switch) + debug "AutoRelayService started" proc reachabilityStr*(autonat: Option[AutonatV2Service]): string = if autonat.isSome: diff --git a/storage/presets.nim b/storage/presets.nim index a8600ac2..81d441a6 100644 --- a/storage/presets.nim +++ b/storage/presets.nim @@ -53,7 +53,6 @@ proc `bootstrapNodes`*(self: NetworkPreset): seq[SignedPeerRecord] = result.add(parse(SignedPeerRecord, record).tryGet()) const NetworkPresets* = [ - NetworkPreset.init("none", "No bootstrap nodes", @[]), NetworkPreset.init( "logos.test", "Logos testnet", diff --git a/storage/storage.nim b/storage/storage.nim index 5632d94e..25fe2bf5 100644 --- a/storage/storage.nim +++ b/storage/storage.nim @@ -181,7 +181,7 @@ proc stop*(s: StorageServer) {.async.} = if s.autoRelayService.isSome and s.autoRelayService.get.isRunning: proc stopAutoRelay(): Future[void] {.async: (raises: []).} = - discard await noCancel s.autoRelayService.get.stop(s.storageNode.switch) + await noCancel s.autoRelayService.get.stop(s.storageNode.switch) futures.add(stopAutoRelay()) @@ -277,6 +277,25 @@ proc new*( .withSignedPeerRecord(true) .withCircuitRelay(relay) + let bootstrapNodes = + if config.noBootstrapNode: + # Sanity checks that the user isn't doing anything funny. + if config.bootstrapNodes.len > 0: + error "Cannot specify bootstrap nodes when using no-bootstrap flag" + raise newException( + ValueError, "Cannot specify bootstrap nodes when using no-bootstrap flag" + ) + + warn "Node has been marked with --no-bootstrap-node and will NOT be bootstrapped" + seq[SignedPeerRecord](@[]) + elif config.bootstrapNodes.len > 0: + warn "Overriding network preset using custom bootstrap nodes", + nodes = config.bootstrapNodes + config.bootstrapNodes + else: + info "Bootstrapping node using a predefined network", network = $config.network + config.network.bootstrapNodes + var autonatConfig = none(AutonatV2ServiceConfig) if config.autonatServer: info "AutoNAT server enabled" @@ -301,8 +320,14 @@ proc new*( enableAddressMapper = false, ) ) + # At the first AutoNAT probe, the only identify observations available come + # from the bootstrap nodes, so requiring more observations than there are + # bootstrap nodes would make the threshold unreachable. The floor of 1 + # covers the case where the bootstrap list is empty. + let observedAddrMinCount = + max(1, min(config.natObservedAddrMinCount, bootstrapNodes.len)) switchBuilder = switchBuilder.withObservedAddrManager( - ObservedAddrManager.new(minCount = config.natObservedAddrMinCount) + ObservedAddrManager.new(minCount = observedAddrMinCount) ) var natRouter: Option[NatRouter] @@ -323,8 +348,6 @@ proc new*( .build() var taskPool: Taskpool - autonatClient.setup(switch) - switch.mount(autonatClient) # AutoNAT's first reachability probe fires immediately on start. # Wired via withAutonatV2 it lands in switch.services and runs at switch.start, @@ -382,25 +405,6 @@ proc new*( error "Failed to initialize discovery datastore", path = providersPath, err = discoveryStoreRes.error.msg - let bootstrapNodes = - if config.noBootstrapNode: - # Sanity checks that the user isn't doing anything funny. - if config.bootstrapNodes.len > 0: - error "Cannot specify bootstrap nodes when using no-bootstrap flag" - raise newException( - ValueError, "Cannot specify bootstrap nodes when using no-bootstrap flag" - ) - - warn "Node has been marked with --no-bootstrap-node and will NOT be bootstrapped" - seq[SignedPeerRecord](@[]) - elif config.bootstrapNodes.len > 0: - warn "Overriding network preset using custom bootstrap nodes", - nodes = config.bootstrapNodes - config.bootstrapNodes - else: - info "Bootstrapping node using a predefined network", network = $config.network - config.network.bootstrapNodes - let discoveryStore = Datastore(discoveryStoreRes.expect("Should create discovery datastore!")) @@ -411,7 +415,6 @@ proc new*( bindPort = config.discoveryPort, bootstrapNodes = bootstrapNodes, discoveryPort = config.discoveryPort, - bootstrapNodes = config.bootstrapNodes, store = discoveryStore, ) @@ -491,6 +494,7 @@ proc new*( rng = random.Rng.instance(), ) + relayService.setup(switch) autoRelayService = some(relayService) natMapper = some( diff --git a/tests/integration/5_minutes/testnatdownload.nim b/tests/integration/5_minutes/testnatdownload.nim index 95554326..89299760 100644 --- a/tests/integration/5_minutes/testnatdownload.nim +++ b/tests/integration/5_minutes/testnatdownload.nim @@ -51,12 +51,16 @@ multinodesuite "NAT download": pollInterval = PollInterval, ) - # Verify natNode advertises a relay circuit address. seed has never dialed - # natNode, so APDF blocks any direct inbound connection from seed — the - # only reachable address is the p2p-circuit one. - let info = (await natNode.client.info()).get - let addrs = info["addrs"].getElems.mapIt(it.getStr) - check addrs.anyIt("p2p-circuit" in it) + # relayRunning only means the service started: the reservation itself + # takes a few more seconds, so we have to poll. + proc advertisesCircuitAddr(): Future[bool] {.async.} = + let info = (await natNode.client.info()).get + let addrs = info["addrs"].getElems.mapIt(it.getStr) + return addrs.anyIt("p2p-circuit" in it) + + check eventuallySafe( + await advertisesCircuitAddr(), timeout = RelayTimeout, pollInterval = PollInterval + ) let content = "content seeded from nat node" let cid = (await natNode.client.upload(content)).get diff --git a/tests/integration/multinodes.nim b/tests/integration/multinodes.nim index 75fdb9d1..d5419255 100644 --- a/tests/integration/multinodes.nim +++ b/tests/integration/multinodes.nim @@ -127,8 +127,13 @@ template multinodesuite*(suiteName: string, body: untyped) = lastUsedStorageApiPort = apiPort lastUsedStorageDiscPort = discPort - for bootstrapNode in bootstrapNodes: - config.addCliOption("--bootstrap-node", bootstrapNode) + if bootstrapNodes.len == 0: + # Without this flag the node would bootstrap on the default + # network preset. + config.addCliOption("--no-bootstrap-node") + else: + for bootstrapNode in bootstrapNodes: + config.addCliOption("--bootstrap-node", bootstrapNode) config.addCliOption("--data-dir", datadir) except StorageConfigError as e: diff --git a/tests/storage/testdiscovery.nim b/tests/storage/testdiscovery.nim index bba63d64..a32d4401 100644 --- a/tests/storage/testdiscovery.nim +++ b/tests/storage/testdiscovery.nim @@ -20,7 +20,7 @@ suite "Discovery - SPR record logic": udpPort = Port(8090) setup: - key = PrivateKey.random(Rng.instance[]).get() + key = PrivateKey.random(Rng.instance()).get() disc = Discovery.new(key, announceAddrs = @[]) test "updateRecordsAndSpr sets the SPR with both TCP and UDP addresses": diff --git a/tests/storage/testnat.nim b/tests/storage/testnat.nim index 888d0b2f..7d0e7275 100644 --- a/tests/storage/testnat.nim +++ b/tests/storage/testnat.nim @@ -35,7 +35,7 @@ asyncchecksuite "NAT - handleNatStatus": setup: autoRelay = AutoRelayService.new(1, relayClientModule.RelayClient.new(), nil, Rng.instance()) - key = PrivateKey.random(Rng.instance[]).get() + key = PrivateKey.random(Rng.instance()).get() disc = Discovery.new(key, announceAddrs = @[]) sw = newStandardSwitch() await sw.start() @@ -44,7 +44,7 @@ asyncchecksuite "NAT - handleNatStatus": await sw.stop() if autoRelay.isRunning: - discard await autoRelay.stop(sw) + await autoRelay.stop(sw) let discoveryPort = Port(8090) @@ -54,7 +54,7 @@ asyncchecksuite "NAT - handleNatStatus": mappedPorts: some((Port(9000), Port(9001), MappingProtocol.UPnP)) ) - discard await autorelayservice.setup(autoRelay, sw) + autorelayservice.setup(autoRelay, sw) await mapper.handleNatStatus( NotReachable, Opt.some(dialBack), discoveryPort, disc, sw, autoRelay ) @@ -89,7 +89,7 @@ asyncchecksuite "NAT - handleNatStatus": test "handleNatStatus does nothing when Reachable and no dialBackAddr": let mapper = MockNatPortMapper(mappedPorts: none((Port, Port, MappingProtocol))) - discard await autorelayservice.setup(autoRelay, sw) + autorelayservice.setup(autoRelay, sw) disc.protocol.clientMode = true await mapper.handleNatStatus( Reachable, Opt.none(MultiAddress), discoveryPort, disc, sw, autoRelay @@ -104,7 +104,7 @@ asyncchecksuite "NAT - handleNatStatus": let mapper = MockNatPortMapper(mappedPorts: none((Port, Port, MappingProtocol))) disc.protocol.clientMode = true - discard await autorelayservice.setup(autoRelay, sw) + autorelayservice.setup(autoRelay, sw) await mapper.handleNatStatus( Reachable, Opt.some(dialBack), discoveryPort, disc, sw, autoRelay ) diff --git a/tests/storage/testnatsimulation.nim b/tests/storage/testnatsimulation.nim index b1de0b38..f2079c8a 100644 --- a/tests/storage/testnatsimulation.nim +++ b/tests/storage/testnatsimulation.nim @@ -26,7 +26,7 @@ proc newSwitch(rng: Rng): Switch = SwitchBuilder .new() .withRng(rng) - .withPrivateKey(PrivateKey.random(rng[]).get()) + .withPrivateKey(PrivateKey.random(rng).get()) .withAddresses(@[MultiAddress.init(listenAddr).get()]) .withTcpTransport(flags) .withNoise() @@ -37,7 +37,7 @@ proc newNatSwitch(router: NatRouter, rng: Rng): Switch = SwitchBuilder .new() .withRng(rng) - .withPrivateKey(PrivateKey.random(rng[]).get()) + .withPrivateKey(PrivateKey.random(rng).get()) .withAddresses(@[MultiAddress.init(listenAddr).get()]) .withNatTransport(router, flags) .withNoise() diff --git a/vendor/nim-libplum b/vendor/nim-libplum index bf0ace8d..0ca0361f 160000 --- a/vendor/nim-libplum +++ b/vendor/nim-libplum @@ -1 +1 @@ -Subproject commit bf0ace8da2715b6aed1e1ed9f33614c8e3b83893 +Subproject commit 0ca0361f50452147d52c7d613c5d6d27a7ac3471 From 564398bc62055d04d053f6468f3207740ee46e27 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Fri, 5 Jun 2026 14:56:08 +0400 Subject: [PATCH 096/167] Restore nim nat traversal for libp2p --- .gitmodules | 3 +++ Makefile | 2 +- vendor/nim-nat-traversal | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) create mode 160000 vendor/nim-nat-traversal diff --git a/.gitmodules b/.gitmodules index 7e178360..b89b3039 100644 --- a/.gitmodules +++ b/.gitmodules @@ -194,3 +194,6 @@ [submodule "vendor/nim-libplum"] path = vendor/nim-libplum url = https://github.com/2-towns/nim-libplum.git +[submodule "vendor/nim-nat-traversal"] + path = vendor/nim-nat-traversal + url = https://github.com/status-im/nim-nat-traversal.git diff --git a/Makefile b/Makefile index 71319a00..6828081d 100644 --- a/Makefile +++ b/Makefile @@ -123,7 +123,7 @@ else NIM_PARAMS := $(NIM_PARAMS) -d:release endif -deps: | deps-common libplum +deps: | deps-common nat-libs libplum ifneq ($(USE_LIBBACKTRACE), 0) deps: | libbacktrace endif diff --git a/vendor/nim-nat-traversal b/vendor/nim-nat-traversal new file mode 160000 index 00000000..860e18c3 --- /dev/null +++ b/vendor/nim-nat-traversal @@ -0,0 +1 @@ +Subproject commit 860e18c37667b5dd005b94c63264560c35d88004 From 48d81dcc0ef9d8ed7e680483cec2f0ec5e961b4c Mon Sep 17 00:00:00 2001 From: Arnaud Date: Fri, 5 Jun 2026 16:31:01 +0400 Subject: [PATCH 097/167] Announce address when peer info is updated --- storage/nat.nim | 40 ++++++++++++++++++++++++++++++---------- storage/storage.nim | 42 ++++++++++++------------------------------ 2 files changed, 42 insertions(+), 40 deletions(-) diff --git a/storage/nat.nim b/storage/nat.nim index 6abcea63..4a709ab6 100644 --- a/storage/nat.nim +++ b/storage/nat.nim @@ -8,7 +8,7 @@ {.push raises: [].} -import std/[options, net, os] +import std/[options, net, os, sequtils] import results import pkg/chronos @@ -129,6 +129,32 @@ proc close*(m: NatPortMapper) = proc isPortMapped*(m: NatPortMapper, port: Port): bool = m.activeTcpPort.isSome and m.activeTcpPort.get == port +proc announcePeerInfoAddrs*(discovery: Discovery, peerInfo: PeerInfo, udpPort: Port) = + ## Announces peerInfo.addrs to the DHT, excluding relay circuit addresses: + ## they are announced via onReservation and must not enter the DHT routing + ## record. No-op when the addresses are already announced, so peerInfo + ## updates that only touch filtered-out addresses do not re-announce. + let addrs = peerInfo.addrs.filterIt(not it.isCircuitRelayMA()) + if addrs.len == 0 or addrs == discovery.announceAddrs: + return + discovery.updateRecordsAndSpr(addrs, udpPort = udpPort) + +proc setupPeerInfoObserver*( + switch: Switch, autonat: AutonatV2Service, discovery: Discovery, udpPort: Port +): PeerInfoObserver = + ## AutoNAT's address mapper resolves peerInfo.addrs into public addresses + ## once the node is Reachable; peerInfo.update() then notifies observers. + ## Keep the DHT records in sync with what libp2p announces. + let observer: PeerInfoObserver = proc(peerInfo: PeerInfo) {.gcsafe, raises: [].} = + info "PeerInfo updated", + addrs = peerInfo.addrs, reachability = autonat.networkReachability + if autonat.networkReachability != NetworkReachability.Reachable: + return + announcePeerInfoAddrs(discovery, peerInfo, udpPort) + + switch.peerInfo.addObserver(observer) + observer + method handleNatStatus*( m: NatPortMapper, networkReachability: NetworkReachability, @@ -142,19 +168,13 @@ method handleNatStatus*( of Unknown: discard of Reachable: - if dialBackAddr.isNone: - warn "Got empty dialback address in AutoNat when node is Reachable" - # Reachable but no address to announce: incomplete information, do nothing - # and wait for the next AutoNAT cycle. - return - if autoRelayService.isRunning: await autoRelayService.stop(switch) debug "AutoRelayService stopped" - # Update the record first, then flip to server mode: otherwise the node - # briefly serves DHT queries with the previous (possibly empty) record. - discovery.updateRecordsAndSpr(@[dialBackAddr.get], udpPort = discoveryPort) + # No announce here: AutoNAT refreshes peerInfo right after this handler, + # its address mapper (active now that the node is Reachable) resolves the + # public addresses and the peerInfo observer announces them. discovery.protocol.clientMode = false of NotReachable: var hasPortMapping = false diff --git a/storage/storage.nim b/storage/storage.nim index 25fe2bf5..79b098da 100644 --- a/storage/storage.nim +++ b/storage/storage.nim @@ -63,7 +63,7 @@ type natMapper*: Option[NatPortMapper] natRouter*: Option[NatRouter] holePunchHandler: Option[connmanager.PeerEventHandler] - observedAddrMapper: Option[AddressMapper] + peerInfoObserver: Option[PeerInfoObserver] isStarted: bool StoragePrivateKey* = libp2p.PrivateKey # alias @@ -139,11 +139,9 @@ proc start*(s: StorageServer) {.async.} = except CatchableError as e: warn "Cannot connect to bootstrap node", error = e.msg - # Refresh peerInfo.addrs so the observed address collected during the bootstrap - # Identify exchange is applied to peerInfo via the address mapper, then start - # AutoNAT here (we own it, it is not in switch.services) so its first probe targets - # the now-connected bootstrap peers instead of firing at switch.start on no peers. - await s.storageNode.switch.peerInfo.update() + # Start AutoNAT here (we own it, it is not in switch.services) so its first + # probe targets the now-connected bootstrap peers instead of firing at + # switch.start on no peers. if s.autonatService.isSome: await s.autonatService.get.start(s.storageNode.switch) @@ -167,10 +165,8 @@ proc stop*(s: StorageServer) {.async.} = s.holePunchHandler.get, PeerEventKind.Joined ) - if s.observedAddrMapper.isSome: - s.storageNode.switch.peerInfo.addressMappers.keepItIf( - it != s.observedAddrMapper.get - ) + if s.peerInfoObserver.isSome: + s.storageNode.switch.peerInfo.removeObserver(s.peerInfoObserver.get) var futures = @[ s.storageNode.switch.stop(), @@ -313,11 +309,6 @@ proc new*( numPeersToAsk = config.natNumPeersToAsk, maxQueueSize = config.natMaxQueueSize, minConfidence = config.natMinConfidence, - # The AddressMapper in libp2p injects the observed address - # only when the node is detected Reachable. - # We need it before, so we define our custom mapper below, - # and disable this one to avoid having 2 mappers. - enableAddressMapper = false, ) ) # At the first AutoNAT probe, the only identify observations available come @@ -365,20 +356,6 @@ proc new*( else: none(AutonatV2Service) - # Inject observed addresses into peerInfo.addrs so AutoNAT advertises a - # dialable (public) address. nim-libp2p collects observations via Identify - # but does not wire them into peerInfo automatically; without this, the - # AutoNAT DialRequest carries only private listen addresses and the server - # responds EDialRefused. - var observedAddrMapper: Option[AddressMapper] - if not config.autonatServer and not config.nat.hasExtIp: - let mapper: AddressMapper = proc( - addrs: seq[MultiAddress] - ): Future[seq[MultiAddress]] {.async: (raises: [CancelledError]).} = - addrs.mapIt(switch.peerStore.guessDialableAddr(it)) - switch.peerInfo.addressMappers.add(mapper) - observedAddrMapper = some(mapper) - # Storage infrastructure try: @@ -482,6 +459,7 @@ proc new*( var natMapper: Option[NatPortMapper] var autoRelayService: Option[AutoRelayService] var holePunchHandler: Option[connmanager.PeerEventHandler] + var peerInfoObserver: Option[PeerInfoObserver] if autonatService.isSome: let relayService = AutoRelayService.new( @@ -512,6 +490,10 @@ proc new*( if natRouter.isSome: natRouter.get.natMapper = natMapper + peerInfoObserver = some( + setupPeerInfoObserver(switch, autonatService.get, discovery, config.discoveryPort) + ) + autonatService.get.setStatusAndConfidenceHandler( proc( networkReachability: NetworkReachability, @@ -556,5 +538,5 @@ proc new*( natMapper: natMapper, natRouter: natRouter, holePunchHandler: holePunchHandler, - observedAddrMapper: observedAddrMapper, + peerInfoObserver: peerInfoObserver, ) From 5548b8af5e80ba99489eca8a9aea0c2d57d730d8 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Fri, 5 Jun 2026 16:31:17 +0400 Subject: [PATCH 098/167] Bump libplum --- vendor/nim-libplum | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/nim-libplum b/vendor/nim-libplum index 0ca0361f..433e4878 160000 --- a/vendor/nim-libplum +++ b/vendor/nim-libplum @@ -1 +1 @@ -Subproject commit 0ca0361f50452147d52c7d613c5d6d27a7ac3471 +Subproject commit 433e48789dfef0a1435db97946b4fee8595d8fb9 From 8530e5e7187e8bd671794ee2a3017c61740f8c45 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Fri, 5 Jun 2026 16:31:58 +0400 Subject: [PATCH 099/167] Ensure that the addresses is announced when Reachable --- tests/integration/nathelper.nim | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/tests/integration/nathelper.nim b/tests/integration/nathelper.nim index 95b3da17..8436cd7d 100644 --- a/tests/integration/nathelper.nim +++ b/tests/integration/nathelper.nim @@ -1,3 +1,4 @@ +import std/algorithm import std/json import std/sequtils import pkg/chronos @@ -25,6 +26,18 @@ proc checkNatStatus*( let rr = nat["relayRunning"].getBool() let ha = addrs.anyIt("p2p-circuit" in it) let pm = nat["portMapping"].getStr() + let aa = info["announceAddresses"].getElems.mapIt(it.getStr) + + # Reachable nodes must announce their dialable (non-circuit) addrs to + # the DHT (peerInfo observer); relayed nodes must announce their + # circuit addrs (onReservation). + let announceOk = + if reachability == "Reachable": + aa.len > 0 and aa.sorted == addrs.filterIt("p2p-circuit" notin it).sorted + elif relayRunning: + aa.len > 0 and aa.allIt("p2p-circuit" in it) + else: + true # It is important to check all the conditions together to avoid race # (new autonat iteration) @@ -33,10 +46,10 @@ proc checkNatStatus*( "reachability=" & r & " (want " & reachability & ")" & " clientMode=" & $cm & " (want " & $clientMode & ")" & " relayRunning=" & $rr & " (want " & $relayRunning & ")" & " p2p-circuit=" & $ha & " (want " & $relayRunning & ")" & - " portMapping=" & pm + " portMapping=" & pm & " announceAddresses=" & $aa ) r == reachability and cm == clientMode and rr == relayRunning and - ha == relayRunning, + ha == relayRunning and announceOk, timeout = RelayTimeout, pollInterval = PollInterval, ) From 6e730658e61b35c96434810ee9c5cc3d74aec283 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Fri, 5 Jun 2026 16:32:30 +0400 Subject: [PATCH 100/167] Add a test to ensure that libp2p uses the observed address to dial --- tests/storage/testnat.nim | 100 +++++++++++++++++++++++++++++++------- vendor/nim-libp2p | 2 +- 2 files changed, 84 insertions(+), 18 deletions(-) diff --git a/tests/storage/testnat.nim b/tests/storage/testnat.nim index 7d0e7275..a64fa0f3 100644 --- a/tests/storage/testnat.nim +++ b/tests/storage/testnat.nim @@ -2,6 +2,9 @@ import std/[net] import pkg/chronos import pkg/libp2p/[multiaddress, multihash, multicodec] import pkg/libp2p/protocols/connectivity/autonat/types +import pkg/libp2p/protocols/connectivity/autonatv2/service except setup +import pkg/libp2p/protocols/connectivity/autonatv2/client except setup +import pkg/libp2p/protocols/connectivity/autonatv2/types as autonatv2Types import pkg/libp2p/protocols/connectivity/relay/client as relayClientModule import pkg/libp2p/protocols/connectivity/dcutr/core as dcutrCore import pkg/libp2p/multistream @@ -26,6 +29,17 @@ method mapNatPorts*( .} = m.mappedPorts +type MockAutonatV2Client = ref object of AutonatV2Client + reqAddrs: seq[MultiAddress] + +method sendDialRequest*( + self: MockAutonatV2Client, pid: PeerId, testAddrs: seq[MultiAddress] +): Future[AutonatV2Response] {. + async: (raises: [AutonatV2Error, CancelledError, DialFailedError, LPStreamError]) +.} = + self.reqAddrs = testAddrs + AutonatV2Response(reachability: Unknown) + asyncchecksuite "NAT - handleNatStatus": var sw: Switch var key: PrivateKey @@ -67,6 +81,7 @@ asyncchecksuite "NAT - handleNatStatus": test "handleNatStatus starts autoRelay when NotReachable and no dialBackAddr": let mapper = MockNatPortMapper(mappedPorts: none((Port, Port, MappingProtocol))) + autorelayservice.setup(autoRelay, sw) await mapper.handleNatStatus( NotReachable, Opt.none(MultiAddress), discoveryPort, disc, sw, autoRelay ) @@ -78,6 +93,7 @@ asyncchecksuite "NAT - handleNatStatus": let dialBack = MultiAddress.init("/ip4/1.2.3.4/tcp/8080").expect("valid") let mapper = MockNatPortMapper(mappedPorts: none((Port, Port, MappingProtocol))) + autorelayservice.setup(autoRelay, sw) await mapper.handleNatStatus( NotReachable, Opt.some(dialBack), discoveryPort, disc, sw, autoRelay ) @@ -86,33 +102,83 @@ asyncchecksuite "NAT - handleNatStatus": check disc.announceAddrs == newSeq[MultiAddress]() check disc.protocol.clientMode - test "handleNatStatus does nothing when Reachable and no dialBackAddr": + test "handleNatStatus stops relay and exits client mode when Reachable": let mapper = MockNatPortMapper(mappedPorts: none((Port, Port, MappingProtocol))) - autorelayservice.setup(autoRelay, sw) disc.protocol.clientMode = true + autorelayservice.setup(autoRelay, sw) await mapper.handleNatStatus( Reachable, Opt.none(MultiAddress), discoveryPort, disc, sw, autoRelay ) - check autoRelay.isRunning - check disc.announceAddrs == newSeq[MultiAddress]() - check disc.protocol.clientMode - - test "handleNatStatus stops relay and announces dialBackAddr when Reachable": - let dialBack = MultiAddress.init("/ip4/1.2.3.4/tcp/8080").expect("valid") - let mapper = MockNatPortMapper(mappedPorts: none((Port, Port, MappingProtocol))) - - disc.protocol.clientMode = true - autorelayservice.setup(autoRelay, sw) - await mapper.handleNatStatus( - Reachable, Opt.some(dialBack), discoveryPort, disc, sw, autoRelay - ) - check not autoRelay.isRunning - check disc.announceAddrs == @[dialBack] check not disc.protocol.clientMode + test "announcePeerInfoAddrs excludes relay circuit addresses": + let circuitAddr = MultiAddress + .init("/ip4/1.2.3.4/tcp/4040/p2p/" & $sw.peerInfo.peerId & "/p2p-circuit") + .expect("valid") + sw.peerInfo.addrs.add(circuitAddr) + + announcePeerInfoAddrs(disc, sw.peerInfo, discoveryPort) + + check circuitAddr notin disc.announceAddrs + check disc.announceAddrs == sw.peerInfo.addrs.filterIt(it != circuitAddr) + + test "announcePeerInfoAddrs does nothing when addresses are already announced": + announcePeerInfoAddrs(disc, sw.peerInfo, discoveryPort) + let seqNo = disc.getSpr().data.seqNo + + announcePeerInfoAddrs(disc, sw.peerInfo, discoveryPort) + + check disc.getSpr().data.seqNo == seqNo + + test "peerInfo observer announces addresses when Reachable": + let autonat = AutonatV2Service.new(Rng.instance()) + discard setupPeerInfoObserver(sw, autonat, disc, discoveryPort) + autonat.networkReachability = Reachable + + sw.peerInfo.listenAddrs.add( + MultiAddress.init("/ip4/1.2.3.4/tcp/9999").expect("valid") + ) + await sw.peerInfo.update() + + check disc.announceAddrs == sw.peerInfo.addrs + + test "peerInfo observer does not announce when the node is not Reachable": + let autonat = AutonatV2Service.new(Rng.instance()) + discard setupPeerInfoObserver(sw, autonat, disc, discoveryPort) + autonat.networkReachability = NotReachable + + sw.peerInfo.listenAddrs.add( + MultiAddress.init("/ip4/1.2.3.4/tcp/9999").expect("valid") + ) + await sw.peerInfo.update() + + check disc.announceAddrs == newSeq[MultiAddress]() + + test "autonat dial request includes the observed addresses as candidates": + # Reproduces vacp2p/nim-libp2p#2600: until that fix is vendored, the + # dial request only contains peerInfo.addrs (private listen addrs), so + # a NATed node never submits a dialable candidate. + let client = MockAutonatV2Client() + let autonat = AutonatV2Service.new(Rng.instance(), client) + service.setup(autonat, sw) + await autonat.start(sw) + + let observed = MultiAddress.init("/ip4/8.8.8.8/tcp/4001").expect("valid") + for _ in 0 ..< 3: # minCount: 3 observations before the manager trusts an addr + discard sw.peerStore.identify.observedAddrManager.addObservation(observed) + + let sw2 = newStandardSwitch() + await sw2.start() + await sw.connect(sw2.peerInfo.peerId, sw2.peerInfo.addrs) + + check eventually(observed in client.reqAddrs) + + await autonat.stop(sw) + await sw2.stop() + asyncchecksuite "NAT - Hole punching": test "setupHolePunching mounts the dcutr protocol on the switch": let sw = newStandardSwitch() diff --git a/vendor/nim-libp2p b/vendor/nim-libp2p index c4319937..2be4e5ed 160000 --- a/vendor/nim-libp2p +++ b/vendor/nim-libp2p @@ -1 +1 @@ -Subproject commit c43199378f46d0aaf61be1cad1ee1d63e8f665d6 +Subproject commit 2be4e5edb39c113be7c293d53ec17c6cadbb9640 From ccb80ac79b5ef74dd710f3b5f2e546555154cfa2 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Fri, 5 Jun 2026 17:41:24 +0400 Subject: [PATCH 101/167] Announce the mapped external UDP port instead of the discovery port --- storage/nat.nim | 8 +++++++- storage/storage.nim | 5 ++--- tests/storage/testnat.nim | 25 +++++++++++++++++++++++-- 3 files changed, 32 insertions(+), 6 deletions(-) diff --git a/storage/nat.nim b/storage/nat.nim index 4a709ab6..e9769011 100644 --- a/storage/nat.nim +++ b/storage/nat.nim @@ -140,7 +140,10 @@ proc announcePeerInfoAddrs*(discovery: Discovery, peerInfo: PeerInfo, udpPort: P discovery.updateRecordsAndSpr(addrs, udpPort = udpPort) proc setupPeerInfoObserver*( - switch: Switch, autonat: AutonatV2Service, discovery: Discovery, udpPort: Port + switch: Switch, + autonat: AutonatV2Service, + discovery: Discovery, + natMapper: NatPortMapper, ): PeerInfoObserver = ## AutoNAT's address mapper resolves peerInfo.addrs into public addresses ## once the node is Reachable; peerInfo.update() then notifies observers. @@ -150,6 +153,9 @@ proc setupPeerInfoObserver*( addrs = peerInfo.addrs, reachability = autonat.networkReachability if autonat.networkReachability != NetworkReachability.Reachable: return + # When a NAT mapping is active, announce its external UDP port: the router + # may have assigned a different port than the requested discoveryPort. + let udpPort = natMapper.activeUdpPort.get(natMapper.discoveryPort) announcePeerInfoAddrs(discovery, peerInfo, udpPort) switch.peerInfo.addObserver(observer) diff --git a/storage/storage.nim b/storage/storage.nim index 79b098da..18a5270f 100644 --- a/storage/storage.nim +++ b/storage/storage.nim @@ -490,9 +490,8 @@ proc new*( if natRouter.isSome: natRouter.get.natMapper = natMapper - peerInfoObserver = some( - setupPeerInfoObserver(switch, autonatService.get, discovery, config.discoveryPort) - ) + peerInfoObserver = + some(setupPeerInfoObserver(switch, autonatService.get, discovery, natMapper.get)) autonatService.get.setStatusAndConfidenceHandler( proc( diff --git a/tests/storage/testnat.nim b/tests/storage/testnat.nim index a64fa0f3..8dd87b1e 100644 --- a/tests/storage/testnat.nim +++ b/tests/storage/testnat.nim @@ -135,7 +135,9 @@ asyncchecksuite "NAT - handleNatStatus": test "peerInfo observer announces addresses when Reachable": let autonat = AutonatV2Service.new(Rng.instance()) - discard setupPeerInfoObserver(sw, autonat, disc, discoveryPort) + discard setupPeerInfoObserver( + sw, autonat, disc, NatPortMapper(discoveryPort: discoveryPort) + ) autonat.networkReachability = Reachable sw.peerInfo.listenAddrs.add( @@ -145,9 +147,28 @@ asyncchecksuite "NAT - handleNatStatus": check disc.announceAddrs == sw.peerInfo.addrs + test "peerInfo observer announces the mapped external UDP port when a mapping is active": + let autonat = AutonatV2Service.new(Rng.instance()) + let mapper = + NatPortMapper(discoveryPort: discoveryPort, activeUdpPort: some(Port(40001))) + discard setupPeerInfoObserver(sw, autonat, disc, mapper) + autonat.networkReachability = Reachable + + sw.peerInfo.listenAddrs.add( + MultiAddress.init("/ip4/1.2.3.4/tcp/9999").expect("valid") + ) + await sw.peerInfo.update() + + let sprAddrs = disc.getSpr().data.addresses.mapIt(it.address) + check MultiAddress.init("/ip4/1.2.3.4/udp/40001").expect("valid") in sprAddrs + check MultiAddress.init("/ip4/1.2.3.4/udp/" & $discoveryPort).expect("valid") notin + sprAddrs + test "peerInfo observer does not announce when the node is not Reachable": let autonat = AutonatV2Service.new(Rng.instance()) - discard setupPeerInfoObserver(sw, autonat, disc, discoveryPort) + discard setupPeerInfoObserver( + sw, autonat, disc, NatPortMapper(discoveryPort: discoveryPort) + ) autonat.networkReachability = NotReachable sw.peerInfo.listenAddrs.add( From a4196d6a5d30e72f14fdda441248a783ab9919b7 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Fri, 5 Jun 2026 17:57:01 +0400 Subject: [PATCH 102/167] Include relay addresses in the SPR and refactoring --- storage/discovery.nim | 13 ++++++++----- storage/nat.nim | 8 ++++---- storage/storage.nim | 4 ++-- tests/storage/helpers/nodeutils.nim | 2 +- tests/storage/testdiscovery.nim | 14 +++++++------- 5 files changed, 22 insertions(+), 19 deletions(-) diff --git a/storage/discovery.nim b/storage/discovery.nim index 919ff237..87c88262 100644 --- a/storage/discovery.nim +++ b/storage/discovery.nim @@ -180,7 +180,7 @@ proc getSpr*(d: Discovery): SignedPeerRecord = ## Returns the node's current Signed Peer Record as registered in the DHT. d.protocol.getRecord() -proc updateRecordsAndSpr*( +proc announceDirectAddrs*( d: Discovery, announceAddrs: openArray[MultiAddress], udpPort: Port ) = # UDP addresses are derived from TCP announce addresses by remapping protocol and port. @@ -204,15 +204,18 @@ proc updateRecordsAndSpr*( .expect("Should construct signed record").some d.protocol.updateRecord(spr).expect("Should update SPR") -proc updateAnnounceRecord*(d: Discovery, addrs: openArray[MultiAddress]) = - # Updates announce addresses only, not the DHT routing record. - # Relay addresses should not pollute DHT routing. +proc announceRelayAddrs*(d: Discovery, addrs: openArray[MultiAddress]) = + ## Updates only announce addresses + ## When using relay, the DHT routing record is not updated to not pollute the DHT. d.announceAddrs = @addrs info "Updating announce record", addrs = d.announceAddrs d.providerRecord = SignedPeerRecord .init(d.key, PeerRecord.init(d.peerId, d.announceAddrs)) .expect("Should construct signed record").some + if not d.protocol.isNil: + d.protocol.updateRecord(d.providerRecord).expect("Should update SPR") + proc start*(d: Discovery) {.async: (raises: []).} = try: d.protocol.open() @@ -264,7 +267,7 @@ proc new*( # Called even when announceAddrs is empty: newProtocol below requires # providerRecord to be set, and it will be updated with real addresses in start(). - self.updateRecordsAndSpr(announceAddrs, udpPort = discoveryPort) + self.announceDirectAddrs(announceAddrs, udpPort = discoveryPort) let discoveryConfig = DiscoveryConfig(tableIpLimits: tableIpLimits, bitsPerHop: DefaultBitsPerHop) diff --git a/storage/nat.nim b/storage/nat.nim index e9769011..5a12b675 100644 --- a/storage/nat.nim +++ b/storage/nat.nim @@ -137,7 +137,7 @@ proc announcePeerInfoAddrs*(discovery: Discovery, peerInfo: PeerInfo, udpPort: P let addrs = peerInfo.addrs.filterIt(not it.isCircuitRelayMA()) if addrs.len == 0 or addrs == discovery.announceAddrs: return - discovery.updateRecordsAndSpr(addrs, udpPort = udpPort) + discovery.announceDirectAddrs(addrs, udpPort = udpPort) proc setupPeerInfoObserver*( switch: Switch, @@ -193,7 +193,7 @@ method handleNatStatus*( if m.tcpMappingId.isSome and m.udpMappingId.isSome: m.close() - discovery.updateRecordsAndSpr(@[], udpPort = discoveryPort) + discovery.announceDirectAddrs(@[], udpPort = discoveryPort) elif m.tcpMappingId.isSome and m.udpMappingId.isSome: warn "Not Reachable with active port mapping. The port mapping will be deleted and relay will start." @@ -203,7 +203,7 @@ method handleNatStatus*( # We remove the announced records. # Eventually, it will we updated by the relay when it started - discovery.updateRecordsAndSpr(@[], udpPort = discoveryPort) + discovery.announceDirectAddrs(@[], udpPort = discoveryPort) elif autoRelayService.isRunning: # The mapping was already tried and did not make the node reachable. # If the relay is running, there is nothing to do. @@ -233,7 +233,7 @@ method handleNatStatus*( # The client mode will be updated on the next iteration of autonat. # Trying to check manually that the node is reachable is not trivial, # this is exactly what Autonat is for. - discovery.updateRecordsAndSpr(@[announceAddress], udpPort = udpPort) + discovery.announceDirectAddrs(@[announceAddress], udpPort = udpPort) hasPortMapping = true else: # In case of failure, close the port mapping in order to rerun discover diff --git a/storage/storage.nim b/storage/storage.nim index 18a5270f..e2479bdd 100644 --- a/storage/storage.nim +++ b/storage/storage.nim @@ -113,7 +113,7 @@ proc start*(s: StorageServer) {.async.} = ip = some(s.config.nat.extIp), port = none(Port) ) ] - s.storageNode.discovery.updateRecordsAndSpr( + s.storageNode.discovery.announceDirectAddrs( announceAddresses, udpPort = s.config.discoveryPort ) else: @@ -468,7 +468,7 @@ proc new*( onReservation = proc(addresses: seq[MultiAddress]) {.gcsafe, raises: [].} = info "Relay reservation updated", addresses # relay addresses are for download traffic only, not DHT routing - discovery.updateAnnounceRecord(addresses), + discovery.announceRelayAddrs(addresses), rng = random.Rng.instance(), ) diff --git a/tests/storage/helpers/nodeutils.nim b/tests/storage/helpers/nodeutils.nim index f34daa21..97d43add 100644 --- a/tests/storage/helpers/nodeutils.nim +++ b/tests/storage/helpers/nodeutils.nim @@ -224,7 +224,7 @@ proc generateNodes*( if config.enableBootstrap: waitFor switch.peerInfo.update() - blockDiscovery.updateRecordsAndSpr( + blockDiscovery.announceDirectAddrs( switch.peerInfo.addrs, udpPort = bindPort.Port ) bootstrapNodes.add blockDiscovery.getSpr() diff --git a/tests/storage/testdiscovery.nim b/tests/storage/testdiscovery.nim index a32d4401..8178319b 100644 --- a/tests/storage/testdiscovery.nim +++ b/tests/storage/testdiscovery.nim @@ -23,18 +23,18 @@ suite "Discovery - SPR record logic": key = PrivateKey.random(Rng.instance()).get() disc = Discovery.new(key, announceAddrs = @[]) - test "updateRecordsAndSpr sets the SPR with both TCP and UDP addresses": - disc.updateRecordsAndSpr(@[directAddr], udpPort) + test "announceDirectAddrs sets the SPR with both TCP and UDP addresses": + disc.announceDirectAddrs(@[directAddr], udpPort) let spr = disc.getSpr() let addrs = spr.data.addresses.mapIt($it.address) check addrs.anyIt(it.contains("/tcp/")) check addrs.anyIt(it.contains("/udp/")) - test "updateAnnounceRecord does not update the SPR": - disc.updateRecordsAndSpr(@[directAddr], udpPort) - let sprBefore = disc.getSpr() + test "announceRelayAddrs updates the SPR with the announce addresses": + disc.announceDirectAddrs(@[directAddr], udpPort) - disc.updateAnnounceRecord(@[relayAddr]) + disc.announceRelayAddrs(@[relayAddr]) - check disc.getSpr() == sprBefore + let addrs = disc.getSpr().data.addresses.mapIt($it.address) + check addrs == @[$relayAddr] From d1a44ef997cb6d94212ca39c89a15de6f0ab3e21 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Fri, 5 Jun 2026 18:06:10 +0400 Subject: [PATCH 103/167] Prevent libplum re-initialization after shutdown --- storage/nat.nim | 11 ++++++++++- storage/storage.nim | 2 +- tests/storage/testnat.nim | 15 +++++++++++++++ 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/storage/nat.nim b/storage/nat.nim index 5a12b675..24a3c902 100644 --- a/storage/nat.nim +++ b/storage/nat.nim @@ -47,13 +47,14 @@ type NatPortMapper* = ref object of RootObj activeTcpPort*: Option[Port] activeUdpPort*: Option[Port] plumInitialized: bool + closed: bool method mapNatPorts*( m: NatPortMapper ): Future[Option[(Port, Port, MappingProtocol)]] {. async: (raises: [CancelledError]), base, gcsafe .} = - if m.natConfig.hasExtIp: + if m.closed or m.natConfig.hasExtIp: return none((Port, Port, MappingProtocol)) # If both mappings are still active, return the stored ports without recreating. @@ -126,6 +127,11 @@ proc close*(m: NatPortMapper) = discard cleanup() m.plumInitialized = false +proc stop*(m: NatPortMapper) = + ## Ensure that any future AutoNAT callback does not re-initialize libplum. + m.closed = true + m.close() + proc isPortMapped*(m: NatPortMapper, port: Port): bool = m.activeTcpPort.isSome and m.activeTcpPort.get == port @@ -170,6 +176,9 @@ method handleNatStatus*( switch: Switch, autoRelayService: AutoRelayService, ) {.async: (raises: [CancelledError]), base, gcsafe.} = + if m.closed: + return + case networkReachability of Unknown: discard diff --git a/storage/storage.nim b/storage/storage.nim index e2479bdd..c23cbac6 100644 --- a/storage/storage.nim +++ b/storage/storage.nim @@ -158,7 +158,7 @@ proc stop*(s: StorageServer) {.async.} = notice "Stopping Storage node" if s.natMapper.isSome: - s.natMapper.get.close() + s.natMapper.get.stop() if s.holePunchHandler.isSome: s.storageNode.switch.removePeerEventHandler( diff --git a/tests/storage/testnat.nim b/tests/storage/testnat.nim index 8dd87b1e..9b367fa5 100644 --- a/tests/storage/testnat.nim +++ b/tests/storage/testnat.nim @@ -114,6 +114,21 @@ asyncchecksuite "NAT - handleNatStatus": check not autoRelay.isRunning check not disc.protocol.clientMode + test "handleNatStatus does nothing after the mapper is stopped": + let dialBack = MultiAddress.init("/ip4/1.2.3.4/tcp/8080").expect("valid") + let mapper = MockNatPortMapper( + mappedPorts: some((Port(9000), Port(9001), MappingProtocol.UPnP)) + ) + mapper.stop() + + autorelayservice.setup(autoRelay, sw) + await mapper.handleNatStatus( + NotReachable, Opt.some(dialBack), discoveryPort, disc, sw, autoRelay + ) + + check not autoRelay.isRunning + check disc.announceAddrs == newSeq[MultiAddress]() + test "announcePeerInfoAddrs excludes relay circuit addresses": let circuitAddr = MultiAddress .init("/ip4/1.2.3.4/tcp/4040/p2p/" & $sw.peerInfo.peerId & "/p2p-circuit") From 5b37ee92101d40bdc0a1689cad268c3f979d9cc0 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Fri, 5 Jun 2026 18:09:54 +0400 Subject: [PATCH 104/167] Store bootstrapNodes to reuse them when during the start --- storage/storage.nim | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/storage/storage.nim b/storage/storage.nim index c23cbac6..af7032b4 100644 --- a/storage/storage.nim +++ b/storage/storage.nim @@ -64,6 +64,7 @@ type natRouter*: Option[NatRouter] holePunchHandler: Option[connmanager.PeerEventHandler] peerInfoObserver: Option[PeerInfoObserver] + bootstrapNodes: seq[SignedPeerRecord] isStarted: bool StoragePrivateKey* = libp2p.PrivateKey # alias @@ -126,13 +127,7 @@ proc start*(s: StorageServer) {.async.} = # Connect to the bootstrap nodes in order to have connected peers # for Autonat. - let bootstrapNodes = - if s.config.bootstrapNodes.len > 0: - s.config.bootstrapNodes - else: - s.config.network.bootstrapNodes - - for spr in findReachableNodes(bootstrapNodes): + for spr in findReachableNodes(s.bootstrapNodes): try: let addrs = spr.data.addresses.mapIt(it.address) await s.storageNode.switch.connect(spr.data.peerId, addrs) @@ -538,4 +533,5 @@ proc new*( natRouter: natRouter, holePunchHandler: holePunchHandler, peerInfoObserver: peerInfoObserver, + bootstrapNodes: bootstrapNodes, ) From 5c8391d2f6e760892ebb1c7d5aab16264968914f Mon Sep 17 00:00:00 2001 From: Arnaud Date: Fri, 5 Jun 2026 18:12:14 +0400 Subject: [PATCH 105/167] Re raise cancelled error --- storage/nat.nim | 2 ++ 1 file changed, 2 insertions(+) diff --git a/storage/nat.nim b/storage/nat.nim index 24a3c902..9facf336 100644 --- a/storage/nat.nim +++ b/storage/nat.nim @@ -299,6 +299,8 @@ proc tryStartingDirectConn( let isRelayedAddr = address.contains(multiCodec("p2p-circuit")) if not isRelayedAddr.get(false) and address.isPublicMA(): return await tryConnect(address) + except CancelledError as exc: + raise exc except CatchableError as err: debug "Failed to create direct connection.", description = err.msg continue From 6cfb255785286d83481e133cb7bbcd1a91bbf1b3 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Fri, 5 Jun 2026 18:13:29 +0400 Subject: [PATCH 106/167] Re-raise cancelled error --- storage/storage.nim | 2 ++ 1 file changed, 2 insertions(+) diff --git a/storage/storage.nim b/storage/storage.nim index af7032b4..19488ad5 100644 --- a/storage/storage.nim +++ b/storage/storage.nim @@ -131,6 +131,8 @@ proc start*(s: StorageServer) {.async.} = try: let addrs = spr.data.addresses.mapIt(it.address) await s.storageNode.switch.connect(spr.data.peerId, addrs) + except CancelledError as exc: + raise exc except CatchableError as e: warn "Cannot connect to bootstrap node", error = e.msg From da1066c32a28bbd6b95347172ee92b123da323d3 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Fri, 5 Jun 2026 18:14:54 +0400 Subject: [PATCH 107/167] Improve config error message for nat --- storage/conf.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/conf.nim b/storage/conf.nim index c45b62b4..10e1f81b 100644 --- a/storage/conf.nim +++ b/storage/conf.nim @@ -469,7 +469,7 @@ func parse*(T: type NatConfig, p: string): Result[NatConfig, string] = let error = "Not a valid IP address: " & p[6 ..^ 1] return err(error) else: - return err("Not a valid NAT option: " & p) + return err("Not a valid NAT option: " & p & ". Valid options: auto, extip:") proc parseCmdArg*(T: type NatConfig, p: string): T = let res = NatConfig.parse(p) From 5b7ff0513813fa6fb01d2b4ca396c760ea3e9897 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Fri, 5 Jun 2026 18:17:25 +0400 Subject: [PATCH 108/167] Connect boostrap nodes concurrently --- storage/storage.nim | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/storage/storage.nim b/storage/storage.nim index 19488ad5..af4e378b 100644 --- a/storage/storage.nim +++ b/storage/storage.nim @@ -126,8 +126,11 @@ proc start*(s: StorageServer) {.async.} = await s.storageNode.start() # Connect to the bootstrap nodes in order to have connected peers - # for Autonat. - for spr in findReachableNodes(s.bootstrapNodes): + # for Autonat. The dials are run concurrently in case of + # a dead bootstrap node that could timeout. + proc connectBootstrapNode( + spr: SignedPeerRecord + ) {.async: (raises: [CancelledError]).} = try: let addrs = spr.data.addresses.mapIt(it.address) await s.storageNode.switch.connect(spr.data.peerId, addrs) @@ -136,6 +139,8 @@ proc start*(s: StorageServer) {.async.} = except CatchableError as e: warn "Cannot connect to bootstrap node", error = e.msg + await allFutures(findReachableNodes(s.bootstrapNodes).mapIt(connectBootstrapNode(it))) + # Start AutoNAT here (we own it, it is not in switch.services) so its first # probe targets the now-connected bootstrap peers instead of firing at # switch.start on no peers. From e44eccae955fab827ae55e1826c84f036ed72c79 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Fri, 5 Jun 2026 18:18:05 +0400 Subject: [PATCH 109/167] Add missing clientMode --- library/storage_thread_requests/requests/node_debug_request.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/library/storage_thread_requests/requests/node_debug_request.nim b/library/storage_thread_requests/requests/node_debug_request.nim index 2bbead69..b01fd444 100644 --- a/library/storage_thread_requests/requests/node_debug_request.nim +++ b/library/storage_thread_requests/requests/node_debug_request.nim @@ -64,6 +64,7 @@ proc getDebug( "table": table, "nat": { "reachability": reachabilityStr(storage[].autonatService), + "clientMode": node.discovery.protocol.clientMode, "relayRunning": storage[].autoRelayService.isSome and storage[].autoRelayService.get.isRunning, "portMapping": portMappingStr(storage[].natMapper), From 7ac6819b4e139cb2804781af576944fe51216ed5 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Fri, 5 Jun 2026 18:18:42 +0400 Subject: [PATCH 110/167] Update comment --- storage/storage.nim | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/storage/storage.nim b/storage/storage.nim index af4e378b..45200da1 100644 --- a/storage/storage.nim +++ b/storage/storage.nim @@ -141,9 +141,8 @@ proc start*(s: StorageServer) {.async.} = await allFutures(findReachableNodes(s.bootstrapNodes).mapIt(connectBootstrapNode(it))) - # Start AutoNAT here (we own it, it is not in switch.services) so its first - # probe targets the now-connected bootstrap peers instead of firing at - # switch.start on no peers. + # AutoNAT is not in switch.services: start it after the bootstrap dials + # so its first probe has peers to ask. if s.autonatService.isSome: await s.autonatService.get.start(s.storageNode.switch) From a46d7b18b388b1dc75152b8f28ece4a207189b21 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Fri, 5 Jun 2026 18:25:19 +0400 Subject: [PATCH 111/167] Add config validation --- storage/conf.nim | 21 +++++++++ storage/storage.nim | 11 +---- tests/storage/testconf.nim | 89 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 112 insertions(+), 9 deletions(-) create mode 100644 tests/storage/testconf.nim diff --git a/storage/conf.nim b/storage/conf.nim index 10e1f81b..5af6c497 100644 --- a/storage/conf.nim +++ b/storage/conf.nim @@ -394,6 +394,27 @@ func defaultAddress*(conf: StorageConf): IpAddress = func defaultNatConfig*(): NatConfig = result = NatConfig(hasExtIp: false, nat: NatStrategy.NatAuto) +func validateAutonatConfig*(config: StorageConf): ?!void = + # An autonat or relay server must be Reachable, assumed with extIp. + # In other words, a node cannot be autonat server AND autonat client. + # Currently, only bootstrap nodes should be autonat servers. + if config.autonatServer and not config.nat.hasExtIp: + return failure "--autonat-server requires --nat=extip:" + + if config.isRelayServer and not config.nat.hasExtIp: + return failure "--relay-server requires --nat=extip:" + + if config.natMaxQueueSize < 1: + return failure "--nat-max-queue-size must be at least 1" + + if config.natNumPeersToAsk < 1: + return failure "--nat-num-peers-to-ask must be at least 1" + + if config.natMinConfidence < 0.0 or config.natMinConfidence > 1.0: + return failure "--nat-min-confidence must be between 0 and 1" + + success() + proc getStorageVersion(): string = let tag = strip(staticExec("git describe --tags --abbrev=0")) if tag.isEmptyOrWhitespace: diff --git a/storage/storage.nim b/storage/storage.nim index 45200da1..08d3f71f 100644 --- a/storage/storage.nim +++ b/storage/storage.nim @@ -241,15 +241,8 @@ proc new*( ): StorageServer = ## create StorageServer including setting up datastore, repostore, etc - # Ensure that you can run an autonat server if the node is Reachable, assumed - # with extIp. - # In other words, a node cannot have autonat server AND autonat client. - # Currently, only bootstrap node should be autonat server. - if config.autonatServer and not config.nat.hasExtIp: - raise newException(StorageError, "--autonat-server requires --nat=extip:") - - if config.isRelayServer and not config.nat.hasExtIp: - raise newException(StorageError, "--relay-server requires --nat=extip:") + if err =? config.validateAutonatConfig().errorOption: + raise newException(StorageError, err.msg) # Switch let listenMultiAddr = getMultiAddrWithIpAndTcpPort(config.listenIp, config.listenPort) diff --git a/tests/storage/testconf.nim b/tests/storage/testconf.nim new file mode 100644 index 00000000..2a3e23c4 --- /dev/null +++ b/tests/storage/testconf.nim @@ -0,0 +1,89 @@ +import std/net +import pkg/questionable/results + +import ../asynctest +import ./helpers +import ../../storage/conf + +proc validConfig(): StorageConf = + StorageConf( + nat: defaultNatConfig(), + natMaxQueueSize: 3, + natNumPeersToAsk: 5, + natMinConfidence: 0.7, + ) + +suite "Conf - validateAutonatConfig": + test "accepts a valid config": + check validConfig().validateAutonatConfig().isOk + + test "rejects autonat server without extip": + var config = validConfig() + config.autonatServer = true + + check config.validateAutonatConfig().isErr + + test "accepts autonat server with extip": + var config = validConfig() + config.autonatServer = true + config.nat = NatConfig(hasExtIp: true, extIp: parseIpAddress("1.2.3.4")) + + check config.validateAutonatConfig().isOk + + test "rejects relay server without extip": + var config = validConfig() + config.isRelayServer = true + + check config.validateAutonatConfig().isErr + + test "accepts relay server with extip": + var config = validConfig() + config.isRelayServer = true + config.nat = NatConfig(hasExtIp: true, extIp: parseIpAddress("1.2.3.4")) + + check config.validateAutonatConfig().isOk + + test "rejects nat-max-queue-size below 1": + var config = validConfig() + config.natMaxQueueSize = 0 + + check config.validateAutonatConfig().isErr + + test "accepts nat-max-queue-size of 1": + var config = validConfig() + config.natMaxQueueSize = 1 + + check config.validateAutonatConfig().isOk + + test "rejects nat-num-peers-to-ask below 1": + var config = validConfig() + config.natNumPeersToAsk = 0 + + check config.validateAutonatConfig().isErr + + test "accepts nat-num-peers-to-ask of 1": + var config = validConfig() + config.natNumPeersToAsk = 1 + + check config.validateAutonatConfig().isOk + + test "rejects negative nat-min-confidence": + var config = validConfig() + config.natMinConfidence = -0.1 + + check config.validateAutonatConfig().isErr + + test "rejects nat-min-confidence above 1": + var config = validConfig() + config.natMinConfidence = 1.1 + + check config.validateAutonatConfig().isErr + + test "accepts nat-min-confidence bounds": + var config = validConfig() + + config.natMinConfidence = 0.0 + check config.validateAutonatConfig().isOk + + config.natMinConfidence = 1.0 + check config.validateAutonatConfig().isOk From 9a7554dbe93f671dd3338f4c9b233da1da5a56f1 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Fri, 5 Jun 2026 18:53:37 +0400 Subject: [PATCH 112/167] Use compilation flag to exclude nat simulation on release --- build.nims | 4 +++- storage/conf.nim | 1 + storage/rest/api.nim | 33 +++++++++++++++++---------------- storage/storage.nim | 30 ++++++++++++++++++++---------- 4 files changed, 41 insertions(+), 27 deletions(-) diff --git a/build.nims b/build.nims index 6318e459..2f46a6ab 100644 --- a/build.nims +++ b/build.nims @@ -72,7 +72,9 @@ task testStorage, "Build & run Logos Storage tests": task testIntegration, "Run integration tests": buildBinary "storage", outName = "storage", - params = "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE" + params = + "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE " & + "-d:storage_enable_nat_simulation=true" test "testIntegration" # use params to enable logging from the integration test executable # test "testIntegration", params = "-d:chronicles_sinks=textlines[notimestamps,stdout],textlines[dynamic] " & diff --git a/storage/conf.nim b/storage/conf.nim index 5af6c497..af23bd0d 100644 --- a/storage/conf.nim +++ b/storage/conf.nim @@ -73,6 +73,7 @@ proc defaultDataDir*(): string = const storage_enable_api_debug_peers* {.booldefine.} = false storage_enable_log_counter* {.booldefine.} = false + storage_enable_nat_simulation* {.booldefine.} = false DefaultThreadCount* = ThreadCount(0) diff --git a/storage/rest/api.nim b/storage/rest/api.nim index e9a14007..91edef58 100644 --- a/storage/rest/api.nim +++ b/storage/rest/api.nim @@ -629,26 +629,27 @@ proc initDebugApi( trace "Excepting processing request", exc = exc.msg return RestApiResponse.error(Http500, headers = headers) - router.api(MethodPost, "/api/storage/v1/debug/nat/filtering") do( - filtering: Option[string] - ) -> RestApiResponse: - var headers = buildCorsHeaders("POST", allowedOrigin) + when storage_enable_nat_simulation: + router.api(MethodPost, "/api/storage/v1/debug/nat/filtering") do( + filtering: Option[string] + ) -> RestApiResponse: + var headers = buildCorsHeaders("POST", allowedOrigin) - without natSimulation =? natRouter: - return RestApiResponse.error( - Http400, "NAT simulation not active on this node", headers = headers - ) + without natSimulation =? natRouter: + return RestApiResponse.error( + Http400, "NAT simulation not active on this node", headers = headers + ) - without res =? filtering and filtering =? res: - return - RestApiResponse.error(Http400, "Missing filtering value", headers = headers) + without res =? filtering and filtering =? res: + return + RestApiResponse.error(Http400, "Missing filtering value", headers = headers) - let behavior = FilteringBehavior.fromString(filtering).valueOr: - return - RestApiResponse.error(Http400, "Invalid filtering value", headers = headers) + let behavior = FilteringBehavior.fromString(filtering).valueOr: + return + RestApiResponse.error(Http400, "Invalid filtering value", headers = headers) - natSimulation.setFiltering(behavior) - return RestApiResponse.response("", headers = headers) + natSimulation.setFiltering(behavior) + return RestApiResponse.response("", headers = headers) when storage_enable_api_debug_peers: router.api(MethodGet, "/api/storage/v1/debug/peer/{peerId}") do( diff --git a/storage/storage.nim b/storage/storage.nim index 08d3f71f..ac32f0f2 100644 --- a/storage/storage.nim +++ b/storage/storage.nim @@ -317,17 +317,27 @@ proc new*( var natRouter: Option[NatRouter] let switch = - if config.natSimulation.isSome: - # Provide a NAT simulation useful for testing NAT Traversal - let filtering = FilteringBehavior.fromString(config.natSimulation.get).valueOr( - AddressAndPortDependent - ) - let router = NatRouter.new(filtering) - natRouter = some(router) - switchBuilder - .withNatTransport(router, {ServerFlags.ReuseAddr, ServerFlags.TcpNoDelay}) - .build() + when storage_enable_nat_simulation: + if config.natSimulation.isSome: + # Provide a NAT simulation useful for testing NAT Traversal + let filtering = FilteringBehavior.fromString(config.natSimulation.get).valueOr( + AddressAndPortDependent + ) + let router = NatRouter.new(filtering) + natRouter = some(router) + switchBuilder + .withNatTransport(router, {ServerFlags.ReuseAddr, ServerFlags.TcpNoDelay}) + .build() + else: + switchBuilder + .withTcpTransport({ServerFlags.ReuseAddr, ServerFlags.TcpNoDelay}) + .build() else: + if config.natSimulation.isSome: + raise newException( + StorageError, + "--nat-simulation requires a build with -d:storage_enable_nat_simulation=true", + ) switchBuilder .withTcpTransport({ServerFlags.ReuseAddr, ServerFlags.TcpNoDelay}) .build() From 10c406d9f054adcfd48d50acc2f1e7badea8d6c1 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Fri, 5 Jun 2026 18:54:06 +0400 Subject: [PATCH 113/167] Add comment --- storage/utils/addrutils.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/storage/utils/addrutils.nim b/storage/utils/addrutils.nim index ae5441fa..abc3de07 100644 --- a/storage/utils/addrutils.nim +++ b/storage/utils/addrutils.nim @@ -23,6 +23,7 @@ func remapAddr*( ): MultiAddress = ## Remap addresses to new IP, port, and/or transport protocol (e.g. "tcp" → "udp") ## + ## Assumes a /ip4|ip6//tcp|udp/ address: anything else crashes (Defect). var parts = ($address).split("/") From e9d6c5c0b94566b718877dfb4ea5684189243315 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Fri, 5 Jun 2026 18:58:26 +0400 Subject: [PATCH 114/167] Fix import --- tests/storage/testconf.nim | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/storage/testconf.nim b/tests/storage/testconf.nim index 2a3e23c4..2a697f4d 100644 --- a/tests/storage/testconf.nim +++ b/tests/storage/testconf.nim @@ -4,6 +4,7 @@ import pkg/questionable/results import ../asynctest import ./helpers import ../../storage/conf +import ../../storage/nat proc validConfig(): StorageConf = StorageConf( @@ -26,7 +27,7 @@ suite "Conf - validateAutonatConfig": test "accepts autonat server with extip": var config = validConfig() config.autonatServer = true - config.nat = NatConfig(hasExtIp: true, extIp: parseIpAddress("1.2.3.4")) + config.nat = nat.NatConfig(hasExtIp: true, extIp: parseIpAddress("1.2.3.4")) check config.validateAutonatConfig().isOk @@ -39,7 +40,7 @@ suite "Conf - validateAutonatConfig": test "accepts relay server with extip": var config = validConfig() config.isRelayServer = true - config.nat = NatConfig(hasExtIp: true, extIp: parseIpAddress("1.2.3.4")) + config.nat = nat.NatConfig(hasExtIp: true, extIp: parseIpAddress("1.2.3.4")) check config.validateAutonatConfig().isOk From aa28578ca7d5103e9113ae5ec14285c1f10e7947 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Fri, 5 Jun 2026 19:33:21 +0400 Subject: [PATCH 115/167] Expose DHT addresses --- .../storage_thread_requests/requests/node_debug_request.nim | 1 + openapi.yaml | 5 +++++ storage/discovery.nim | 6 ++---- storage/rest/api.nim | 1 + 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/library/storage_thread_requests/requests/node_debug_request.nim b/library/storage_thread_requests/requests/node_debug_request.nim index b01fd444..8d5fe2f4 100644 --- a/library/storage_thread_requests/requests/node_debug_request.nim +++ b/library/storage_thread_requests/requests/node_debug_request.nim @@ -61,6 +61,7 @@ proc getDebug( "addrs": node.switch.peerInfo.addrs.mapIt($it), "spr": nodeSpr.toURI, "announceAddresses": node.discovery.announceAddrs, + "dhtAddresses": node.discovery.dhtAddrs, "table": table, "nat": { "reachability": reachabilityStr(storage[].autonatService), diff --git a/openapi.yaml b/openapi.yaml index 3608cc75..c310efa2 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -106,6 +106,7 @@ components: - repo - spr - announceAddresses + - dhtAddresses - table - storage properties: @@ -124,6 +125,10 @@ components: type: array items: $ref: "#/components/schemas/MultiAddress" + dhtAddresses: + type: array + items: + $ref: "#/components/schemas/MultiAddress" table: $ref: "#/components/schemas/PeersTable" storage: diff --git a/storage/discovery.nim b/storage/discovery.nim index 87c88262..c7f17403 100644 --- a/storage/discovery.nim +++ b/storage/discovery.nim @@ -43,7 +43,7 @@ type Discovery* = ref object of RootObj providerRecord*: ?SignedPeerRecord # record to advertice node connection information, this carry any # address that the node can be connected on - dhtRecord*: ?SignedPeerRecord # record to advertice DHT connection information + dhtAddrs*: seq[MultiAddress] # UDP discovery addresses, exposed for debugging isStarted: bool store: Datastore @@ -191,12 +191,10 @@ proc announceDirectAddrs*( info "Updating announce and DHT records", tcpAddrs, udpAddrs d.announceAddrs = tcpAddrs + d.dhtAddrs = udpAddrs d.providerRecord = SignedPeerRecord .init(d.key, PeerRecord.init(d.peerId, tcpAddrs)) .expect("Should construct signed record").some - d.dhtRecord = SignedPeerRecord - .init(d.key, PeerRecord.init(d.peerId, udpAddrs)) - .expect("Should construct signed record").some if not d.protocol.isNil: let spr = SignedPeerRecord diff --git a/storage/rest/api.nim b/storage/rest/api.nim index 91edef58..c251d406 100644 --- a/storage/rest/api.nim +++ b/storage/rest/api.nim @@ -585,6 +585,7 @@ proc initDebugApi( "repo": $conf.dataDir, "spr": nodeSpr.toURI, "announceAddresses": node.discovery.announceAddrs, + "dhtAddresses": node.discovery.dhtAddrs, "table": table, "storage": {"version": $storageVersion, "revision": $storageRevision}, "nat": { From 37ba19221a0e0fa5cf2fa83c10df427799c514f1 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Fri, 5 Jun 2026 19:49:29 +0400 Subject: [PATCH 116/167] Cleanup --- storage/nat.nim | 37 +++++++++++++++---------------------- storage/storage.nim | 2 -- storage/utils/addrutils.nim | 30 +++++++++--------------------- storage/utils/natutils.nim | 4 ---- 4 files changed, 24 insertions(+), 49 deletions(-) diff --git a/storage/nat.nim b/storage/nat.nim index 9facf336..0189480f 100644 --- a/storage/nat.nim +++ b/storage/nat.nim @@ -49,6 +49,19 @@ type NatPortMapper* = ref object of RootObj plumInitialized: bool closed: bool +proc resetMappings(m: NatPortMapper) = + if m.tcpMappingId.isSome: + destroyMapping(m.tcpMappingId.get) + m.tcpMappingId = none(cint) + + if m.udpMappingId.isSome: + destroyMapping(m.udpMappingId.get) + m.udpMappingId = none(cint) + + m.activeMappingProtocol = none(MappingProtocol) + m.activeTcpPort = none(Port) + m.activeUdpPort = none(Port) + method mapNatPorts*( m: NatPortMapper ): Future[Option[(Port, Port, MappingProtocol)]] {. @@ -79,17 +92,7 @@ method mapNatPorts*( # If there is only one mapping, something went wrong somewhere # so we delete the mappings to recreate them. - if m.tcpMappingId.isSome: - destroyMapping(m.tcpMappingId.get) - m.tcpMappingId = none(cint) - - if m.udpMappingId.isSome: - destroyMapping(m.udpMappingId.get) - m.udpMappingId = none(cint) - - m.activeMappingProtocol = none(MappingProtocol) - m.activeTcpPort = none(Port) - m.activeUdpPort = none(Port) + m.resetMappings() let tcpRes = await createMapping(TCP, m.tcpPort.uint16, m.tcpPort.uint16) if tcpRes.isErr: @@ -111,17 +114,7 @@ method mapNatPorts*( some((m.activeTcpPort.get, m.activeUdpPort.get, m.activeMappingProtocol.get)) proc close*(m: NatPortMapper) = - if m.tcpMappingId.isSome: - destroyMapping(m.tcpMappingId.get) - m.tcpMappingId = none(cint) - - if m.udpMappingId.isSome: - destroyMapping(m.udpMappingId.get) - m.udpMappingId = none(cint) - - m.activeMappingProtocol = none(MappingProtocol) - m.activeTcpPort = none(Port) - m.activeUdpPort = none(Port) + m.resetMappings() if m.plumInitialized: discard cleanup() diff --git a/storage/storage.nim b/storage/storage.nim index ac32f0f2..20c1e018 100644 --- a/storage/storage.nim +++ b/storage/storage.nim @@ -61,7 +61,6 @@ type autonatService*: Option[AutonatV2Service] autoRelayService*: Option[AutoRelayService] natMapper*: Option[NatPortMapper] - natRouter*: Option[NatRouter] holePunchHandler: Option[connmanager.PeerEventHandler] peerInfoObserver: Option[PeerInfoObserver] bootstrapNodes: seq[SignedPeerRecord] @@ -539,7 +538,6 @@ proc new*( autonatService: autonatService, autoRelayService: autoRelayService, natMapper: natMapper, - natRouter: natRouter, holePunchHandler: holePunchHandler, peerInfoObserver: peerInfoObserver, bootstrapNodes: bootstrapNodes, diff --git a/storage/utils/addrutils.nim b/storage/utils/addrutils.nim index abc3de07..600a38f5 100644 --- a/storage/utils/addrutils.nim +++ b/storage/utils/addrutils.nim @@ -14,6 +14,7 @@ import std/strutils import std/options import pkg/libp2p +import pkg/stew/endians2 func remapAddr*( address: MultiAddress, @@ -47,28 +48,15 @@ func remapAddr*( MultiAddress.init(parts.join("/")).expect("Should construct multiaddress") -proc getMultiAddrWithIPAndUDPPort*(ip: IpAddress, port: Port): MultiAddress = - ## Creates a MultiAddress with the specified IP address and UDP port - ## - ## Parameters: - ## - ip: A valid IP address (IPv4 or IPv6) - ## - port: The UDP port number - ## - ## Returns: - ## A MultiAddress in the format "/ip4/
/udp/" or "/ip6/
/udp/" - - let ipFamily = if ip.family == IpAddressFamily.IPv4: "/ip4/" else: "/ip6/" - return MultiAddress.init(ipFamily & $ip & "/udp/" & $port).expect("valid multiaddr") - func getTcpPort*(ma: MultiAddress): Option[Port] = - let parts = ($ma).split("/") - for i, part in parts: - if part == "tcp" and i + 1 < parts.len: - try: - return some(Port(parseInt(parts[i + 1]))) - except ValueError: - return Port.none - Port.none + ## Extracts the TCP port from a multiaddress; none when there is no TCP part. + let tcpPart = ma[multiCodec("tcp")] + if tcpPart.isErr: + return Port.none + let portBytes = tcpPart.get().protoArgument() + if portBytes.isErr or portBytes.get().len != 2: + return Port.none + some(Port(fromBytesBE(uint16, portBytes.get()))) proc getMultiAddrWithIpAndTcpPort*(ip: IpAddress, port: Port): MultiAddress = ## Creates a MultiAddress with the specified IP address and TCP port diff --git a/storage/utils/natutils.nim b/storage/utils/natutils.nim index f96d4579..82d0edc8 100644 --- a/storage/utils/natutils.nim +++ b/storage/utils/natutils.nim @@ -1,15 +1,11 @@ {.push raises: [].} import std/[options, net] -import pkg/chronicles import results import libplum/plum import libplum/libplum export plum, libplum, results, options, net -logScope: - topics = "nat" - type NatStrategy* = enum NatAuto From 8e2750b080c4d45e083b394c2db4b84ae316996f Mon Sep 17 00:00:00 2001 From: Arnaud Date: Fri, 5 Jun 2026 19:51:54 +0400 Subject: [PATCH 117/167] Update nim-libplum URL --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index b89b3039..6c323ffb 100644 --- a/.gitmodules +++ b/.gitmodules @@ -193,7 +193,7 @@ branch = main [submodule "vendor/nim-libplum"] path = vendor/nim-libplum - url = https://github.com/2-towns/nim-libplum.git + url = https://github.com/logos-storage/nim-libplum.git [submodule "vendor/nim-nat-traversal"] path = vendor/nim-nat-traversal url = https://github.com/status-im/nim-nat-traversal.git From 7bec4ad0597cc3efaeeae0f7f79eeb25423fea25 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Fri, 5 Jun 2026 19:53:42 +0400 Subject: [PATCH 118/167] Remove unused import --- library/storage_thread_requests/requests/node_info_request.nim | 1 - 1 file changed, 1 deletion(-) diff --git a/library/storage_thread_requests/requests/node_info_request.nim b/library/storage_thread_requests/requests/node_info_request.nim index 7c0d818e..931e8ca6 100644 --- a/library/storage_thread_requests/requests/node_info_request.nim +++ b/library/storage_thread_requests/requests/node_info_request.nim @@ -1,6 +1,5 @@ ## This file contains the lifecycle request type that will be handled. -import std/[options] import chronos import chronicles import confutils From cd4e4aa6f7cbc7ea1cbb58ab2d411fb05f410fcd Mon Sep 17 00:00:00 2001 From: Arnaud Date: Wed, 10 Jun 2026 10:54:02 +0400 Subject: [PATCH 119/167] Add address policy filter --- storage/conf.nim | 2 +- storage/storage.nim | 14 ++++++++++-- storage/utils/addrutils.nim | 23 ++++++++++++++++++- tests/storage/testaddrutils.nim | 40 +++++++++++++++++++++++++++++++++ 4 files changed, 75 insertions(+), 4 deletions(-) diff --git a/storage/conf.nim b/storage/conf.nim index af23bd0d..618d3d4f 100644 --- a/storage/conf.nim +++ b/storage/conf.nim @@ -53,7 +53,7 @@ export DefaultQuotaBytes, DefaultBlockTtl, DefaultBlockInterval, DefaultNumBlocksPerInterval, DefaultBlockRetries -const DefaultNatScheduleInterval* = 5.minutes +const DefaultNatScheduleInterval* = 2.minutes type ThreadCount* = distinct Natural diff --git a/storage/storage.nim b/storage/storage.nim index 20c1e018..958dcd64 100644 --- a/storage/storage.nim +++ b/storage/storage.nim @@ -313,6 +313,11 @@ proc new*( switchBuilder = switchBuilder.withObservedAddrManager( ObservedAddrManager.new(minCount = observedAddrMinCount) ) + # libp2p keeps the private address in peerInfo.addrs. + # Since Autonat V2 uses the observed public address, + # we can filter the private addresses to keep only the dialable + # addresses. + switchBuilder = switchBuilder.withAddressPolicy(dialableAddressPolicy) var natRouter: Option[NatRouter] let switch = @@ -469,9 +474,14 @@ proc new*( maxNumRelays = config.natMaxRelays, client = relayClient, onReservation = proc(addresses: seq[MultiAddress]) {.gcsafe, raises: [].} = - info "Relay reservation updated", addresses + # A relay server is required to have a public extip, so its + # circuit addresses always include a public one. The relay's reservation + # response can also carry loopback/private addresses: + # they are never dialable by a remote peer, so drop them. + let publicAddrs = addresses.filterIt(it.hasPublicRelayTransport()) + info "Relay reservation updated", addresses = publicAddrs # relay addresses are for download traffic only, not DHT routing - discovery.announceRelayAddrs(addresses), + discovery.announceRelayAddrs(publicAddrs), rng = random.Rng.instance(), ) diff --git a/storage/utils/addrutils.nim b/storage/utils/addrutils.nim index 600a38f5..f142cfec 100644 --- a/storage/utils/addrutils.nim +++ b/storage/utils/addrutils.nim @@ -14,6 +14,7 @@ import std/strutils import std/options import pkg/libp2p +import pkg/libp2p/wire import pkg/stew/endians2 func remapAddr*( @@ -58,9 +59,29 @@ func getTcpPort*(ma: MultiAddress): Option[Port] = return Port.none some(Port(fromBytesBE(uint16, portBytes.get()))) +proc hasPublicRelayTransport*(ma: MultiAddress): bool = + ## True when ``ma`` is a circuit address whose relay is publicly dialable. + ## A circuit address is /p2p//p2p-circuit; the part + ## before /p2p/ is the relay's wire address, which isPublicMA can check. + ## Unlike libp2p's publicRoutableAddressPolicy we drop non-public relays: our + ## relay path only runs on a genuine NAT, where the relay is always public. + let relayWireStr = ($ma).split("/p2p/")[0] + let relayWireAddr = MultiAddress.init(relayWireStr).valueOr: + return false + relayWireAddr.isPublicMA() + +proc dialableAddressPolicy*(ma: MultiAddress): bool {.gcsafe, raises: [].} = + # Use with switchBuilder.withAddressPolicy. + # Filter the peerInfo.addrs updated by libp2p without + # declaring another address mapper. + if ma.isCircuitRelayMA(): + ma.hasPublicRelayTransport() + else: + ma.isPublicMA() + proc getMultiAddrWithIpAndTcpPort*(ip: IpAddress, port: Port): MultiAddress = ## Creates a MultiAddress with the specified IP address and TCP port - ## + ## ## Parameters: ## - ip: A valid IP address (IPv4 or IPv6) ## - port: The TCP port number diff --git a/tests/storage/testaddrutils.nim b/tests/storage/testaddrutils.nim index b336fb1a..fc119e56 100644 --- a/tests/storage/testaddrutils.nim +++ b/tests/storage/testaddrutils.nim @@ -35,3 +35,43 @@ suite "addrutils - remapAddr": let ma = MultiAddress.init("/ip4/1.2.3.4/tcp/5000").expect("valid") let remapped = ma.remapAddr(ip = some(parseIpAddress("8.8.8.8"))) check remapped == MultiAddress.init("/ip4/8.8.8.8/tcp/5000").expect("valid") + +suite "addrutils - hasPublicRelayTransport": + const relayId = "16Uiu2HAkyRvHo1AyyQY1xiHC8QbYjXCHkZbneVC8dBtJjp1SZcGD" + + proc circuitAddr(relayIp: string): MultiAddress = + MultiAddress.init("/ip4/" & relayIp & "/tcp/8070/p2p/" & relayId & "/p2p-circuit").expect( + "valid" + ) + + test "true when the relay has a public ip": + check circuitAddr("204.168.234.45").hasPublicRelayTransport() + + test "false when the relay is loopback": + check not circuitAddr("127.0.0.1").hasPublicRelayTransport() + + test "false when the relay is a private ip": + check not circuitAddr("172.17.0.1").hasPublicRelayTransport() + +suite "addrutils - dialableAddressPolicy": + const relayId = "16Uiu2HAkyRvHo1AyyQY1xiHC8QbYjXCHkZbneVC8dBtJjp1SZcGD" + + proc circuitAddr(relayIp: string): MultiAddress = + MultiAddress.init("/ip4/" & relayIp & "/tcp/8070/p2p/" & relayId & "/p2p-circuit").expect( + "valid" + ) + + test "keeps a public direct address": + check MultiAddress.init("/ip4/204.168.234.45/tcp/8070").expect("valid").dialableAddressPolicy() + + test "drops a loopback direct address": + check not MultiAddress.init("/ip4/127.0.0.1/tcp/8070").expect("valid").dialableAddressPolicy() + + test "drops a private direct address": + check not MultiAddress.init("/ip4/192.168.100.103/tcp/8070").expect("valid").dialableAddressPolicy() + + test "keeps a circuit address through a public relay": + check circuitAddr("204.168.234.45").dialableAddressPolicy() + + test "drops a circuit address through a private relay": + check not circuitAddr("172.17.0.1").dialableAddressPolicy() From 17155488a20973babc19565ea705f67ac307d4f2 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Wed, 10 Jun 2026 10:54:24 +0400 Subject: [PATCH 120/167] Simplify comment --- storage/utils/addrutils.nim | 4 ---- 1 file changed, 4 deletions(-) diff --git a/storage/utils/addrutils.nim b/storage/utils/addrutils.nim index f142cfec..add58d40 100644 --- a/storage/utils/addrutils.nim +++ b/storage/utils/addrutils.nim @@ -61,10 +61,6 @@ func getTcpPort*(ma: MultiAddress): Option[Port] = proc hasPublicRelayTransport*(ma: MultiAddress): bool = ## True when ``ma`` is a circuit address whose relay is publicly dialable. - ## A circuit address is /p2p//p2p-circuit; the part - ## before /p2p/ is the relay's wire address, which isPublicMA can check. - ## Unlike libp2p's publicRoutableAddressPolicy we drop non-public relays: our - ## relay path only runs on a genuine NAT, where the relay is always public. let relayWireStr = ($ma).split("/p2p/")[0] let relayWireAddr = MultiAddress.init(relayWireStr).valueOr: return false From b33031cdef0a2ffb01e925f1c16246b4f20d1e0c Mon Sep 17 00:00:00 2001 From: Arnaud Date: Fri, 12 Jun 2026 09:20:04 +0400 Subject: [PATCH 121/167] Update libp2p to introduce enableDialableCandidates --- storage/storage.nim | 1 + tests/storage/testaddrutils.nim | 27 ++++++++++++++++++--------- tests/storage/testconf.nim | 2 -- tests/storage/testnat.nim | 11 +++++++---- 4 files changed, 26 insertions(+), 15 deletions(-) diff --git a/storage/storage.nim b/storage/storage.nim index 958dcd64..c933bc27 100644 --- a/storage/storage.nim +++ b/storage/storage.nim @@ -302,6 +302,7 @@ proc new*( numPeersToAsk = config.natNumPeersToAsk, maxQueueSize = config.natMaxQueueSize, minConfidence = config.natMinConfidence, + enableDialableCandidates = true, ) ) # At the first AutoNAT probe, the only identify observations available come diff --git a/tests/storage/testaddrutils.nim b/tests/storage/testaddrutils.nim index fc119e56..94c3db76 100644 --- a/tests/storage/testaddrutils.nim +++ b/tests/storage/testaddrutils.nim @@ -40,9 +40,9 @@ suite "addrutils - hasPublicRelayTransport": const relayId = "16Uiu2HAkyRvHo1AyyQY1xiHC8QbYjXCHkZbneVC8dBtJjp1SZcGD" proc circuitAddr(relayIp: string): MultiAddress = - MultiAddress.init("/ip4/" & relayIp & "/tcp/8070/p2p/" & relayId & "/p2p-circuit").expect( - "valid" - ) + MultiAddress + .init("/ip4/" & relayIp & "/tcp/8070/p2p/" & relayId & "/p2p-circuit") + .expect("valid") test "true when the relay has a public ip": check circuitAddr("204.168.234.45").hasPublicRelayTransport() @@ -57,18 +57,27 @@ suite "addrutils - dialableAddressPolicy": const relayId = "16Uiu2HAkyRvHo1AyyQY1xiHC8QbYjXCHkZbneVC8dBtJjp1SZcGD" proc circuitAddr(relayIp: string): MultiAddress = - MultiAddress.init("/ip4/" & relayIp & "/tcp/8070/p2p/" & relayId & "/p2p-circuit").expect( - "valid" - ) + MultiAddress + .init("/ip4/" & relayIp & "/tcp/8070/p2p/" & relayId & "/p2p-circuit") + .expect("valid") test "keeps a public direct address": - check MultiAddress.init("/ip4/204.168.234.45/tcp/8070").expect("valid").dialableAddressPolicy() + check MultiAddress + .init("/ip4/204.168.234.45/tcp/8070") + .expect("valid") + .dialableAddressPolicy() test "drops a loopback direct address": - check not MultiAddress.init("/ip4/127.0.0.1/tcp/8070").expect("valid").dialableAddressPolicy() + check not MultiAddress + .init("/ip4/127.0.0.1/tcp/8070") + .expect("valid") + .dialableAddressPolicy() test "drops a private direct address": - check not MultiAddress.init("/ip4/192.168.100.103/tcp/8070").expect("valid").dialableAddressPolicy() + check not MultiAddress + .init("/ip4/192.168.100.103/tcp/8070") + .expect("valid") + .dialableAddressPolicy() test "keeps a circuit address through a public relay": check circuitAddr("204.168.234.45").dialableAddressPolicy() diff --git a/tests/storage/testconf.nim b/tests/storage/testconf.nim index 2a697f4d..ac096568 100644 --- a/tests/storage/testconf.nim +++ b/tests/storage/testconf.nim @@ -1,6 +1,4 @@ import std/net -import pkg/questionable/results - import ../asynctest import ./helpers import ../../storage/conf diff --git a/tests/storage/testnat.nim b/tests/storage/testnat.nim index 9b367fa5..4f2c81df 100644 --- a/tests/storage/testnat.nim +++ b/tests/storage/testnat.nim @@ -194,11 +194,14 @@ asyncchecksuite "NAT - handleNatStatus": check disc.announceAddrs == newSeq[MultiAddress]() test "autonat dial request includes the observed addresses as candidates": - # Reproduces vacp2p/nim-libp2p#2600: until that fix is vendored, the - # dial request only contains peerInfo.addrs (private listen addrs), so - # a NATed node never submits a dialable candidate. + # The dial request includes the addresses observed by other peers, so a NATed node submits + # a dialable candidate even though its listen addrs are private. let client = MockAutonatV2Client() - let autonat = AutonatV2Service.new(Rng.instance(), client) + let autonat = AutonatV2Service.new( + Rng.instance(), + client, + AutonatV2ServiceConfig.new(enableDialableCandidates = true), + ) service.setup(autonat, sw) await autonat.start(sw) From 37fd43221f05253e4c6115806acdfe5619195bd0 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Fri, 12 Jun 2026 14:42:26 +0400 Subject: [PATCH 122/167] Refactor tests --- build.nims | 11 +- openapi.yaml | 23 -- storage/conf.nim | 9 - storage/nat.nim | 7 +- storage/rest/api.nim | 27 +- storage/storage.nim | 44 +--- tests/integration/1_minute/testnat.nim | 132 ---------- .../integration/5_minutes/testnatdownload.nim | 74 ------ .../5_minutes/testrestapivalidation.nim | 22 -- tests/integration/storageclient.nim | 9 - tests/integration/storageconfig.nim | 9 - tests/nat/testnatpcp.nim | 93 ------- tests/nat/testnatupnp.nim | 94 ------- .../utils => tests/storage}/natsimulation.nim | 11 +- tests/storage/testnatdetection.nim | 241 ++++++++++++++++++ .../{testnat.nim => testnatreaction.nim} | 117 ++++----- tests/storage/testnatsimulation.nim | 12 +- vendor/nim-boringssl | 2 +- vendor/nim-lsquic | 2 +- vendor/nim-protobuf-serialization | 2 +- 20 files changed, 326 insertions(+), 615 deletions(-) delete mode 100644 tests/integration/1_minute/testnat.nim delete mode 100644 tests/integration/5_minutes/testnatdownload.nim delete mode 100644 tests/nat/testnatpcp.nim delete mode 100644 tests/nat/testnatupnp.nim rename {storage/utils => tests/storage}/natsimulation.nim (92%) create mode 100644 tests/storage/testnatdetection.nim rename tests/storage/{testnat.nim => testnatreaction.nim} (69%) diff --git a/build.nims b/build.nims index 2f46a6ab..1eb7acbb 100644 --- a/build.nims +++ b/build.nims @@ -72,9 +72,7 @@ task testStorage, "Build & run Logos Storage tests": task testIntegration, "Run integration tests": buildBinary "storage", outName = "storage", - params = - "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE " & - "-d:storage_enable_nat_simulation=true" + params = "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE" test "testIntegration" # use params to enable logging from the integration test executable # test "testIntegration", params = "-d:chronicles_sinks=textlines[notimestamps,stdout],textlines[dynamic] " & @@ -94,6 +92,13 @@ task testNatPcpMapping, "Run PCP NAT integration test (requires miniupnpd contai putEnv("STORAGE_INTEGRATION_TEST_INCLUDES", "nat/testnatpcp.nim") test "testIntegration", outName = "testIntegrationNatPcp" +task testNatNotReachable, + "Run NAT not-reachable scenario (needs the image + podman-compose)": + test "integration/nat/not-reachable/testnotreachable", outName = "testNatNotReachable" + +task testNatReachable, "Run NAT reachable scenario (needs the image + podman-compose)": + test "integration/nat/reachable/testreachable", outName = "testNatReachable" + task build, "build Logos Storage binary": storageTask() diff --git a/openapi.yaml b/openapi.yaml index c310efa2..9bb3ab03 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -564,29 +564,6 @@ paths: "500": description: Well it was bad-bad - "/debug/nat/filtering": - post: - summary: "Set NAT simulation filtering behavior at runtime" - description: "Only available on nodes started with --nat-simulation. Used for testing NAT transitions." - tags: [Debug] - operationId: setNatFiltering - - parameters: - - in: query - name: filtering - required: true - schema: - type: string - enum: [endpoint-independent, address-dependent, address-and-port-dependent, double-nat] - - responses: - "200": - description: Filtering behavior updated successfully - "400": - description: Missing or invalid filtering value, or NAT simulation not active - "500": - description: Internal error - "/debug/info": get: summary: "Gets node information" diff --git a/storage/conf.nim b/storage/conf.nim index 618d3d4f..b1ea573c 100644 --- a/storage/conf.nim +++ b/storage/conf.nim @@ -73,7 +73,6 @@ proc defaultDataDir*(): string = const storage_enable_api_debug_peers* {.booldefine.} = false storage_enable_log_counter* {.booldefine.} = false - storage_enable_nat_simulation* {.booldefine.} = false DefaultThreadCount* = ThreadCount(0) @@ -368,14 +367,6 @@ type name: "nat-port-mapping-recheck-period" .}: int - natSimulation* {. - desc: - "Simulate NAT filtering behavior for testing: endpoint-independent, address-dependent, address-and-port-dependent", - defaultValue: string.none, - name: "nat-simulation", - hidden - .}: Option[string] - autonatServer* {. desc: "Enable AutoNAT server to help other nodes check their reachability", defaultValue: false, diff --git a/storage/nat.nim b/storage/nat.nim index 0189480f..cbda208e 100644 --- a/storage/nat.nim +++ b/storage/nat.nim @@ -128,6 +128,9 @@ proc stop*(m: NatPortMapper) = proc isPortMapped*(m: NatPortMapper, port: Port): bool = m.activeTcpPort.isSome and m.activeTcpPort.get == port +method hasActiveMapping*(m: NatPortMapper): bool {.base, gcsafe.} = + m.tcpMappingId.isSome and m.udpMappingId.isSome + proc announcePeerInfoAddrs*(discovery: Discovery, peerInfo: PeerInfo, udpPort: Port) = ## Announces peerInfo.addrs to the DHT, excluding relay circuit addresses: ## they are announced via onReservation and must not enter the DHT routing @@ -192,11 +195,11 @@ method handleNatStatus*( if dialBackAddr.isNone: warn "Got empty dialback address in AutoNat when node is NotReachable" - if m.tcpMappingId.isSome and m.udpMappingId.isSome: + if m.hasActiveMapping(): m.close() discovery.announceDirectAddrs(@[], udpPort = discoveryPort) - elif m.tcpMappingId.isSome and m.udpMappingId.isSome: + elif m.hasActiveMapping(): warn "Not Reachable with active port mapping. The port mapping will be deleted and relay will start." # The mapping was created the the node is still not reachable. diff --git a/storage/rest/api.nim b/storage/rest/api.nim index c251d406..597ff321 100644 --- a/storage/rest/api.nim +++ b/storage/rest/api.nim @@ -40,7 +40,6 @@ import ../stores/repostore import ../blockexchange import ../units import ../utils/options -import ../utils/natsimulation import ../nat import ./coders @@ -565,7 +564,6 @@ proc initDebugApi( autonat: Option[AutonatV2Service], autoRelay: Option[AutoRelayService], natMapper: Option[NatPortMapper], - natRouter: Option[NatRouter], router: var RestRouter, ) = let allowedOrigin = router.allowedOrigin @@ -630,28 +628,6 @@ proc initDebugApi( trace "Excepting processing request", exc = exc.msg return RestApiResponse.error(Http500, headers = headers) - when storage_enable_nat_simulation: - router.api(MethodPost, "/api/storage/v1/debug/nat/filtering") do( - filtering: Option[string] - ) -> RestApiResponse: - var headers = buildCorsHeaders("POST", allowedOrigin) - - without natSimulation =? natRouter: - return RestApiResponse.error( - Http400, "NAT simulation not active on this node", headers = headers - ) - - without res =? filtering and filtering =? res: - return - RestApiResponse.error(Http400, "Missing filtering value", headers = headers) - - let behavior = FilteringBehavior.fromString(filtering).valueOr: - return - RestApiResponse.error(Http400, "Invalid filtering value", headers = headers) - - natSimulation.setFiltering(behavior) - return RestApiResponse.response("", headers = headers) - when storage_enable_api_debug_peers: router.api(MethodGet, "/api/storage/v1/debug/peer/{peerId}") do( peerId: PeerId @@ -679,13 +655,12 @@ proc initRestApi*( autonat: Option[AutonatV2Service], autoRelay: Option[AutoRelayService], natMapper: Option[NatPortMapper], - natRouter: Option[NatRouter], corsAllowedOrigin: ?string, ): RestRouter = var router = RestRouter.init(validate, corsAllowedOrigin) initDataApi(node, repoStore, router) initNodeApi(node, conf, router) - initDebugApi(node, conf, autonat, autoRelay, natMapper, natRouter, router) + initDebugApi(node, conf, autonat, autoRelay, natMapper, router) return router diff --git a/storage/storage.nim b/storage/storage.nim index c933bc27..a7c1b2c6 100644 --- a/storage/storage.nim +++ b/storage/storage.nim @@ -22,6 +22,7 @@ import pkg/libp2p/protocols/connectivity/autonatv2/[service, client] import pkg/libp2p/protocols/connectivity/relay/client as relayClientModule import pkg/libp2p/protocols/connectivity/relay/relay as relayModule import pkg/libp2p/services/autorelayservice +import pkg/libp2p/transports/tcptransport import pkg/confutils import pkg/confutils/defs import pkg/stew/io2 @@ -43,11 +44,15 @@ import ./storagetypes import ./logutils import ./nat import ./utils/natutils -import ./utils/natsimulation logScope: topics = "storage node" +const StorageTransportFlags = {ServerFlags.ReuseAddr, ServerFlags.TcpNoDelay} + +proc tcpTransportBuilder(config: TransportConfig): Transport {.gcsafe, raises: [].} = + TcpTransport.new(StorageTransportFlags, config.upgr) + type StorageServer* = ref object config: StorageConf @@ -237,8 +242,10 @@ proc new*( config: StorageConf, privateKey: StoragePrivateKey, logFile: Option[IoHandle] = IoHandle.none, + transportBuilder: TransportBuilder = tcpTransportBuilder, ): StorageServer = - ## create StorageServer including setting up datastore, repostore, etc + ## create StorageServer including setting up datastore, repostore, etc. + ## ``transportBuilder`` defaults to TCP; tests inject a simulated NAT transport. if err =? config.validateAutonatConfig().errorOption: raise newException(StorageError, err.msg) @@ -320,32 +327,7 @@ proc new*( # addresses. switchBuilder = switchBuilder.withAddressPolicy(dialableAddressPolicy) - var natRouter: Option[NatRouter] - let switch = - when storage_enable_nat_simulation: - if config.natSimulation.isSome: - # Provide a NAT simulation useful for testing NAT Traversal - let filtering = FilteringBehavior.fromString(config.natSimulation.get).valueOr( - AddressAndPortDependent - ) - let router = NatRouter.new(filtering) - natRouter = some(router) - switchBuilder - .withNatTransport(router, {ServerFlags.ReuseAddr, ServerFlags.TcpNoDelay}) - .build() - else: - switchBuilder - .withTcpTransport({ServerFlags.ReuseAddr, ServerFlags.TcpNoDelay}) - .build() - else: - if config.natSimulation.isSome: - raise newException( - StorageError, - "--nat-simulation requires a build with -d:storage_enable_nat_simulation=true", - ) - switchBuilder - .withTcpTransport({ServerFlags.ReuseAddr, ServerFlags.TcpNoDelay}) - .build() + let switch = switchBuilder.withTransport(transportBuilder).build() var taskPool: Taskpool @@ -500,10 +482,6 @@ proc new*( ) ) - # natRouter is some only when using nat simulation - if natRouter.isSome: - natRouter.get.natMapper = natMapper - peerInfoObserver = some(setupPeerInfoObserver(switch, autonatService.get, discovery, natMapper.get)) @@ -529,7 +507,7 @@ proc new*( restServer = RestServerRef .new( storageNode.initRestApi( - config, repoStore, autonatService, autoRelayService, natMapper, natRouter, + config, repoStore, autonatService, autoRelayService, natMapper, config.apiCorsAllowedOrigin, ), initTAddress(config.apiBindAddress.get(), config.apiPort), diff --git a/tests/integration/1_minute/testnat.nim b/tests/integration/1_minute/testnat.nim deleted file mode 100644 index f7981030..00000000 --- a/tests/integration/1_minute/testnat.nim +++ /dev/null @@ -1,132 +0,0 @@ -import std/options -import pkg/chronos -import pkg/questionable/results - -import ../multinodes -import ../storageclient -import ../storageconfig -import ../nathelper - -export nathelper - -const DetectionTimeout = 15_000 - -# Reminder: multinodesuite setup the first node as bootstrap node -multinodesuite "AutoNAT detection": - let natConfig = NodeConfigs( - clients: StorageConfigs - .init(nodes = 2) - .withRelay(0) - .withNatNumPeersToAsk(1) - .withNatMinConfidence(0.5) - .withNatScheduleInterval(NatScheduleInterval) - .withNatMaxQueueSize(1).some - ) - test "node is reachable when using bootstrap node on same network", natConfig: - let node2 = clients()[1] - await node2.client.checkReachable() - - let endpointIndependentConfig = NodeConfigs( - clients: StorageConfigs - .init(nodes = 2) - .withRelay(0) - .withNatSimulation(idx = 1, "endpoint-independent") - .withNatNumPeersToAsk(1) - .withNatMinConfidence(0.5) - .withNatScheduleInterval(NatScheduleInterval) - .withNatMaxQueueSize(1).some - ) - # EIF = Endpoint Independent Filtering - test "node with simulated EIF nat is detected as reachable", endpointIndependentConfig: - let node2 = clients()[1] - await node2.client.checkReachable() - - let autonatConfig = NodeConfigs( - clients: StorageConfigs - .init(nodes = 2) - .withRelay(0) - .withNatSimulation(idx = 1, "address-and-port-dependent") - .withNatNumPeersToAsk(1) - .withNatMinConfidence(0.5) - .withNatScheduleInterval(NatScheduleInterval) - .withNatMaxQueueSize(1).some - ) - # APDF = Address and Port-Dependent Filtering - test "node with simulated APDF nat is detected as not reachable and starts relay", - autonatConfig: - let node2 = clients()[1] - await node2.client.checkNotReachable() - - let transitionConfig = NodeConfigs( - clients: StorageConfigs - .init(nodes = 2) - .withRelay(0) - .withNatSimulation(idx = 1, "address-and-port-dependent") - .withNatNumPeersToAsk(1) - .withNatMinConfidence(0.5) - .withNatScheduleInterval(NatScheduleInterval) - .withNatMaxQueueSize(1).some - ) - # APDF = Address and Port-Dependent Filtering - # EIF = Endpoint Independent Filtering - test "node with simulated APDF nat recovers to reachable and stops relay when nat switches to EIF nat", - transitionConfig: - let node2 = clients()[1] - - await node2.client.checkNotReachable() - check (await node2.client.setNatFiltering("endpoint-independent")).isOk - await node2.client.checkReachable() - - let natToSimConfig = NodeConfigs( - clients: StorageConfigs - .init(nodes = 2) - .withRelay(0) - .withNatSimulation(idx = 1, "endpoint-independent") - .withNatNumPeersToAsk(1) - .withNatMinConfidence(0.5) - .withNatScheduleInterval(NatScheduleInterval) - .withNatMaxQueueSize(1).some - ) - # APDF = Address and Port-Dependent Filtering - test "reachable node becomes not reachable and starts relay when nat switches to APDF nat", - natToSimConfig: - let node2 = clients()[1] - - await node2.client.checkReachable() - check (await node2.client.setNatFiltering("address-and-port-dependent")).isOk - await node2.client.checkNotReachable() - - let doubleNatConfig = NodeConfigs( - clients: StorageConfigs - .init(nodes = 2) - .withRelay(0) - .withNatSimulation(idx = 1, "double-nat") - .withNatNumPeersToAsk(1) - .withNatMinConfidence(0.5) - .withNatScheduleInterval(NatScheduleInterval) - .withNatMaxQueueSize(1).some - ) - test "node behind double NAT is detected as not reachable and starts relay", - doubleNatConfig: - let node2 = clients()[1] - await node2.client.checkNotReachable() - - let multiNatConfig = NodeConfigs( - clients: StorageConfigs - .init(nodes = 3) - .withRelay(0) - .withNatSimulation(idx = 1, "address-and-port-dependent") - .withNatSimulation(idx = 2, "address-and-port-dependent") - .withNatNumPeersToAsk(1) - .withNatMinConfidence(0.5) - .withNatScheduleInterval(NatScheduleInterval) - .withNatMaxQueueSize(1).some - ) - # APDF = Address and Port-Dependent Filtering - test "two nodes with simulated APDF nat starts relay through the same relay node", - multiNatConfig: - let node2 = clients()[1] - let node3 = clients()[2] - - await node2.client.checkNotReachable() - await node3.client.checkNotReachable() diff --git a/tests/integration/5_minutes/testnatdownload.nim b/tests/integration/5_minutes/testnatdownload.nim deleted file mode 100644 index 89299760..00000000 --- a/tests/integration/5_minutes/testnatdownload.nim +++ /dev/null @@ -1,74 +0,0 @@ -import std/[json, sequtils] -import pkg/chronos -import pkg/questionable/results - -import ../multinodes -import ../storageclient -import ../storageconfig -import ../nathelper - -const - RelayTimeout = 30_000 - PollInterval = 1_000 - -multinodesuite "NAT download": - let natDownloadConfig = NodeConfigs( - clients: StorageConfigs - .init(nodes = 3) - .withRelay(idx = 0) - .withNatSimulation(idx = 2, "address-and-port-dependent") - .withNatNumPeersToAsk(1) - .withNatMinConfidence(0.5) - .withNatScheduleInterval(NatScheduleInterval) - .withNatMaxQueueSize(1).some - ) - # APDF = Address and Port-Dependent Filtering - test "node 3 with simulated APDF downloads content from reachable seed node 2", - natDownloadConfig: - let seed = clients()[1] - let natNode = clients()[2] - - let content = "content for nat download test" - let cid = (await seed.client.upload(content)).get - - check eventuallySafe( - (await natNode.client.download(cid)).isOk, - timeout = RelayTimeout, - pollInterval = PollInterval, - ) - - check (await natNode.client.download(cid)).get == content - - # APDF = Address and Port-Dependent Filtering - test "reachable node 2 downloads content from node 3 with simulated APDF via relay", - natDownloadConfig: - let seed = clients()[1] - let natNode = clients()[2] - - check eventuallySafe( - (await natNode.client.natRelayRunning()).get(), - timeout = RelayTimeout, - pollInterval = PollInterval, - ) - - # relayRunning only means the service started: the reservation itself - # takes a few more seconds, so we have to poll. - proc advertisesCircuitAddr(): Future[bool] {.async.} = - let info = (await natNode.client.info()).get - let addrs = info["addrs"].getElems.mapIt(it.getStr) - return addrs.anyIt("p2p-circuit" in it) - - check eventuallySafe( - await advertisesCircuitAddr(), timeout = RelayTimeout, pollInterval = PollInterval - ) - - let content = "content seeded from nat node" - let cid = (await natNode.client.upload(content)).get - - check eventuallySafe( - (await seed.client.download(cid)).isOk, - timeout = RelayTimeout, - pollInterval = PollInterval, - ) - - check (await seed.client.download(cid)).get == content diff --git a/tests/integration/5_minutes/testrestapivalidation.nim b/tests/integration/5_minutes/testrestapivalidation.nim index ab0a1b99..20a3ad40 100644 --- a/tests/integration/5_minutes/testrestapivalidation.nim +++ b/tests/integration/5_minutes/testrestapivalidation.nim @@ -44,25 +44,3 @@ multinodesuite "Rest API validation": check: response.status == 400 (await response.body) == "Incorrect Cid" - - test "nat/filtering returns 400 when nat simulation not active", config: - let response = await client.post( - client.buildUrl("/debug/nat/filtering?filtering=endpoint-independent") - ) - check response.status == 400 - - let natSimConfig = NodeConfigs( - clients: StorageConfigs - .init(nodes = 1) - .withNatSimulation(idx = 0, "address-and-port-dependent").some - ) - - test "nat/filtering returns 400 for invalid filtering value", natSimConfig: - let response = await client.post( - client.buildUrl("/debug/nat/filtering?filtering=not-a-valid-value") - ) - check response.status == 400 - - test "nat/filtering returns 400 when filtering param is missing", natSimConfig: - let response = await client.post(client.buildUrl("/debug/nat/filtering")) - check response.status == 400 diff --git a/tests/integration/storageclient.nim b/tests/integration/storageclient.nim index dc852533..50b03376 100644 --- a/tests/integration/storageclient.nim +++ b/tests/integration/storageclient.nim @@ -283,12 +283,3 @@ proc natPortMapping*( return info.get()["nat"]["portMapping"].getStr().success except KeyError as e: return failure e.msg - -proc setNatFiltering*( - client: StorageClient, filtering: string -): Future[?!void] {.async: (raises: [CancelledError, HttpError]).} = - let response = - await client.post(client.baseurl & "/debug/nat/filtering?filtering=" & filtering) - if response.status != 200: - return failure "Failed to set NAT filtering: " & $response.status - return success() diff --git a/tests/integration/storageconfig.nim b/tests/integration/storageconfig.nim index 32d5b536..2f64ef79 100644 --- a/tests/integration/storageconfig.nim +++ b/tests/integration/storageconfig.nim @@ -346,15 +346,6 @@ proc isBootstrapNode*(config: StorageConfig): bool {.raises: [].} = return false -proc withNatSimulation*( - self: StorageConfigs, idx: int, filtering: string -): StorageConfigs {.raises: [StorageConfigError].} = - self.checkBounds idx - - var startConfig = self - startConfig.configs[idx].addCliOption("--nat-simulation", filtering) - return startConfig - proc withAutonatServer*( self: StorageConfigs, idx: int ): StorageConfigs {.raises: [StorageConfigError].} = diff --git a/tests/nat/testnatpcp.nim b/tests/nat/testnatpcp.nim deleted file mode 100644 index 25f5902b..00000000 --- a/tests/nat/testnatpcp.nim +++ /dev/null @@ -1,93 +0,0 @@ -import std/[json, strutils, sequtils] -import pkg/chronos -import pkg/questionable/results - -import ../integration/multinodes -import ../integration/storageclient -import ../integration/storageconfig - -import ../integration/nathelper - -multinodesuite "AutoNAT PCP port mapping": - let pcpConfig = NodeConfigs( - clients: StorageConfigs - .init(nodes = 2) - .withRelay(0) - .withNatSimulation(idx = 1, "address-and-port-dependent") - .withNatNumPeersToAsk(1) - .withNatMinConfidence(0.5) - .withNatScheduleInterval(NatScheduleInterval) - .withNatMaxQueueSize(1).some - ) - - test "node behind NAT maps ports via PCP and exposes mapping in debug info", pcpConfig: - let node2 = clients()[1] - - await node2.client.checkNotReachable(relayRunning = false) - - check eventuallySafe( - block: - let res = await node2.client.natPortMapping() - res.isOk and res.get == "pcp", - timeout = RelayTimeout, - pollInterval = PollInterval, - ) - - await node2.client.checkReachable() - - await node2.stop() - - let relayFallbackConfig = NodeConfigs( - clients: StorageConfigs - .init(nodes = 2) - .withRelay(0) - .withNatSimulation(idx = 1, "double-nat") - .withNatNumPeersToAsk(1) - .withNatMinConfidence(0.5) - .withNatScheduleInterval(NatScheduleInterval) - # Increase the max queue to trigger the AutoNat 2 times - .withNatMaxQueueSize(2).some - ) - - test "node behind double NAT falls back to relay after PCP mapping does not help", - relayFallbackConfig: - let node2 = clients()[1] - - await node2.client.checkNotReachable(relayRunning = false) - - check eventuallySafe( - block: - let res = await node2.client.natPortMapping() - res.isOk and res.get == "pcp", - timeout = RelayTimeout, - pollInterval = PollInterval, - ) - - # Wait for next Autonat iteration - await sleepAsync(6.seconds) - - await node2.client.checkNotReachable() - - test "reachable node downloads content uploaded by node behind NAT after PCP mapping", - pcpConfig: - let node1 = clients()[0] - let node2 = clients()[1] - - check eventuallySafe( - block: - let res = await node2.client.natPortMapping() - res.isOk and res.get == "pcp", - timeout = RelayTimeout, - pollInterval = PollInterval, - ) - - let content = "content uploaded by nat node" - let cid = (await node2.client.upload(content)).get - - check eventuallySafe( - (await node1.client.download(cid)).isOk, - timeout = RelayTimeout, - pollInterval = PollInterval, - ) - - check (await node1.client.download(cid)).get == content diff --git a/tests/nat/testnatupnp.nim b/tests/nat/testnatupnp.nim deleted file mode 100644 index 9f7c29bf..00000000 --- a/tests/nat/testnatupnp.nim +++ /dev/null @@ -1,94 +0,0 @@ -import std/[json, strutils, sequtils] -import pkg/chronos -import pkg/questionable/results - -import ../integration/multinodes -import ../integration/storageclient -import ../integration/storageconfig - -import ../integration/nathelper - -multinodesuite "AutoNAT UPnP port mapping": - let upnpConfig = NodeConfigs( - clients: StorageConfigs - .init(nodes = 2) - .withRelay(0) - .withNatSimulation(idx = 1, "address-and-port-dependent") - .withNatNumPeersToAsk(1) - .withNatMinConfidence(0.5) - .withNatScheduleInterval(NatScheduleInterval) - .withNatMaxQueueSize(1).some - ) - - test "node behind NAT maps ports via UPnP and exposes mapping in debug info", - upnpConfig: - let node2 = clients()[1] - - await node2.client.checkNotReachable(relayRunning = false) - - check eventuallySafe( - block: - let res = await node2.client.natPortMapping() - res.isOk and res.get == "upnp", - timeout = RelayTimeout, - pollInterval = PollInterval, - ) - - await node2.client.checkReachable() - - await node2.stop() - - let relayFallbackConfig = NodeConfigs( - clients: StorageConfigs - .init(nodes = 2) - .withRelay(0) - .withNatSimulation(idx = 1, "double-nat") - .withNatNumPeersToAsk(1) - .withNatMinConfidence(0.5) - .withNatScheduleInterval(NatScheduleInterval) - # Increase the max queue to trigger the AutoNat 2 times - .withNatMaxQueueSize(2).some - ) - - test "node behind double NAT falls back to relay after UPnP mapping does not help", - relayFallbackConfig: - let node2 = clients()[1] - - await node2.client.checkNotReachable(relayRunning = false) - - check eventuallySafe( - block: - let res = await node2.client.natPortMapping() - res.isOk and res.get == "upnp", - timeout = RelayTimeout, - pollInterval = PollInterval, - ) - - # Wait for next Autonat iteration - await sleepAsync(6.seconds) - - await node2.client.checkNotReachable() - - test "reachable node downloads content uploaded by node behind NAT after UPnP mapping", - upnpConfig: - let node1 = clients()[0] - let node2 = clients()[1] - - check eventuallySafe( - block: - let res = await node2.client.natPortMapping() - res.isOk and res.get == "upnp", - timeout = RelayTimeout, - pollInterval = PollInterval, - ) - - let content = "content uploaded by nat node" - let cid = (await node2.client.upload(content)).get - - check eventuallySafe( - (await node1.client.download(cid)).isOk, - timeout = RelayTimeout, - pollInterval = PollInterval, - ) - - check (await node1.client.download(cid)).get == content diff --git a/storage/utils/natsimulation.nim b/tests/storage/natsimulation.nim similarity index 92% rename from storage/utils/natsimulation.nim rename to tests/storage/natsimulation.nim index 514684e7..2ae0b77d 100644 --- a/storage/utils/natsimulation.nim +++ b/tests/storage/natsimulation.nim @@ -1,11 +1,8 @@ # NAT simulation for integration testing. # -# Testing NAT traversal in CI requires controlling inbound/outbound filtering -# rules, which is not possible with real network interfaces. This module wraps -# the TCP transport to enforce configurable filtering behaviors (endpoint- -# independent, address-dependent, address-and-port-dependent, double NAT) at -# the connection level, so the full AutoNAT detection and relay -# stack can be exercised without actual NAT hardware. +# It simulates the filtering behaviors (endpoint-independent, address-dependent, +# address-and-port-dependent, double NAT) at the connection level, so the full +# AutoNAT detection and relay stack can be exercised without actual NAT hardware. {.push raises: [].} @@ -18,7 +15,7 @@ import pkg/libp2p/transports/tcptransport import pkg/libp2p/transports/transport import pkg/libp2p/wire -import ../nat +import ../../storage/nat logScope: topics = "nat simulation" diff --git a/tests/storage/testnatdetection.nim b/tests/storage/testnatdetection.nim new file mode 100644 index 00000000..3cc01f34 --- /dev/null +++ b/tests/storage/testnatdetection.nim @@ -0,0 +1,241 @@ +## NAT detection unit tests: real AutoNAT v2 detecting +## through the NAT simulation, feeding storage's handleNatStatus, which drives +## the relay and client mode. +## +## The MockNatPortMapper simulates a failing port mapping. This is not the aspect +## of the NAT detection that is being tested: we want to test the NAT detection +## logic itself, not the port mapping logic. + +import std/options +import pkg/chronos +import pkg/libp2p except setup +import pkg/libp2p/protocols/connectivity/autonatv2/service except setup +import pkg/libp2p/protocols/connectivity/autonatv2/client except setup +import pkg/libp2p/protocols/connectivity/autonatv2/types as autonatv2Types +import pkg/libp2p/protocols/connectivity/relay/client as relayClientModule +import pkg/libp2p/services/autorelayservice except setup +import pkg/libp2p/observedaddrmanager + +import ./helpers +import ./natsimulation +import ../asynctest +import ../../storage/utils/natutils +import ../../storage/nat +import ../../storage/discovery +import ../../storage/rng + +const + flags = {ServerFlags.ReuseAddr} + listenAddr = "/ip4/127.0.0.1/tcp/0" + discoveryPort = Port(8090) + # ms — AutoNAT probe + confidence + reaction + detectTimeout = 20000 + +type MockNatPortMapper = ref object of NatPortMapper + +method mapNatPorts*( + m: MockNatPortMapper +): Future[Option[(Port, Port, MappingProtocol)]] {. + async: (raises: [CancelledError]), gcsafe +.} = + none((Port, Port, MappingProtocol)) + +# Captures the candidate addresses the service sends and answers Reachable, so +# the service flips to reachable and runs its address mapper — without dialing. +type MockAutonatV2Client = ref object of AutonatV2Client + reqAddrs: seq[MultiAddress] + +method sendDialRequest*( + self: MockAutonatV2Client, pid: PeerId, testAddrs: seq[MultiAddress] +): Future[AutonatV2Response] {. + async: (raises: [AutonatV2Error, CancelledError, DialFailedError, LPStreamError]) +.} = + self.reqAddrs = testAddrs + AutonatV2Response(reachability: Reachable) + +proc serverSwitch(): Switch = + SwitchBuilder + .new() + .withRng(Rng.instance()) + .withPrivateKey(PrivateKey.random(Rng.instance()).get()) + .withAddresses(@[MultiAddress.init(listenAddr).get()]) + .withTcpTransport(flags) + .withNoise() + .withYamux() + .withAutonatV2Server() + .build() + +asyncchecksuite "NAT detection - simulated NAT": + var + natNode: Switch + autonat: AutonatV2Service + relay: AutoRelayService + disc: Discovery + server: Switch + + proc setupTopology(router: NatRouter) {.async.} = + let relayClient = relayClientModule.RelayClient.new() + natNode = SwitchBuilder + .new() + .withRng(Rng.instance()) + .withPrivateKey(PrivateKey.random(Rng.instance()).get()) + .withAddresses(@[MultiAddress.init(listenAddr).get()]) + .withNatTransport(router, flags) + .withNoise() + .withYamux() + .withCircuitRelay(relayClient) + .build() + + relay = AutoRelayService.new(1, relayClient, nil, Rng.instance()) + autorelayservice.setup(relay, natNode) + disc = Discovery.new(PrivateKey.random(Rng.instance()).get(), announceAddrs = @[]) + # nodes start in client mode until Reachable + disc.protocol.clientMode = true + + # Setup real AutoNAT v2 client using nat simulation + let autonatClient = AutonatV2Client.new(natNode.rng) + client.setup(autonatClient, natNode) + natNode.mount(autonatClient) + + # Setup AutoNAT v2 service with maxQueueSize=1 and minConfidence=0.5, + # so a single dial-back answer (confidence 1.0) is needed. + let config = AutonatV2ServiceConfig.new( + scheduleInterval = Opt.some(1.seconds), + askNewConnectedPeers = true, + numPeersToAsk = 1, + maxQueueSize = 1, + minConfidence = 0.5, + ) + autonat = AutonatV2Service.new(natNode.rng, autonatClient, config) + service.setup(autonat, natNode) + + autonat.setStatusAndConfidenceHandler( + proc( + reachability: NetworkReachability, + confidence: Opt[float], + addrs: Opt[MultiAddress], + ) {.async: (raises: [CancelledError]).} = + # One call to our handleNatStatus handler + await MockNatPortMapper().handleNatStatus( + reachability, addrs, discoveryPort, disc, natNode, relay + ) + ) + + # Create and start one Autonat server (maxQueueSize=1 and minConfidence=0.5) + server = serverSwitch() + await server.start() + + # Start the NAT node and connect to the Autonat server (bootstrap node in our network). + # Then start the Autonat service on the NAT node. + await natNode.start() + await natNode.connect(server.peerInfo.peerId, server.peerInfo.addrs) + await autonat.start(natNode) + + teardown: + await autonat.stop(natNode) + + if relay.isRunning: + await relay.stop(natNode) + + await natNode.stop() + await server.stop() + + test "node behind EIF nat ends up reachable: no relay, not in client mode": + await setupTopology(NatRouter.new(EndpointIndependent)) + check eventually( + not relay.isRunning and not disc.protocol.clientMode, timeout = detectTimeout + ) + + test "node behind APDF nat ends up not reachable: relay running, client mode": + await setupTopology(NatRouter.new(AddressAndPortDependent)) + check eventually( + relay.isRunning and disc.protocol.clientMode, timeout = detectTimeout + ) + + test "node behind double NAT ends up not reachable: relay running, client mode": + await setupTopology(NatRouter.new(DoubleNat)) + check eventually( + relay.isRunning and disc.protocol.clientMode, timeout = detectTimeout + ) + + test "node recovers (relay stops) when nat switches from APDF to EIF": + let router = NatRouter.new(AddressAndPortDependent) + await setupTopology(router) + check eventually( + relay.isRunning and disc.protocol.clientMode, timeout = detectTimeout + ) + + router.setFiltering(EndpointIndependent) + check eventually( + not relay.isRunning and not disc.protocol.clientMode, timeout = detectTimeout + ) + + test "node degrades (relay starts) when nat switches from EIF to APDF": + let router = NatRouter.new(EndpointIndependent) + await setupTopology(router) + check eventually( + not relay.isRunning and not disc.protocol.clientMode, timeout = detectTimeout + ) + + router.setFiltering(AddressAndPortDependent) + check eventually( + relay.isRunning and disc.protocol.clientMode, timeout = detectTimeout + ) + +asyncchecksuite "NAT detection - dial request candidates": + # This detects is useful to detect behaviour changes in libp2p + # that may break our autonat dial request candidate handling. + # + # By default, Autonat V2 dials the addresses passed to peerInfo.addrs and + # uses the first attempt to dial as the primary candidate. + # + # With enableDialableCandidates, Autonat V2 also dials the guessDialableAddress + # as first candidate and use the most observed address as the fallback. + var sw: Switch + + setup: + sw = newStandardSwitch() + await sw.start() + + teardown: + await sw.stop() + + test "autonat handles the observed dialable address": + let mockClient = MockAutonatV2Client() + let autonat = AutonatV2Service.new( + Rng.instance(), + mockClient, + AutonatV2ServiceConfig.new( + enableDialableCandidates = true, maxQueueSize = 1, minConfidence = 0.5 + ), + ) + service.setup(autonat, sw) + await autonat.start(sw) # registers the address mapper on peerInfo + + # observations before the manager trusts an addr in libp2p; the observed + # port (4001) differs from our listen port on purpose (see dialable below) + let quorum = 3 + let observed = MultiAddress.init("/ip4/8.8.8.8/tcp/4001").expect("valid") + for _ in 0 ..< quorum: + discard sw.peerStore.identify.observedAddrManager.addObservation(observed) + + let sw2 = newStandardSwitch() + await sw2.start() + await sw.connect(sw2.peerInfo.peerId, sw2.peerInfo.addrs) + + # The dialable candidate keeps our real listen port and swaps in the + # observed IP. It must be reached via AutoNAT and peerInfo, so it can only + # come from guessDialableAddr. + let tcpPart = sw.peerInfo.listenAddrs[0][1].expect("valid") + let dialable = + concat(MultiAddress.init("/ip4/8.8.8.8").expect("valid"), tcpPart).expect("valid") + + # Phase 1: it is submitted as a dial candidate, not 127.0.0.1 from + # newStandardSwitch. + check eventually(dialable in mockClient.reqAddrs) + + # Phase 2: the address mapper promotes it into peerInfo. + check eventually(dialable in sw.peerInfo.addrs) + + await autonat.stop(sw) + await sw2.stop() diff --git a/tests/storage/testnat.nim b/tests/storage/testnatreaction.nim similarity index 69% rename from tests/storage/testnat.nim rename to tests/storage/testnatreaction.nim index 4f2c81df..1f0d17ef 100644 --- a/tests/storage/testnat.nim +++ b/tests/storage/testnatreaction.nim @@ -3,11 +3,7 @@ import pkg/chronos import pkg/libp2p/[multiaddress, multihash, multicodec] import pkg/libp2p/protocols/connectivity/autonat/types import pkg/libp2p/protocols/connectivity/autonatv2/service except setup -import pkg/libp2p/protocols/connectivity/autonatv2/client except setup -import pkg/libp2p/protocols/connectivity/autonatv2/types as autonatv2Types import pkg/libp2p/protocols/connectivity/relay/client as relayClientModule -import pkg/libp2p/protocols/connectivity/dcutr/core as dcutrCore -import pkg/libp2p/multistream import pkg/libp2p/services/autorelayservice except setup import pkg/results @@ -21,6 +17,7 @@ import ../../storage/utils type MockNatPortMapper = ref object of NatPortMapper mappedPorts: Option[(Port, Port, MappingProtocol)] + activeMapping: bool method mapNatPorts*( m: MockNatPortMapper @@ -29,18 +26,10 @@ method mapNatPorts*( .} = m.mappedPorts -type MockAutonatV2Client = ref object of AutonatV2Client - reqAddrs: seq[MultiAddress] +method hasActiveMapping*(m: MockNatPortMapper): bool = + m.activeMapping -method sendDialRequest*( - self: MockAutonatV2Client, pid: PeerId, testAddrs: seq[MultiAddress] -): Future[AutonatV2Response] {. - async: (raises: [AutonatV2Error, CancelledError, DialFailedError, LPStreamError]) -.} = - self.reqAddrs = testAddrs - AutonatV2Response(reachability: Unknown) - -asyncchecksuite "NAT - handleNatStatus": +asyncchecksuite "NAT reaction - port mapping": var sw: Switch var key: PrivateKey var disc: Discovery @@ -78,7 +67,7 @@ asyncchecksuite "NAT - handleNatStatus": check not autoRelay.isRunning check disc.protocol.clientMode - test "handleNatStatus starts autoRelay when NotReachable and no dialBackAddr": + test "handleNatStatus starts autoRelay when NotReachable and no dialBackAddr but no mapped ports": let mapper = MockNatPortMapper(mappedPorts: none((Port, Port, MappingProtocol))) autorelayservice.setup(autoRelay, sw) @@ -102,7 +91,32 @@ asyncchecksuite "NAT - handleNatStatus": check disc.announceAddrs == newSeq[MultiAddress]() check disc.protocol.clientMode - test "handleNatStatus stops relay and exits client mode when Reachable": + test "handleNatStatus tears down an active mapping and starts relay when NotReachable with dialBackAddr": + let dialBack = MultiAddress.init("/ip4/1.2.3.4/tcp/8080").expect("valid") + let mapper = MockNatPortMapper(activeMapping: true) + + autorelayservice.setup(autoRelay, sw) + await mapper.handleNatStatus( + NotReachable, Opt.some(dialBack), discoveryPort, disc, sw, autoRelay + ) + + check autoRelay.isRunning + check disc.announceAddrs == newSeq[MultiAddress]() + check disc.protocol.clientMode + + test "handleNatStatus tears down an active mapping and starts relay when NotReachable without dialBackAddr": + let mapper = MockNatPortMapper(activeMapping: true) + + autorelayservice.setup(autoRelay, sw) + await mapper.handleNatStatus( + NotReachable, Opt.none(MultiAddress), discoveryPort, disc, sw, autoRelay + ) + + check autoRelay.isRunning + check disc.announceAddrs == newSeq[MultiAddress]() + check disc.protocol.clientMode + + test "handleNatStatus stops relay and exits client mode when mapping is created and node is Reachable": let mapper = MockNatPortMapper(mappedPorts: none((Port, Port, MappingProtocol))) disc.protocol.clientMode = true @@ -129,6 +143,22 @@ asyncchecksuite "NAT - handleNatStatus": check not autoRelay.isRunning check disc.announceAddrs == newSeq[MultiAddress]() +asyncchecksuite "NAT reaction - address announcing": + var sw: Switch + var key: PrivateKey + var disc: Discovery + + setup: + key = PrivateKey.random(Rng.instance()).get() + disc = Discovery.new(key, announceAddrs = @[]) + sw = newStandardSwitch() + await sw.start() + + teardown: + await sw.stop() + + let discoveryPort = Port(8090) + test "announcePeerInfoAddrs excludes relay circuit addresses": let circuitAddr = MultiAddress .init("/ip4/1.2.3.4/tcp/4040/p2p/" & $sw.peerInfo.peerId & "/p2p-circuit") @@ -192,56 +222,3 @@ asyncchecksuite "NAT - handleNatStatus": await sw.peerInfo.update() check disc.announceAddrs == newSeq[MultiAddress]() - - test "autonat dial request includes the observed addresses as candidates": - # The dial request includes the addresses observed by other peers, so a NATed node submits - # a dialable candidate even though its listen addrs are private. - let client = MockAutonatV2Client() - let autonat = AutonatV2Service.new( - Rng.instance(), - client, - AutonatV2ServiceConfig.new(enableDialableCandidates = true), - ) - service.setup(autonat, sw) - await autonat.start(sw) - - let observed = MultiAddress.init("/ip4/8.8.8.8/tcp/4001").expect("valid") - for _ in 0 ..< 3: # minCount: 3 observations before the manager trusts an addr - discard sw.peerStore.identify.observedAddrManager.addObservation(observed) - - let sw2 = newStandardSwitch() - await sw2.start() - await sw.connect(sw2.peerInfo.peerId, sw2.peerInfo.addrs) - - check eventually(observed in client.reqAddrs) - - await autonat.stop(sw) - await sw2.stop() - -asyncchecksuite "NAT - Hole punching": - test "setupHolePunching mounts the dcutr protocol on the switch": - let sw = newStandardSwitch() - discard setupHolePunching(sw) - check sw.ms.handlers.anyIt(dcutrCore.DcutrCodec in it.protos) - - test "holePunchIfRelayed returns early when the peer has no connections": - let sw1 = newStandardSwitch() - let sw2 = newStandardSwitch() - await allFutures(sw1.start(), sw2.start()) - - await holePunchIfRelayed(sw1, sw2.peerInfo.peerId) - - await allFutures(sw1.stop(), sw2.stop()) - - test "holePunchIfRelayed returns early when a direct connection already exists": - let sw1 = newStandardSwitch() - let sw2 = newStandardSwitch() - await allFutures(sw1.start(), sw2.start()) - - await sw1.connect(sw2.peerInfo.peerId, sw2.peerInfo.addrs) - check sw1.isConnected(sw2.peerInfo.peerId) - - await holePunchIfRelayed(sw1, sw2.peerInfo.peerId) - - check sw1.isConnected(sw2.peerInfo.peerId) - await allFutures(sw1.stop(), sw2.stop()) diff --git a/tests/storage/testnatsimulation.nim b/tests/storage/testnatsimulation.nim index f2079c8a..00f6652d 100644 --- a/tests/storage/testnatsimulation.nim +++ b/tests/storage/testnatsimulation.nim @@ -6,7 +6,7 @@ import ./helpers import ../asynctest import ../../storage/rng import ../../storage/nat -import ../../storage/utils/natsimulation +import ./natsimulation const flags = {ServerFlags.ReuseAddr} const listenAddr = "/ip4/127.0.0.1/tcp/0" @@ -44,7 +44,7 @@ proc newNatSwitch(router: NatRouter, rng: Rng): Switch = .withYamux() .build() -asyncchecksuite "NatTransport - Endpoint-Independent Filtering": +asyncchecksuite "Nat transport - Endpoint-Independent Filtering": var bootstrap, natNode: Switch setup: @@ -62,7 +62,7 @@ asyncchecksuite "NatTransport - Endpoint-Independent Filtering": await bootstrap.connect(natNode.peerInfo.peerId, natNode.peerInfo.addrs) check bootstrap.isConnected(natNode.peerInfo.peerId) -asyncchecksuite "NatTransport - Address-Dependent Filtering": +asyncchecksuite "Nat transport - Address-Dependent Filtering": var bootstrap, thirdNode, natNode: Switch setup: @@ -94,7 +94,7 @@ asyncchecksuite "NatTransport - Address-Dependent Filtering": test "bootstrap cannot connect to nat node without a pre-existing connection": check await cannotConnect(bootstrap, natNode) -asyncchecksuite "NatTransport - Address-and-Port-Dependent Filtering": +asyncchecksuite "Nat transport - Address-and-Port-Dependent Filtering": var bootstrap, thirdNode, natNode: Switch setup: @@ -125,7 +125,7 @@ asyncchecksuite "NatTransport - Address-and-Port-Dependent Filtering": await natNode.connect(bootstrap.peerInfo.peerId, bootstrap.peerInfo.addrs) check await cannotConnect(thirdNode, natNode) -asyncchecksuite "NatTransport - Double NAT": +asyncchecksuite "Nat transport - Double NAT": var bootstrap, natNode: Switch var router: NatRouter @@ -148,7 +148,7 @@ asyncchecksuite "NatTransport - Double NAT": check await cannotConnect(bootstrap, natNode) -asyncchecksuite "NatTransport - Port Mapping": +asyncchecksuite "Nat transport - Port Mapping": var bootstrap, natNode: Switch var router: NatRouter diff --git a/vendor/nim-boringssl b/vendor/nim-boringssl index f8111056..e77caaba 160000 --- a/vendor/nim-boringssl +++ b/vendor/nim-boringssl @@ -1 +1 @@ -Subproject commit f8111056182cf6abd9e35de77a919e873ef94652 +Subproject commit e77caabae78fbc9aa5b78a0a521181b077c82571 diff --git a/vendor/nim-lsquic b/vendor/nim-lsquic index 00e4b7df..2f01046b 160000 --- a/vendor/nim-lsquic +++ b/vendor/nim-lsquic @@ -1 +1 @@ -Subproject commit 00e4b7dfaa197cd120267aa897b33b0914166b45 +Subproject commit 2f01046bf1d513de8b5f8296c3d8bec819ab0cb9 diff --git a/vendor/nim-protobuf-serialization b/vendor/nim-protobuf-serialization index f45476a3..d9aa950b 160000 --- a/vendor/nim-protobuf-serialization +++ b/vendor/nim-protobuf-serialization @@ -1 +1 @@ -Subproject commit f45476a3c1f4e7bff73845e6450d686be040ddeb +Subproject commit d9aa950b9d9e8bfc8a201740042b5e8ea5880875 From 20e820f07baec3a47824e0f4b205230db76ace08 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Fri, 12 Jun 2026 15:05:18 +0400 Subject: [PATCH 123/167] Cleanup --- build.nims | 14 ------- storage/nat.nim | 8 ++-- storage/storage.nim | 4 ++ tests/integration/nathelper.nim | 65 ------------------------------- tests/storage/testaddrutils.nim | 21 ++++------ tests/storage/testnatreaction.nim | 2 +- 6 files changed, 17 insertions(+), 97 deletions(-) delete mode 100644 tests/integration/nathelper.nim diff --git a/build.nims b/build.nims index 1eb7acbb..f1831020 100644 --- a/build.nims +++ b/build.nims @@ -78,20 +78,6 @@ task testIntegration, "Run integration tests": # test "testIntegration", params = "-d:chronicles_sinks=textlines[notimestamps,stdout],textlines[dynamic] " & # "-d:chronicles_enabled_topics:integration:TRACE" -task testNatPortMapping, "Run UPnP NAT integration test (requires miniupnpd container)": - buildBinary "storage", - outName = "storage", - params = "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE" - putEnv("STORAGE_INTEGRATION_TEST_INCLUDES", "nat/testnatupnp.nim") - test "testIntegration", outName = "testIntegrationNat" - -task testNatPcpMapping, "Run PCP NAT integration test (requires miniupnpd container)": - buildBinary "storage", - outName = "storage", - params = "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE" - putEnv("STORAGE_INTEGRATION_TEST_INCLUDES", "nat/testnatpcp.nim") - test "testIntegration", outName = "testIntegrationNatPcp" - task testNatNotReachable, "Run NAT not-reachable scenario (needs the image + podman-compose)": test "integration/nat/not-reachable/testnotreachable", outName = "testNatNotReachable" diff --git a/storage/nat.nim b/storage/nat.nim index cbda208e..da65e8b5 100644 --- a/storage/nat.nim +++ b/storage/nat.nim @@ -128,7 +128,9 @@ proc stop*(m: NatPortMapper) = proc isPortMapped*(m: NatPortMapper, port: Port): bool = m.activeTcpPort.isSome and m.activeTcpPort.get == port -method hasActiveMapping*(m: NatPortMapper): bool {.base, gcsafe.} = +method hasMappingIds*(m: NatPortMapper): bool {.base, gcsafe.} = + # Only checks that mappings were created, not that they are still live + # (use hasMapping() for liveness check). m.tcpMappingId.isSome and m.udpMappingId.isSome proc announcePeerInfoAddrs*(discovery: Discovery, peerInfo: PeerInfo, udpPort: Port) = @@ -195,11 +197,11 @@ method handleNatStatus*( if dialBackAddr.isNone: warn "Got empty dialback address in AutoNat when node is NotReachable" - if m.hasActiveMapping(): + if m.hasMappingIds(): m.close() discovery.announceDirectAddrs(@[], udpPort = discoveryPort) - elif m.hasActiveMapping(): + elif m.hasMappingIds(): warn "Not Reachable with active port mapping. The port mapping will be deleted and relay will start." # The mapping was created the the node is still not reachable. diff --git a/storage/storage.nim b/storage/storage.nim index a7c1b2c6..dfe0fb12 100644 --- a/storage/storage.nim +++ b/storage/storage.nim @@ -462,6 +462,10 @@ proc new*( # response can also carry loopback/private addresses: # they are never dialable by a remote peer, so drop them. let publicAddrs = addresses.filterIt(it.hasPublicRelayTransport()) + if publicAddrs.len == 0: + warn "Relay reservation has no publicly dialable address, keeping previous announce", + addresses + return info "Relay reservation updated", addresses = publicAddrs # relay addresses are for download traffic only, not DHT routing discovery.announceRelayAddrs(publicAddrs), diff --git a/tests/integration/nathelper.nim b/tests/integration/nathelper.nim deleted file mode 100644 index 8436cd7d..00000000 --- a/tests/integration/nathelper.nim +++ /dev/null @@ -1,65 +0,0 @@ -import std/algorithm -import std/json -import std/sequtils -import pkg/chronos -import pkg/questionable/results - -import ./multinodes -import ./storageclient -import ./storageconfig - -const - RelayTimeout* = 30_000 - PollInterval* = 1_000 - NatScheduleInterval* = 5.seconds - -proc checkNatStatus*( - client: StorageClient, reachability: string, relayRunning: bool, clientMode: bool -) {.async.} = - check eventuallySafe( - block: - let info = (await client.info()).get - let nat = info["nat"] - let addrs = info["addrs"].getElems.mapIt(it.getStr) - let r = nat["reachability"].getStr() - let cm = nat["clientMode"].getBool() - let rr = nat["relayRunning"].getBool() - let ha = addrs.anyIt("p2p-circuit" in it) - let pm = nat["portMapping"].getStr() - let aa = info["announceAddresses"].getElems.mapIt(it.getStr) - - # Reachable nodes must announce their dialable (non-circuit) addrs to - # the DHT (peerInfo observer); relayed nodes must announce their - # circuit addrs (onReservation). - let announceOk = - if reachability == "Reachable": - aa.len > 0 and aa.sorted == addrs.filterIt("p2p-circuit" notin it).sorted - elif relayRunning: - aa.len > 0 and aa.allIt("p2p-circuit" in it) - else: - true - - # It is important to check all the conditions together to avoid race - # (new autonat iteration) - # So we add a checkoint for better debug in case of failures - checkpoint( - "reachability=" & r & " (want " & reachability & ")" & " clientMode=" & $cm & - " (want " & $clientMode & ")" & " relayRunning=" & $rr & " (want " & - $relayRunning & ")" & " p2p-circuit=" & $ha & " (want " & $relayRunning & ")" & - " portMapping=" & pm & " announceAddresses=" & $aa - ) - r == reachability and cm == clientMode and rr == relayRunning and - ha == relayRunning and announceOk, - timeout = RelayTimeout, - pollInterval = PollInterval, - ) - -proc checkReachable*(client: StorageClient) {.async.} = - await client.checkNatStatus("Reachable", relayRunning = false, clientMode = false) - -# Relay might be false when the mapping has been created for UPnP / TCP but -# Autonat didn't detect yet Reachable -proc checkNotReachable*(client: StorageClient, relayRunning = true) {.async.} = - await client.checkNatStatus( - "NotReachable", relayRunning = relayRunning, clientMode = true - ) diff --git a/tests/storage/testaddrutils.nim b/tests/storage/testaddrutils.nim index 94c3db76..d9a4e91e 100644 --- a/tests/storage/testaddrutils.nim +++ b/tests/storage/testaddrutils.nim @@ -3,6 +3,13 @@ import pkg/libp2p/multiaddress import ../asynctest import ../../storage/utils/addrutils +const relayId = "16Uiu2HAkyRvHo1AyyQY1xiHC8QbYjXCHkZbneVC8dBtJjp1SZcGD" + +proc circuitAddr(relayIp: string): MultiAddress = + MultiAddress + .init("/ip4/" & relayIp & "/tcp/8070/p2p/" & relayId & "/p2p-circuit") + .expect("valid") + suite "addrutils - getTcpPort": test "extracts port from ipv4 tcp address": let ma = MultiAddress.init("/ip4/1.2.3.4/tcp/5000").expect("valid") @@ -37,13 +44,6 @@ suite "addrutils - remapAddr": check remapped == MultiAddress.init("/ip4/8.8.8.8/tcp/5000").expect("valid") suite "addrutils - hasPublicRelayTransport": - const relayId = "16Uiu2HAkyRvHo1AyyQY1xiHC8QbYjXCHkZbneVC8dBtJjp1SZcGD" - - proc circuitAddr(relayIp: string): MultiAddress = - MultiAddress - .init("/ip4/" & relayIp & "/tcp/8070/p2p/" & relayId & "/p2p-circuit") - .expect("valid") - test "true when the relay has a public ip": check circuitAddr("204.168.234.45").hasPublicRelayTransport() @@ -54,13 +54,6 @@ suite "addrutils - hasPublicRelayTransport": check not circuitAddr("172.17.0.1").hasPublicRelayTransport() suite "addrutils - dialableAddressPolicy": - const relayId = "16Uiu2HAkyRvHo1AyyQY1xiHC8QbYjXCHkZbneVC8dBtJjp1SZcGD" - - proc circuitAddr(relayIp: string): MultiAddress = - MultiAddress - .init("/ip4/" & relayIp & "/tcp/8070/p2p/" & relayId & "/p2p-circuit") - .expect("valid") - test "keeps a public direct address": check MultiAddress .init("/ip4/204.168.234.45/tcp/8070") diff --git a/tests/storage/testnatreaction.nim b/tests/storage/testnatreaction.nim index 1f0d17ef..258f8606 100644 --- a/tests/storage/testnatreaction.nim +++ b/tests/storage/testnatreaction.nim @@ -26,7 +26,7 @@ method mapNatPorts*( .} = m.mappedPorts -method hasActiveMapping*(m: MockNatPortMapper): bool = +method hasMappingIds*(m: MockNatPortMapper): bool = m.activeMapping asyncchecksuite "NAT reaction - port mapping": From 06bd7842c1d654ee9a4693171bd9dd4d0610bc1c Mon Sep 17 00:00:00 2001 From: Arnaud Date: Fri, 12 Jun 2026 17:14:18 +0400 Subject: [PATCH 124/167] Apply libp2p update --- config.nims | 2 +- storage/blockexchange/network/network.nim | 9 +++++- storage/logutils.nim | 4 ++- storage/storage.nim | 5 +-- tests/storage/node/helpers.nim | 1 - tests/storage/testdiscovery.nim | 2 +- tests/storage/testnatdetection.nim | 16 +++++----- tests/storage/testnatreaction.nim | 15 ++++----- tests/storage/testnatsimulation.nim | 38 +++++++++++------------ vendor/nim-libp2p | 2 +- 10 files changed, 53 insertions(+), 41 deletions(-) diff --git a/config.nims b/config.nims index 5b1ecb00..f369ca80 100644 --- a/config.nims +++ b/config.nims @@ -140,7 +140,7 @@ switch("warning", "ObservableStores:off") # Too many false positives for "Warning: method has lock level , but another method has 0 [LockLevel]" switch("warning", "LockLevel:off") -switch("define", "libp2p_pki_schemes=secp256k1") +switch("define", "libp2p_pki_schemes=secp256k1,rsa") #TODO this infects everything in this folder, ideally it would only # apply to storage.nim, but since storage.nims is used for other purpose # we can't use it. And storage.cfg doesn't work diff --git a/storage/blockexchange/network/network.nim b/storage/blockexchange/network/network.nim index 1d7ebafb..b99d8af7 100644 --- a/storage/blockexchange/network/network.nim +++ b/storage/blockexchange/network/network.nim @@ -314,8 +314,15 @@ proc new*( ## Create a new BlockExcNetwork instance ## + # libp2p now requires a non-nil handler at construction; the real one is set + # by self.init() below. This placeholder only exists until then. + proc placeholder( + conn: Connection, proto: string + ): Future[void] {.async: (raises: [CancelledError]).} = + discard + let self = lp_protocol.new( - BlockExcNetwork, @[Codec], nil, maxIncomingStreamsTotal = maxInflight + BlockExcNetwork, @[Codec], placeholder, maxIncomingStreamsTotal = maxInflight ) self.switch = switch self.getConn = connProvider diff --git a/storage/logutils.nim b/storage/logutils.nim index ebf41fd1..cbf41916 100644 --- a/storage/logutils.nim +++ b/storage/logutils.nim @@ -94,7 +94,7 @@ import std/typetraits import pkg/chronicles except toJson, `%` import json_serialization/writer as json_serialization_writer from pkg/chronos import TransportAddress -from pkg/libp2p import Cid, MultiAddress, `$` +from pkg/libp2p import Cid, MultiAddress, SignedPeerRecord, `$` import pkg/questionable import pkg/questionable/results import ./utils/json except formatIt # TODO: remove exception? @@ -248,6 +248,8 @@ formatIt(UInt256): $it formatIt(MultiAddress): $it +formatIt(SignedPeerRecord): + $it formatIt(LogFormat.textLines, array[32, byte]): it.short0xHexLog formatIt(LogFormat.json, array[32, byte]): diff --git a/storage/storage.nim b/storage/storage.nim index dfe0fb12..aa0ddc84 100644 --- a/storage/storage.nim +++ b/storage/storage.nim @@ -263,7 +263,8 @@ proc new*( var switchBuilder = SwitchBuilder .new() .withPrivateKey(privateKey) - .withAddresses(@[listenMultiAddr], enableWildcardResolver = true) + .withAddresses(@[listenMultiAddr]) + .withWildcardResolver() .withIdentifyPusher(false) .withRng(random.Rng.instance().libp2pRng) .withNoise() @@ -469,7 +470,7 @@ proc new*( info "Relay reservation updated", addresses = publicAddrs # relay addresses are for download traffic only, not DHT routing discovery.announceRelayAddrs(publicAddrs), - rng = random.Rng.instance(), + rng = random.Rng.instance().libp2pRng, ) relayService.setup(switch) diff --git a/tests/storage/node/helpers.nim b/tests/storage/node/helpers.nim index af4012b8..d5e9678f 100644 --- a/tests/storage/node/helpers.nim +++ b/tests/storage/node/helpers.nim @@ -7,7 +7,6 @@ import pkg/storage/chunker import pkg/storage/stores import ../../asynctest -import ../helpers/switchutils type CountingStore* = ref object of NetworkStore lookups*: Table[Cid, int] diff --git a/tests/storage/testdiscovery.nim b/tests/storage/testdiscovery.nim index 8178319b..f0cb5a3a 100644 --- a/tests/storage/testdiscovery.nim +++ b/tests/storage/testdiscovery.nim @@ -20,7 +20,7 @@ suite "Discovery - SPR record logic": udpPort = Port(8090) setup: - key = PrivateKey.random(Rng.instance()).get() + key = PrivateKey.random(Rng.instance().libp2pRng).get() disc = Discovery.new(key, announceAddrs = @[]) test "announceDirectAddrs sets the SPR with both TCP and UDP addresses": diff --git a/tests/storage/testnatdetection.nim b/tests/storage/testnatdetection.nim index 3cc01f34..e6c73d89 100644 --- a/tests/storage/testnatdetection.nim +++ b/tests/storage/testnatdetection.nim @@ -56,8 +56,8 @@ method sendDialRequest*( proc serverSwitch(): Switch = SwitchBuilder .new() - .withRng(Rng.instance()) - .withPrivateKey(PrivateKey.random(Rng.instance()).get()) + .withRng(Rng.instance().libp2pRng) + .withPrivateKey(PrivateKey.random(Rng.instance().libp2pRng).get()) .withAddresses(@[MultiAddress.init(listenAddr).get()]) .withTcpTransport(flags) .withNoise() @@ -77,8 +77,8 @@ asyncchecksuite "NAT detection - simulated NAT": let relayClient = relayClientModule.RelayClient.new() natNode = SwitchBuilder .new() - .withRng(Rng.instance()) - .withPrivateKey(PrivateKey.random(Rng.instance()).get()) + .withRng(Rng.instance().libp2pRng) + .withPrivateKey(PrivateKey.random(Rng.instance().libp2pRng).get()) .withAddresses(@[MultiAddress.init(listenAddr).get()]) .withNatTransport(router, flags) .withNoise() @@ -86,9 +86,11 @@ asyncchecksuite "NAT detection - simulated NAT": .withCircuitRelay(relayClient) .build() - relay = AutoRelayService.new(1, relayClient, nil, Rng.instance()) + relay = AutoRelayService.new(1, relayClient, nil, Rng.instance().libp2pRng) autorelayservice.setup(relay, natNode) - disc = Discovery.new(PrivateKey.random(Rng.instance()).get(), announceAddrs = @[]) + disc = Discovery.new( + PrivateKey.random(Rng.instance().libp2pRng).get(), announceAddrs = @[] + ) # nodes start in client mode until Reachable disc.protocol.clientMode = true @@ -203,7 +205,7 @@ asyncchecksuite "NAT detection - dial request candidates": test "autonat handles the observed dialable address": let mockClient = MockAutonatV2Client() let autonat = AutonatV2Service.new( - Rng.instance(), + Rng.instance().libp2pRng, mockClient, AutonatV2ServiceConfig.new( enableDialableCandidates = true, maxQueueSize = 1, minConfidence = 0.5 diff --git a/tests/storage/testnatreaction.nim b/tests/storage/testnatreaction.nim index 258f8606..6607c1a0 100644 --- a/tests/storage/testnatreaction.nim +++ b/tests/storage/testnatreaction.nim @@ -36,9 +36,10 @@ asyncchecksuite "NAT reaction - port mapping": var autoRelay: AutoRelayService setup: - autoRelay = - AutoRelayService.new(1, relayClientModule.RelayClient.new(), nil, Rng.instance()) - key = PrivateKey.random(Rng.instance()).get() + autoRelay = AutoRelayService.new( + 1, relayClientModule.RelayClient.new(), nil, Rng.instance().libp2pRng + ) + key = PrivateKey.random(Rng.instance().libp2pRng).get() disc = Discovery.new(key, announceAddrs = @[]) sw = newStandardSwitch() await sw.start() @@ -149,7 +150,7 @@ asyncchecksuite "NAT reaction - address announcing": var disc: Discovery setup: - key = PrivateKey.random(Rng.instance()).get() + key = PrivateKey.random(Rng.instance().libp2pRng).get() disc = Discovery.new(key, announceAddrs = @[]) sw = newStandardSwitch() await sw.start() @@ -179,7 +180,7 @@ asyncchecksuite "NAT reaction - address announcing": check disc.getSpr().data.seqNo == seqNo test "peerInfo observer announces addresses when Reachable": - let autonat = AutonatV2Service.new(Rng.instance()) + let autonat = AutonatV2Service.new(Rng.instance().libp2pRng) discard setupPeerInfoObserver( sw, autonat, disc, NatPortMapper(discoveryPort: discoveryPort) ) @@ -193,7 +194,7 @@ asyncchecksuite "NAT reaction - address announcing": check disc.announceAddrs == sw.peerInfo.addrs test "peerInfo observer announces the mapped external UDP port when a mapping is active": - let autonat = AutonatV2Service.new(Rng.instance()) + let autonat = AutonatV2Service.new(Rng.instance().libp2pRng) let mapper = NatPortMapper(discoveryPort: discoveryPort, activeUdpPort: some(Port(40001))) discard setupPeerInfoObserver(sw, autonat, disc, mapper) @@ -210,7 +211,7 @@ asyncchecksuite "NAT reaction - address announcing": sprAddrs test "peerInfo observer does not announce when the node is not Reachable": - let autonat = AutonatV2Service.new(Rng.instance()) + let autonat = AutonatV2Service.new(Rng.instance().libp2pRng) discard setupPeerInfoObserver( sw, autonat, disc, NatPortMapper(discoveryPort: discoveryPort) ) diff --git a/tests/storage/testnatsimulation.nim b/tests/storage/testnatsimulation.nim index 00f6652d..6e046526 100644 --- a/tests/storage/testnatsimulation.nim +++ b/tests/storage/testnatsimulation.nim @@ -4,7 +4,7 @@ import pkg/libp2p/wire import ./helpers import ../asynctest -import ../../storage/rng +import ../../storage/rng as storage_rng import ../../storage/nat import ./natsimulation @@ -22,22 +22,22 @@ proc cannotConnect(a, b: Switch): Future[bool] {.async.} = return false return not a.isConnected(b.peerInfo.peerId) -proc newSwitch(rng: Rng): Switch = +proc newSwitch(rng: storage_rng.Rng): Switch = SwitchBuilder .new() - .withRng(rng) - .withPrivateKey(PrivateKey.random(rng).get()) + .withRng(rng.libp2pRng) + .withPrivateKey(PrivateKey.random(rng.libp2pRng).get()) .withAddresses(@[MultiAddress.init(listenAddr).get()]) .withTcpTransport(flags) .withNoise() .withYamux() .build() -proc newNatSwitch(router: NatRouter, rng: Rng): Switch = +proc newNatSwitch(router: NatRouter, rng: storage_rng.Rng): Switch = SwitchBuilder .new() - .withRng(rng) - .withPrivateKey(PrivateKey.random(rng).get()) + .withRng(rng.libp2pRng) + .withPrivateKey(PrivateKey.random(rng.libp2pRng).get()) .withAddresses(@[MultiAddress.init(listenAddr).get()]) .withNatTransport(router, flags) .withNoise() @@ -49,8 +49,8 @@ asyncchecksuite "Nat transport - Endpoint-Independent Filtering": setup: let router = NatRouter.new(EndpointIndependent) - bootstrap = newSwitch(Rng.instance()) - natNode = newNatSwitch(router, Rng.instance()) + bootstrap = newSwitch(storage_rng.Rng.instance()) + natNode = newNatSwitch(router, storage_rng.Rng.instance()) await bootstrap.start() await natNode.start() @@ -67,9 +67,9 @@ asyncchecksuite "Nat transport - Address-Dependent Filtering": setup: let router = NatRouter.new(AddressDependent) - bootstrap = newSwitch(Rng.instance()) - thirdNode = newSwitch(Rng.instance()) - natNode = newNatSwitch(router, Rng.instance()) + bootstrap = newSwitch(storage_rng.Rng.instance()) + thirdNode = newSwitch(storage_rng.Rng.instance()) + natNode = newNatSwitch(router, storage_rng.Rng.instance()) await bootstrap.start() await thirdNode.start() await natNode.start() @@ -99,9 +99,9 @@ asyncchecksuite "Nat transport - Address-and-Port-Dependent Filtering": setup: let router = NatRouter.new(AddressAndPortDependent) - bootstrap = newSwitch(Rng.instance()) - thirdNode = newSwitch(Rng.instance()) - natNode = newNatSwitch(router, Rng.instance()) + bootstrap = newSwitch(storage_rng.Rng.instance()) + thirdNode = newSwitch(storage_rng.Rng.instance()) + natNode = newNatSwitch(router, storage_rng.Rng.instance()) await bootstrap.start() await thirdNode.start() await natNode.start() @@ -131,8 +131,8 @@ asyncchecksuite "Nat transport - Double NAT": setup: router = NatRouter.new(DoubleNat) - bootstrap = newSwitch(Rng.instance()) - natNode = newNatSwitch(router, Rng.instance()) + bootstrap = newSwitch(storage_rng.Rng.instance()) + natNode = newNatSwitch(router, storage_rng.Rng.instance()) await bootstrap.start() await natNode.start() @@ -154,8 +154,8 @@ asyncchecksuite "Nat transport - Port Mapping": setup: router = NatRouter.new(AddressAndPortDependent) - bootstrap = newSwitch(Rng.instance()) - natNode = newNatSwitch(router, Rng.instance()) + bootstrap = newSwitch(storage_rng.Rng.instance()) + natNode = newNatSwitch(router, storage_rng.Rng.instance()) await bootstrap.start() await natNode.start() diff --git a/vendor/nim-libp2p b/vendor/nim-libp2p index 2be4e5ed..1bd3b986 160000 --- a/vendor/nim-libp2p +++ b/vendor/nim-libp2p @@ -1 +1 @@ -Subproject commit 2be4e5edb39c113be7c293d53ec17c6cadbb9640 +Subproject commit 1bd3b986c82ab37a509fc84ca0ad7ea67a705a1b From 56e8ebe36b74db149bad70c0e90c3c89650df0e1 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Fri, 12 Jun 2026 19:27:16 +0400 Subject: [PATCH 125/167] Add NAT integration tests --- .github/workflows/ci-reusable.yml | 17 ++-- Makefile | 18 ++-- build.nims | 9 +- tests/imports.nim | 24 +++-- tests/integration/nat/Dockerfile | 36 ++++---- tests/integration/nat/composehelper.nim | 28 ++++++ tests/integration/nat/docker-entrypoint.sh | 70 -------------- tests/integration/nat/miniupnpd_stub_rdr.c | 2 +- tests/integration/nat/node-entrypoint.sh | 26 ++++++ tests/integration/nat/not-reachable/README.md | 55 +++++++++++ .../integration/nat/not-reachable/compose.yml | 91 ++++++++++++++++++ .../nat/not-reachable/router-entrypoint.sh | 7 ++ .../nat/not-reachable/testnotreachable.nim | 68 ++++++++++++++ tests/integration/nat/reachable/README.md | 54 +++++++++++ tests/integration/nat/reachable/compose.yml | 92 +++++++++++++++++++ .../nat/reachable/router-entrypoint.sh | 12 +++ .../nat/reachable/testreachable.nim | 77 ++++++++++++++++ tests/integration/nat/router-common.sh | 25 +++++ tests/testIntegration.nim | 5 +- tests/testNatIntegration.nim | 16 ++++ tests/testStorage.nim | 2 +- tools/scripts/ci-job-matrix.sh | 14 ++- 22 files changed, 617 insertions(+), 131 deletions(-) create mode 100644 tests/integration/nat/composehelper.nim delete mode 100644 tests/integration/nat/docker-entrypoint.sh create mode 100644 tests/integration/nat/node-entrypoint.sh create mode 100644 tests/integration/nat/not-reachable/README.md create mode 100644 tests/integration/nat/not-reachable/compose.yml create mode 100755 tests/integration/nat/not-reachable/router-entrypoint.sh create mode 100644 tests/integration/nat/not-reachable/testnotreachable.nim create mode 100644 tests/integration/nat/reachable/README.md create mode 100644 tests/integration/nat/reachable/compose.yml create mode 100755 tests/integration/nat/reachable/router-entrypoint.sh create mode 100644 tests/integration/nat/reachable/testreachable.nim create mode 100644 tests/integration/nat/router-common.sh create mode 100644 tests/testNatIntegration.nim diff --git a/.github/workflows/ci-reusable.yml b/.github/workflows/ci-reusable.yml index 89d42b00..dd4227ca 100644 --- a/.github/workflows/ci-reusable.yml +++ b/.github/workflows/ci-reusable.yml @@ -57,7 +57,7 @@ jobs: - name: Upload integration tests log files uses: actions/upload-artifact@v7 - if: (matrix.tests == 'integration' || matrix.tests == 'all') && always() + if: (matrix.tests == 'integration' || matrix.tests == 'nat-integration' || matrix.tests == 'all') && always() with: name: ${{ matrix.os }}-${{ matrix.cpu }}-${{ matrix.nim_version }}-${{ matrix.job_number }}-integration-tests-logs path: tests/integration/logs/ @@ -69,13 +69,14 @@ jobs: run: make -j${ncpu} testLibstorage ## Part 4 Tests ## - - name: NAT UPnP integration tests - if: matrix.tests == 'nat-upnp-integration' - run: make testNatUpnpIntegration - - - name: NAT PCP integration tests - if: matrix.tests == 'nat-pcp-integration' - run: make testNatPcpIntegration + - name: NAT integration tests + if: matrix.tests == 'nat-integration' + env: + STORAGE_INTEGRATION_TEST_INCLUDES: ${{ matrix.includes }} + run: | + sudo modprobe iptable_nat nf_conntrack + pipx install podman-compose + make testNatIntegration status: if: always() diff --git a/Makefile b/Makefile index 6828081d..10333663 100644 --- a/Makefile +++ b/Makefile @@ -87,8 +87,8 @@ endif testAll \ testIntegration \ testLibstorage \ - testNatUpnpIntegration \ - testNatPcpIntegration \ + buildNatImage \ + testNatIntegration \ update ifeq ($(NIM_PARAMS),) @@ -152,13 +152,15 @@ testIntegration: | build deps DOCKER := $(or $(shell which podman 2>/dev/null), $(shell which docker 2>/dev/null)) -testNatUpnpIntegration: - $(DOCKER) build -t miniupnpd-test -f tests/integration/nat/Dockerfile . - $(DOCKER) run --rm --cap-add NET_ADMIN -e DEBUG=$(DEBUG) miniupnpd-test +# NAT real-topology scenarios (podman-compose), all sharing one image built +# here. Runs every scenario; run one with +# `make testNatIntegration STORAGE_INTEGRATION_TEST_INCLUDES=` (the +# scenario's folder name, e.g. reachable). +buildNatImage: + $(DOCKER) build -t localhost/storage-nat -f tests/integration/nat/Dockerfile . -testNatPcpIntegration: - $(DOCKER) build -t miniupnpd-test -f tests/integration/nat/Dockerfile . - $(DOCKER) run --rm --cap-add NET_ADMIN -e DEBUG=$(DEBUG) -e TEST_PCP=1 miniupnpd-test +testNatIntegration: | deps buildNatImage + $(ENV_SCRIPT) nim testNatIntegration $(NIM_PARAMS) build.nims # Builds a C example that uses the libstorage C library and runs it testLibstorage: | build deps diff --git a/build.nims b/build.nims index f1831020..25a31538 100644 --- a/build.nims +++ b/build.nims @@ -78,12 +78,9 @@ task testIntegration, "Run integration tests": # test "testIntegration", params = "-d:chronicles_sinks=textlines[notimestamps,stdout],textlines[dynamic] " & # "-d:chronicles_enabled_topics:integration:TRACE" -task testNatNotReachable, - "Run NAT not-reachable scenario (needs the image + podman-compose)": - test "integration/nat/not-reachable/testnotreachable", outName = "testNatNotReachable" - -task testNatReachable, "Run NAT reachable scenario (needs the image + podman-compose)": - test "integration/nat/reachable/testreachable", outName = "testNatReachable" +task testNatIntegration, + "Run NAT real-topology scenarios (needs the storage-nat image + podman-compose)": + test "testNatIntegration" task build, "build Logos Storage binary": storageTask() diff --git a/tests/imports.nim b/tests/imports.nim index fbe642fc..da22f3c1 100644 --- a/tests/imports.nim +++ b/tests/imports.nim @@ -2,17 +2,25 @@ import std/macros import std/os import std/strutils -macro importTests*(dir: static string): untyped = - ## imports all files in the specified directory whose filename - ## starts with "test" and ends in ".nim" +macro importTests*( + dir: static string, exclude: static string, only: static string +): untyped = + ## imports every test*.nim file under `dir` (recursively). + ## `exclude` (when non-empty) skips files whose path contains it. + ## `only` (when non-empty) keeps only files whose path contains it. let imports = newStmtList() for file in walkDirRec(dir): let (_, name, ext) = splitFile(file) - if name.startsWith("test") and ext == ".nim": - imports.add( - quote do: - import `file` - ) + if not (name.startsWith("test") and ext == ".nim"): + continue + if exclude.len > 0 and exclude in file: + continue + if only.len > 0 and only notin file: + continue + imports.add( + quote do: + import `file` + ) imports macro importAll*(paths: static seq[string]): untyped = diff --git a/tests/integration/nat/Dockerfile b/tests/integration/nat/Dockerfile index 1795e1e6..82c35d3c 100644 --- a/tests/integration/nat/Dockerfile +++ b/tests/integration/nat/Dockerfile @@ -1,3 +1,7 @@ +# One image for every podman NAT scenario, built as localhost/storage-nat. +# Carries the storage binary + miniupnpd (for the upnp/pmp routers); scenarios +# differ only in their entrypoint scripts, which compose mounts. +# Build context = project root. FROM ubuntu:24.04 ARG NIM_VERSION=2.2.10 @@ -5,15 +9,12 @@ ARG NIM_VERSION=2.2.10 RUN apt-get update && apt-get install -y --no-install-recommends \ gcc g++ make cmake git curl ca-certificates xz-utils \ libc-dev ccache \ - iproute2 \ + iproute2 iptables jq \ && rm -rf /var/lib/apt/lists/* -# Build miniupnpd with a stub redirector. miniupnpd normally calls iptables/nftables -# to install the actual port forwarding rules when it receives a mapping request. -# In Docker, those calls fail because the container lacks the required kernel -# capabilities, causing every mapping request to return an error to the client. -# The stub replaces the firewall backend with no-ops that always return success, -# so mapping requests complete normally without touching the kernel. +# miniupnpd with a stub firewall backend: the real backend needs kernel caps a +# container lacks, so the stub makes mapping requests succeed without touching +# the kernel. Only the upnp/pmp routers use it. COPY tests/integration/nat/miniupnpd_stub_rdr.c /tmp/stub_rdr.c RUN git clone --depth=1 --branch miniupnpd_2_3_9 \ https://github.com/miniupnp/miniupnp.git /tmp/miniupnp \ @@ -24,32 +25,29 @@ RUN git clone --depth=1 --branch miniupnpd_2_3_9 \ && install -m 755 miniupnpd /usr/local/sbin/miniupnpd \ && rm -rf /tmp/miniupnp /tmp/stub_rdr.c -# Install Nim RUN curl -fsSL "https://nim-lang.org/download/nim-${NIM_VERSION}-linux_x64.tar.xz" \ - | tar -xJ -C /opt && \ - ln -s "/opt/nim-${NIM_VERSION}/bin/nim" /usr/local/bin/nim + | tar -xJ -C /opt +RUN ln -s "/opt/nim-${NIM_VERSION}/bin/nim" /usr/local/bin/nim WORKDIR /app -# Copy project source (build context must be the project root) +# vendor/ already has the checked-out submodules, so no `make update` here. COPY vendor/ vendor/ COPY storage/ storage/ -COPY library/ library/ -COPY tests/ tests/ COPY build.nims config.nims storage.nim ./ -# Build libplum C library. Nim binaries are compiled at test runtime. -# ccache caches C compilation across builds. +# libplum static lib, linked by nim-libplum. RUN --mount=type=cache,target=/root/.ccache \ export PATH="/usr/lib/ccache:$PATH" && \ rm -rf vendor/nim-libplum/vendor/libplum/build && \ cmake -B vendor/nim-libplum/vendor/libplum/build \ -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF \ vendor/nim-libplum/vendor/libplum && \ - make -j$(nproc) -C vendor/nim-libplum/vendor/libplum/build && \ + make -j"$(nproc)" -C vendor/nim-libplum/vendor/libplum/build && \ cp vendor/nim-libplum/vendor/libplum/build/libplum.a \ vendor/nim-libplum/vendor/libplum/libplum.a -COPY tests/integration/nat/docker-entrypoint.sh /entrypoint.sh -RUN chmod +x /entrypoint.sh -ENTRYPOINT ["/entrypoint.sh"] +RUN --mount=type=cache,target=/root/.ccache \ + export PATH="/usr/lib/ccache:$PATH" && \ + USE_SYSTEM_NIM=1 vendor/nimbus-build-system/scripts/env.sh \ + nim storage -d:disable_libbacktrace build.nims diff --git a/tests/integration/nat/composehelper.nim b/tests/integration/nat/composehelper.nim new file mode 100644 index 00000000..bf8e69a0 --- /dev/null +++ b/tests/integration/nat/composehelper.nim @@ -0,0 +1,28 @@ +## Helpers shared by the compose-driven NAT scenario tests (real topology, not +## the in-process simulation). Each scenario provides its own compose.yml and +## the list of services whose logs should be collected. + +import std/[os, osproc] +import ../utils + +proc compose*(composeFile, action: string) = + let cmd = "podman-compose -f \"" & composeFile & "\" " & action + doAssert execShellCmd(cmd) == 0, "command failed: " & cmd + +proc saveContainerLogs*( + composeFile, suiteName, testName, startTime: string, services: openArray[string] +) = + ## Writes each container's log via getLogFile, the same helper and layout as + ## the multinodes suite: tests/integration/logs/__/ + ## /.log. Must run before `down` destroys the containers. + for service in services: + try: + let + logFile = getLogFile("", startTime, suiteName, testName, service) + cmd = "podman-compose -f \"" & composeFile & "\" logs " & service + (output, code) = execCmdEx(cmd) + if code != 0: + echo "warning: '", cmd, "' exited ", code + writeFile(logFile, output) + except CatchableError as e: + echo "could not save logs for ", service, ": ", e.msg diff --git a/tests/integration/nat/docker-entrypoint.sh b/tests/integration/nat/docker-entrypoint.sh deleted file mode 100644 index 921a740e..00000000 --- a/tests/integration/nat/docker-entrypoint.sh +++ /dev/null @@ -1,70 +0,0 @@ -#!/bin/bash -set -euo pipefail - -RUNDIR=/tmp/miniupnpd -mkdir -p "$RUNDIR" - -# miniupnpd must listen on the same interface as the test node. -# We get the default route interface (e.g. eth0) and its IP. -LAN_IF=$(ip route show default | awk '/default/{print $5; exit}') -LAN_IP=$(ip -4 addr show "$LAN_IF" | awk '/inet /{print $2; exit}' | cut -d/ -f1) - -if [[ -z "$LAN_IF" ]]; then - echo "ERROR: could not determine LAN interface" >&2 - exit 1 -fi - -if [[ -z "$LAN_IP" ]]; then - echo "ERROR: could not determine LAN IP on $LAN_IF" >&2 - exit 1 -fi - -# We use a public WAN IP (1.2.3.4) on a dummy interface because miniupnpd -# disables port forwarding when the external interface has a private/RFC1918 -# address (treats it as double-NAT). -ip link add plum-wan type dummy -ip addr add 1.2.3.4/24 dev plum-wan -ip link set plum-wan up - -start_miniupnpd() { - local enable_pcp_pmp=$1 - cat > "$RUNDIR/miniupnpd.conf" << EOF -ext_ifname=plum-wan -listening_ip=$LAN_IF -enable_pcp_pmp=$enable_pcp_pmp -# port=0: pick a random HTTP port to avoid conflicts with host services. -port=0 -# Without an allow rule miniupnpd denies all mapping requests by default. -allow 1024-65535 0.0.0.0/0 1024-65535 -EOF - miniupnpd -d -f "$RUNDIR/miniupnpd.conf" > "$RUNDIR/miniupnpd.log" 2>&1 & - MINIUPNPD_PID=$! - sleep 1 - kill -0 "$MINIUPNPD_PID" 2>/dev/null \ - || { echo "ERROR: miniupnpd failed to start" >&2; cat "$RUNDIR/miniupnpd.log" >&2; exit 1; } - echo "miniupnpd started (pid $MINIUPNPD_PID)" -} - -export DEBUG=${DEBUG:-0} - -if [[ "${TEST_PCP:-0}" == "1" ]]; then - # PCP requires the UDP source IP to match the client_address in the MAP request. - # Point the default route at LAN_IP so libplum uses it as both gateway and PCP target. - ip route replace default via "$LAN_IP" dev "$LAN_IF" - start_miniupnpd yes - failed=0 - USE_SYSTEM_NIM=1 vendor/nimbus-build-system/scripts/env.sh \ - nim testNatPcpMapping -d:debug -d:disable_libbacktrace build.nims || failed=1 -else - start_miniupnpd no - failed=0 - USE_SYSTEM_NIM=1 vendor/nimbus-build-system/scripts/env.sh \ - nim testNatPortMapping -d:debug -d:disable_libbacktrace build.nims || failed=1 -fi - -if [[ "${DEBUG:-0}" == "1" ]]; then - echo "--- miniupnpd log ---" - cat "$RUNDIR/miniupnpd.log" 2>/dev/null || true -fi - -[ $failed -eq 0 ] || exit 1 diff --git a/tests/integration/nat/miniupnpd_stub_rdr.c b/tests/integration/nat/miniupnpd_stub_rdr.c index 9caf44b8..1e9be2b0 100644 --- a/tests/integration/nat/miniupnpd_stub_rdr.c +++ b/tests/integration/nat/miniupnpd_stub_rdr.c @@ -1,7 +1,7 @@ /* Stub firewall backend for miniupnpd used in Docker-based tests. * * miniupnpd normally calls iptables/nftables to install port forwarding rules - * when it processes a UPnP/PCP/NAT-PMP mapping request. In a Docker container + * when it processes a UPnP/PCP/NAT-PMP mapping request. In a container * those calls fail because the container lacks the required kernel capabilities, * causing every mapping request to return an error to the client. * diff --git a/tests/integration/nat/node-entrypoint.sh b/tests/integration/nat/node-entrypoint.sh new file mode 100644 index 00000000..b8b08ce4 --- /dev/null +++ b/tests/integration/nat/node-entrypoint.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# Redirect the traffic to our router instead +# of podman's own gateway to put B behind the NAT. +ip route replace default via "$ROUTER_LAN_IP" + +# Fetch the bootstrap SPR (retry: the bootstrap may still be starting). +echo "fetching bootstrap SPR from $BOOTSTRAP_API ..." +spr="" +for _ in $(seq 1 60); do + spr=$(curl -fsS -H 'Accept: text/plain' "$BOOTSTRAP_API/api/storage/v1/spr" || true) + [[ -n "$spr" ]] && break + sleep 1 +done +[[ -n "$spr" ]] || { echo "ERROR: could not fetch bootstrap SPR" >&2; exit 1; } + +# api-bindaddr=0.0.0.0 so the published host port reaches the REST API. +exec /app/build/storage \ + --listen-ip=0.0.0.0 --api-bindaddr=0.0.0.0 \ + --listen-port=8070 --disc-port=8090 --api-port=8080 \ + --bootstrap-node="$spr" \ + --nat-num-peers-to-ask=1 --nat-max-queue-size=1 \ + --nat-min-confidence=1.0 --nat-schedule-interval=30s \ + --data-dir=/data --log-level=DEBUG diff --git a/tests/integration/nat/not-reachable/README.md b/tests/integration/nat/not-reachable/README.md new file mode 100644 index 00000000..17762bb0 --- /dev/null +++ b/tests/integration/nat/not-reachable/README.md @@ -0,0 +1,55 @@ +# NAT not-reachable scenario + +## Scenario + +A node behind a NAT that cannot be reached from outside must be detected +`NotReachable` and fall back to bootstrap A's relay. + +## Topology + +``` +node B ──── lan ──── router (NAT) ──── wan ──── bootstrap A +``` + +- **bootstrap A** — public node on the wan, runs the relay + autonat server, + started with `--nat=extip` so it advertises its own public address. +- **router** — two interfaces (lan + wan). Does `lan -> wan` masquerade and *no* + inbound forward, so B can dial out but nothing can dial back in. +- **node B** — `nat=auto`, on the lan. Its default route points at the router, + so all wan-bound traffic is NATed. It fetches A's SPR over A's API to join, + then AutoNAT probes A and finds itself unreachable. + +The wan uses a real public range because our address policy keeps only public +dialable addresses: a private observed address would be filtered out and AutoNAT +would stay `Unknown` instead of `NotReachable`. The wan is `internal` so that +range never leaks to host routes. + +## Run + +```bash +make testNatIntegration STORAGE_INTEGRATION_TEST_INCLUDES=not-reachable +``` + +Runs this scenario (omit the var to run every NAT scenario). Builds the shared +image and runs `testnotreachable.nim`, which brings the compose topology up and +down. Rootless, but needs the host netfilter modules — if the router fails on +iptables: `sudo modprobe iptable_nat nf_conntrack`. + +## Expected result + +B ends up `NotReachable` with the relay running, announcing only its circuit +(relay) address — never a direct one. Its `debug/info`: + +```json +{ + "nat": { + "reachability": "NotReachable", + "clientMode": true, + "relayRunning": true, + "portMapping": "none" + } +} +``` + +Per-run container logs (router, bootstrap, node) are written before teardown to +`tests/integration/logs/__NAT_not_reachable//.log`. diff --git a/tests/integration/nat/not-reachable/compose.yml b/tests/integration/nat/not-reachable/compose.yml new file mode 100644 index 00000000..400ef872 --- /dev/null +++ b/tests/integration/nat/not-reachable/compose.yml @@ -0,0 +1,91 @@ +# A node behind a NAT that can't be reached from outside must be detected +# NotReachable and fall back to the relay. This checks it on a real container +# network with real iptables NAT, not the in-process simulation the unit tests +# use. Run via testnotreachable.nim. +# +# node B ──── lan ──── router (NAT) ──── wan ──── bootstrap A +name: nat-not-reachable + +# Topology addresses, named for their role (defined once, referenced below). +x-addresses: + # fake public internet; a routable range so B looks public to A + wan_subnet: &wan_subnet 7.7.7.0/24 + # private network behind the NAT + lan_subnet: &lan_subnet 10.99.0.0/24 + # A: public bootstrap, relay + autonat server + bootstrap_ip: &bootstrap_ip 7.7.7.10 + # router's public face + router_wan_ip: &router_wan_ip 7.7.7.2 + # router's private face = B's gateway + router_lan_ip: &router_lan_ip 10.99.0.2 + # B, behind the NAT + node_ip: &node_ip 10.99.0.10 + +networks: + wan: + # Keep the fake public range private, not exposed to the host + internal: true + ipam: + config: + - subnet: *wan_subnet + lan: + ipam: + config: + - subnet: *lan_subnet + +services: + router: + image: localhost/storage-nat + cap_add: [NET_ADMIN] + sysctls: + net.ipv4.ip_forward: 1 + networks: + wan: + ipv4_address: *router_wan_ip + lan: + ipv4_address: *router_lan_ip + environment: + ROUTER_WAN_IP: *router_wan_ip + LAN_SUBNET: *lan_subnet + # scripts mounted, so editing them needs no image rebuild + volumes: + - ../router-common.sh:/scripts/router-common.sh:ro,z + - ./router-entrypoint.sh:/scripts/router-entrypoint.sh:ro,z + entrypoint: ["bash", "/scripts/router-entrypoint.sh"] + + bootstrap: + image: localhost/storage-nat + networks: + wan: + ipv4_address: *bootstrap_ip + entrypoint: ["/app/build/storage"] + command: + - --listen-ip=0.0.0.0 + - --api-bindaddr=0.0.0.0 + - --listen-port=8070 + - --disc-port=8090 + - --api-port=8080 + # bootstrap_ip (anchors can't go inside a string) + - --nat=extip:7.7.7.10 + - --relay-server + - --autonat-server + - --no-bootstrap-node + - --data-dir=/data + - --log-level=DEBUG + + node: + image: localhost/storage-nat + cap_add: [NET_ADMIN] + depends_on: [router, bootstrap] + networks: + lan: + ipv4_address: *node_ip + ports: + - "127.0.0.1:18080:8080" + environment: + ROUTER_LAN_IP: *router_lan_ip + # B fetches A's SPR from this API at startup to join the network (bootstrap_ip) + BOOTSTRAP_API: http://7.7.7.10:8080 + volumes: + - ../node-entrypoint.sh:/scripts/node-entrypoint.sh:ro,z + entrypoint: ["bash", "/scripts/node-entrypoint.sh"] diff --git a/tests/integration/nat/not-reachable/router-entrypoint.sh b/tests/integration/nat/not-reachable/router-entrypoint.sh new file mode 100755 index 00000000..21a8ef3b --- /dev/null +++ b/tests/integration/nat/not-reachable/router-entrypoint.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +source "$(dirname "$0")/router-common.sh" + +echo "router ready (wan iface $wanif)" + +hold_until_stopped diff --git a/tests/integration/nat/not-reachable/testnotreachable.nim b/tests/integration/nat/not-reachable/testnotreachable.nim new file mode 100644 index 00000000..74b83f2c --- /dev/null +++ b/tests/integration/nat/not-reachable/testnotreachable.nim @@ -0,0 +1,68 @@ +## NAT not-reachable scenario — node behind a real NAT falls back to relay. +## +## Requires podman-compose and the scenario image: +## podman build -t localhost/storage-nat:not-reachable \ +## -f tests/integration/nat/not-reachable/Dockerfile . + +import std/[json, os, sequtils, strutils, times] +import pkg/chronos +import pkg/questionable/results + +import ../../../asynctest +import ../../../checktest +import ../../storageclient +import ../composehelper + +const + composeFile = currentSourcePath.parentDir / "compose.yml" + nodeApiUrl = "http://127.0.0.1:18080/api/storage/v1" + suiteName = "NAT not reachable" + testName = "node behind NAT is NotReachable and falls back to relay" + services = ["router", "bootstrap", "node"] + detectTimeout = 300_000 # ms + pollInterval = 5_000 # ms + +proc announcesCircuitAddr(info: JsonNode): bool = + info{"announceAddresses"}.getElems.anyIt("p2p-circuit" in it.getStr) + +asyncchecksuite suiteName: + let + composeFile = composeFile + nodeApiUrl = nodeApiUrl + suiteName = suiteName + testName = testName + services = services + startTime = now().format("yyyy-MM-dd'_'HH:mm:ss") + var client: StorageClient + + setup: + compose(composeFile, "up -d") + client = StorageClient.new(nodeApiUrl) + + teardown: + await client.close() + saveContainerLogs(composeFile, suiteName, testName, startTime, services) + compose(composeFile, "down -v") + + test testName: + # Wait for the announcements, after the relay reservation is created. + check eventuallySafe( + block: + var settled = false + try: + let info = await client.info() + settled = info.isOk and info.get.announcesCircuitAddr() + except HttpError: + # B's API is not up yet, keep polling + discard + settled, + timeout = detectTimeout, + pollInterval = pollInterval, + ) + + let info = (await client.info()).get + let nat = info{"nat"} + check nat{"reachability"}.getStr == "NotReachable" + check nat{"relayRunning"}.getBool + check nat{"portMapping"}.getStr == "none" + check info.announcesCircuitAddr() diff --git a/tests/integration/nat/reachable/README.md b/tests/integration/nat/reachable/README.md new file mode 100644 index 00000000..5289eed0 --- /dev/null +++ b/tests/integration/nat/reachable/README.md @@ -0,0 +1,54 @@ +# NAT reachable scenario + +## Scenario + +A node behind a NAT whose port is forwarded must be detected `Reachable` and +keep its direct address — no relay fallback. + +## Topology + +``` +node B ──── lan ──── router (NAT + port forward) ──── wan ──── bootstrap A +``` + +- **bootstrap A** — public node on the wan, runs the relay + autonat server. +- **router** — `lan -> wan` masquerade *plus* a static DNAT forwarding B's TCP + listen port (8070) and UDP disc port (8090) inbound. No miniupnpd: the router + opens the port itself, so B maps nothing. +- **node B** — `nat=auto`, on the lan, default route through the router. It dials + out from its listen port (8070) and the masquerade keeps that port, so A + observes it at `7.7.7.2:8070` — exactly what the DNAT forwards back, so the + dial-back reaches it. + +The wan public range and `internal` flag work as in +[not-reachable](../not-reachable/README.md). + +## Run + +```bash +make testNatIntegration STORAGE_INTEGRATION_TEST_INCLUDES=reachable +``` + +Runs this scenario (omit the var to run every NAT scenario). Builds the shared +image and runs `testreachable.nim`, which brings the compose topology up and +down. Rootless, but needs the host netfilter modules — if the router fails on +iptables: `sudo modprobe iptable_nat nf_conntrack`. + +## Expected result + +B ends up `Reachable`, the relay not running, announcing its direct address — +not a circuit one. Its `debug/info`: + +```json +{ + "nat": { + "reachability": "Reachable", + "clientMode": false, + "relayRunning": false, + "portMapping": "none" + } +} +``` + +Per-run container logs (router, bootstrap, node) are written before teardown to +`tests/integration/logs/__NAT_reachable//.log`. diff --git a/tests/integration/nat/reachable/compose.yml b/tests/integration/nat/reachable/compose.yml new file mode 100644 index 00000000..77da9992 --- /dev/null +++ b/tests/integration/nat/reachable/compose.yml @@ -0,0 +1,92 @@ +# Same setup as not-reachable, but the router forwards B's port (DNAT), so +# AutoNAT's dial-back reaches B and it is detected Reachable — no relay needed. +# +# node B ──── lan ──── router (NAT + port forward) ──── wan ──── bootstrap A +name: nat-reachable + +# Topology addresses, named for their role (defined once, referenced below). +x-addresses: + # fake public internet; a routable range so B looks public to A + wan_subnet: &wan_subnet 7.7.7.0/24 + # private network behind the NAT + lan_subnet: &lan_subnet 10.99.0.0/24 + # A: public bootstrap, relay + autonat server + bootstrap_ip: &bootstrap_ip 7.7.7.10 + # router's public face + router_wan_ip: &router_wan_ip 7.7.7.2 + # router's private face = B's gateway + router_lan_ip: &router_lan_ip 10.99.0.2 + # B, behind the NAT (also the DNAT target) + node_ip: &node_ip 10.99.0.10 + +networks: + wan: + # Keep the fake public range private, not exposed to the host + internal: true + ipam: + config: + - subnet: *wan_subnet + lan: + ipam: + config: + - subnet: *lan_subnet + +services: + router: + image: localhost/storage-nat + cap_add: [NET_ADMIN] + sysctls: + net.ipv4.ip_forward: 1 + networks: + wan: + ipv4_address: *router_wan_ip + lan: + ipv4_address: *router_lan_ip + environment: + ROUTER_WAN_IP: *router_wan_ip + LAN_SUBNET: *lan_subnet + # where the router forwards the port + NODE_IP: *node_ip + # scripts mounted, not baked, so editing them needs no image rebuild + volumes: + - ../router-common.sh:/scripts/router-common.sh:ro,z + - ./router-entrypoint.sh:/scripts/router-entrypoint.sh:ro,z + entrypoint: ["bash", "/scripts/router-entrypoint.sh"] + + bootstrap: + image: localhost/storage-nat + networks: + wan: + ipv4_address: *bootstrap_ip + entrypoint: ["/app/build/storage"] + command: + - --listen-ip=0.0.0.0 + - --api-bindaddr=0.0.0.0 + - --listen-port=8070 + - --disc-port=8090 + - --api-port=8080 + # bootstrap_ip (anchors can't go inside a string) + - --nat=extip:7.7.7.10 + - --relay-server + - --autonat-server + - --no-bootstrap-node + - --data-dir=/data + - --log-level=DEBUG + + node: + image: localhost/storage-nat + cap_add: [NET_ADMIN] + depends_on: [router, bootstrap] + networks: + lan: + ipv4_address: *node_ip + # B's API, published so the test can poll it + ports: + - "127.0.0.1:18081:8080" + environment: + ROUTER_LAN_IP: *router_lan_ip + # B fetches A's SPR from this API at startup to join the network (bootstrap_ip) + BOOTSTRAP_API: http://7.7.7.10:8080 + volumes: + - ../node-entrypoint.sh:/scripts/node-entrypoint.sh:ro,z + entrypoint: ["bash", "/scripts/node-entrypoint.sh"] diff --git a/tests/integration/nat/reachable/router-entrypoint.sh b/tests/integration/nat/reachable/router-entrypoint.sh new file mode 100755 index 00000000..1aa07e06 --- /dev/null +++ b/tests/integration/nat/reachable/router-entrypoint.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +source "$(dirname "$0")/router-common.sh" + +# Forward the node's TCP listen port (what AutoNAT dials back) and UDP disc port +# in order to simulate the port mapping. +iptables -t nat -A PREROUTING -i "$wanif" -p tcp --dport 8070 -j DNAT --to-destination "$NODE_IP:8070" +iptables -t nat -A PREROUTING -i "$wanif" -p udp --dport 8090 -j DNAT --to-destination "$NODE_IP:8090" + +echo "router ready (forwarding tcp/8070 + udp/8090 to $NODE_IP, wan iface $wanif)" + +hold_until_stopped diff --git a/tests/integration/nat/reachable/testreachable.nim b/tests/integration/nat/reachable/testreachable.nim new file mode 100644 index 00000000..93e26bdb --- /dev/null +++ b/tests/integration/nat/reachable/testreachable.nim @@ -0,0 +1,77 @@ +## NAT reachable scenario — node behind a real NAT is Reachable because the +## router forwards its port. +## +## Same shape as the not-reachable test: compose.yml brings up a real NAT +## topology, but the router has a static inbound port-forward (DNAT) to the node. +## AutoNAT's dial-back reaches the node, so it is detected Reachable (no relay) — +## a manual port-forward / endpoint-independent NAT, no miniupnpd. +## +## Requires podman-compose and the scenario image: +## podman build -t localhost/storage-nat:reachable \ +## -f tests/integration/nat/reachable/Dockerfile . + +import std/[json, os, sequtils, strutils, times] +import pkg/chronos +import pkg/questionable/results + +import ../../../asynctest +import ../../../checktest +import ../../storageclient +import ../composehelper + +const + composeFile = currentSourcePath.parentDir / "compose.yml" + nodeApiUrl = "http://127.0.0.1:18081/api/storage/v1" + suiteName = "NAT reachable" + testName = "node behind NAT with a forwarded port is Reachable" + services = ["router", "bootstrap", "node"] + detectTimeout = 120_000 # ms + pollInterval = 5_000 # ms + +proc announcesDirectAddr(info: JsonNode): bool = + ## A reachable node announces at least one direct (non-circuit) address. + info{"announceAddresses"}.getElems.anyIt("p2p-circuit" notin it.getStr) + +asyncchecksuite suiteName: + # chronos' async setup/teardown cannot reference module-level GC'ed consts + # (strings/seqs), so rebind the ones they use to suite locals. + let + composeFile = composeFile + nodeApiUrl = nodeApiUrl + suiteName = suiteName + testName = testName + services = services + startTime = now().format("yyyy-MM-dd'_'HH:mm:ss") + var client: StorageClient + + setup: + compose(composeFile, "up -d") + client = StorageClient.new(nodeApiUrl) + + teardown: + await client.close() + saveContainerLogs(composeFile, suiteName, testName, startTime, services) + compose(composeFile, "down -v") + + test testName: + # Reachable is the settling signal: wait for it, then assert each expected + # property separately so a failure points at the exact condition. + check eventuallySafe( + block: + var reachable = false + try: + let info = await client.info() + reachable = + info.isOk and info.get{"nat"}{"reachability"}.getStr == "Reachable" + except HttpError: + discard # B's API is not up yet, keep polling + reachable, + timeout = detectTimeout, + pollInterval = pollInterval, + ) + + let info = (await client.info()).get + let nat = info{"nat"} + check nat{"reachability"}.getStr == "Reachable" + check nat{"relayRunning"}.getBool == false + check info.announcesDirectAddr() diff --git a/tests/integration/nat/router-common.sh b/tests/integration/nat/router-common.sh new file mode 100644 index 00000000..66461f77 --- /dev/null +++ b/tests/integration/nat/router-common.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +# Shared router base, sourced by each scenario's router-entrypoint.sh + +set -euo pipefail + +# iptables needs the wan interface's name (eth0/eth1), but podman assigns those +# names arbitrarily — so look for the name using the wan IP, +# defined in the compose file. +wanif=$(ip -o -4 addr show | awk -v ip="$ROUTER_WAN_IP" '$0 ~ ip {print $2; exit}') + +if ! iptables -t nat -A POSTROUTING -s "$LAN_SUBNET" -o "$wanif" -j MASQUERADE; then + echo "ERROR: iptables NAT failed. Load netfilter modules on the host:" >&2 + echo " sudo modprobe iptable_nat nf_conntrack" >&2 + exit 1 +fi +iptables -P FORWARD ACCEPT + +# Block until `compose down`. sleep runs in the background so the SIGTERM trap +# fires immediately instead of waiting for sleep to return. +hold_until_stopped() { + trap 'exit 0' TERM INT + sleep infinity & + wait +} diff --git a/tests/testIntegration.nim b/tests/testIntegration.nim index 5e289ff5..488e4136 100644 --- a/tests/testIntegration.nim +++ b/tests/testIntegration.nim @@ -11,7 +11,8 @@ when includes != "": # import only the specified tests importAll(includes.split(",")) else: - # import all tests in the integration/ directory - importTests(currentSourcePath().parentDir() / "integration") + # all tests in integration/, except the nat/ real-topology scenarios, which + # need podman + the storage-nat image and run via testNatIntegration instead + importTests(currentSourcePath().parentDir() / "integration", "/nat/", "") {.warning[UnusedImport]: off.} diff --git a/tests/testNatIntegration.nim b/tests/testNatIntegration.nim new file mode 100644 index 00000000..d5d116c0 --- /dev/null +++ b/tests/testNatIntegration.nim @@ -0,0 +1,16 @@ +import std/os +import ./imports + +## Real-topology NAT scenarios (need podman + the storage-nat image). +## Run a single scenario by setting its folder name during compilation, e.g. +## STORAGE_INTEGRATION_TEST_INCLUDES=reachable +const scenario = getEnv("STORAGE_INTEGRATION_TEST_INCLUDES") +const only = + if scenario.len > 0: + "/" & scenario & "/" + else: + "" + +importTests(currentSourcePath().parentDir() / "integration" / "nat", "", only) + +{.warning[UnusedImport]: off.} diff --git a/tests/testStorage.nim b/tests/testStorage.nim index 609a50ac..55b22d4b 100644 --- a/tests/testStorage.nim +++ b/tests/testStorage.nim @@ -1,6 +1,6 @@ import std/os import ./imports -importTests(currentSourcePath().parentDir() / "storage") +importTests(currentSourcePath().parentDir() / "storage", "", "") {.warning[UnusedImport]: off.} diff --git a/tools/scripts/ci-job-matrix.sh b/tools/scripts/ci-job-matrix.sh index 15b5db96..9047df42 100755 --- a/tools/scripts/ci-job-matrix.sh +++ b/tools/scripts/ci-job-matrix.sh @@ -95,9 +95,10 @@ integration_test () { integration_test_job $tests done - # fail when there are integration tests with an unknown duration + # fail when there are integration tests with an unknown duration. nat/ is + # excluded: those real-topology scenarios run via the nat-integration job. local filter='1_minute\|5_minutes\|30_minutes' - local unknown=$(find_tests tests/integration | grep -v "$filter") + local unknown=$(find_tests tests/integration | grep -v "$filter" | grep -v '/nat/') if [ "$unknown" != "" ]; then echo "Error: Integration tests need to be in either the 1_minute," >&2 echo " 5_minutes, or 30_minutes directory, based on the maximum" >&2 @@ -117,13 +118,10 @@ libstorage_test () { job } -# outputs NAT integration test jobs -# Linux-only: miniupnpd is a Linux daemon, network namespace manipulation requires Linux +# outputs the NAT real-topology integration job (all scenarios). +# Linux-only: needs network-namespace + iptables manipulation. nat_integration_tests () { - job_tests="nat-upnp-integration" - job_includes="" - job - job_tests="nat-pcp-integration" + job_tests="nat-integration" job_includes="" job } From 0afa65125861641d4bf7918e6132f90108c353eb Mon Sep 17 00:00:00 2001 From: Arnaud Date: Fri, 12 Jun 2026 19:59:28 +0400 Subject: [PATCH 126/167] Remove only option --- Makefile | 7 +++---- tests/imports.nim | 9 ++------- tests/integration/nat/not-reachable/README.md | 19 ++++++++++++++----- tests/integration/nat/reachable/README.md | 19 ++++++++++++++----- tests/testIntegration.nim | 2 +- tests/testNatIntegration.nim | 17 ++++++++--------- tests/testStorage.nim | 2 +- 7 files changed, 43 insertions(+), 32 deletions(-) diff --git a/Makefile b/Makefile index 10333663..3ecc4c10 100644 --- a/Makefile +++ b/Makefile @@ -152,10 +152,9 @@ testIntegration: | build deps DOCKER := $(or $(shell which podman 2>/dev/null), $(shell which docker 2>/dev/null)) -# NAT real-topology scenarios (podman-compose), all sharing one image built -# here. Runs every scenario; run one with -# `make testNatIntegration STORAGE_INTEGRATION_TEST_INCLUDES=` (the -# scenario's folder name, e.g. reachable). +# NAT real-topology scenarios (podman-compose), all sharing one image built here. +# Runs every scenario; limit it with STORAGE_INTEGRATION_TEST_INCLUDES (test file +# paths), as testIntegration does. buildNatImage: $(DOCKER) build -t localhost/storage-nat -f tests/integration/nat/Dockerfile . diff --git a/tests/imports.nim b/tests/imports.nim index da22f3c1..19ab608a 100644 --- a/tests/imports.nim +++ b/tests/imports.nim @@ -2,12 +2,9 @@ import std/macros import std/os import std/strutils -macro importTests*( - dir: static string, exclude: static string, only: static string -): untyped = +macro importTests*(dir: static string, exclude: static string): untyped = ## imports every test*.nim file under `dir` (recursively). - ## `exclude` (when non-empty) skips files whose path contains it. - ## `only` (when non-empty) keeps only files whose path contains it. + ## `exclude`, when non-empty, skips files whose path contains it. let imports = newStmtList() for file in walkDirRec(dir): let (_, name, ext) = splitFile(file) @@ -15,8 +12,6 @@ macro importTests*( continue if exclude.len > 0 and exclude in file: continue - if only.len > 0 and only notin file: - continue imports.add( quote do: import `file` diff --git a/tests/integration/nat/not-reachable/README.md b/tests/integration/nat/not-reachable/README.md index 17762bb0..32d9e008 100644 --- a/tests/integration/nat/not-reachable/README.md +++ b/tests/integration/nat/not-reachable/README.md @@ -26,14 +26,23 @@ range never leaks to host routes. ## Run +Every NAT scenario: + ```bash -make testNatIntegration STORAGE_INTEGRATION_TEST_INCLUDES=not-reachable +make testNatIntegration ``` -Runs this scenario (omit the var to run every NAT scenario). Builds the shared -image and runs `testnotreachable.nim`, which brings the compose topology up and -down. Rootless, but needs the host netfilter modules — if the router fails on -iptables: `sudo modprobe iptable_nat nf_conntrack`. +Just this one — same `STORAGE_INTEGRATION_TEST_INCLUDES` filter as testIntegration, +with the test file path: + +```bash +make testNatIntegration \ + STORAGE_INTEGRATION_TEST_INCLUDES=tests/integration/nat/not-reachable/testnotreachable.nim +``` + +Builds the shared image and brings the compose topology up and down. Rootless, but +needs the host netfilter modules — if the router fails on iptables: +`sudo modprobe iptable_nat nf_conntrack`. ## Expected result diff --git a/tests/integration/nat/reachable/README.md b/tests/integration/nat/reachable/README.md index 5289eed0..199a55b5 100644 --- a/tests/integration/nat/reachable/README.md +++ b/tests/integration/nat/reachable/README.md @@ -25,14 +25,23 @@ The wan public range and `internal` flag work as in ## Run +Every NAT scenario: + ```bash -make testNatIntegration STORAGE_INTEGRATION_TEST_INCLUDES=reachable +make testNatIntegration ``` -Runs this scenario (omit the var to run every NAT scenario). Builds the shared -image and runs `testreachable.nim`, which brings the compose topology up and -down. Rootless, but needs the host netfilter modules — if the router fails on -iptables: `sudo modprobe iptable_nat nf_conntrack`. +Just this one — same `STORAGE_INTEGRATION_TEST_INCLUDES` filter as testIntegration, +with the test file path: + +```bash +make testNatIntegration \ + STORAGE_INTEGRATION_TEST_INCLUDES=tests/integration/nat/reachable/testreachable.nim +``` + +Builds the shared image and brings the compose topology up and down. Rootless, but +needs the host netfilter modules — if the router fails on iptables: +`sudo modprobe iptable_nat nf_conntrack`. ## Expected result diff --git a/tests/testIntegration.nim b/tests/testIntegration.nim index 488e4136..c5f576ef 100644 --- a/tests/testIntegration.nim +++ b/tests/testIntegration.nim @@ -13,6 +13,6 @@ when includes != "": else: # all tests in integration/, except the nat/ real-topology scenarios, which # need podman + the storage-nat image and run via testNatIntegration instead - importTests(currentSourcePath().parentDir() / "integration", "/nat/", "") + importTests(currentSourcePath().parentDir() / "integration", "/nat/") {.warning[UnusedImport]: off.} diff --git a/tests/testNatIntegration.nim b/tests/testNatIntegration.nim index d5d116c0..5e16eee5 100644 --- a/tests/testNatIntegration.nim +++ b/tests/testNatIntegration.nim @@ -1,16 +1,15 @@ import std/os +import std/strutils import ./imports ## Real-topology NAT scenarios (need podman + the storage-nat image). -## Run a single scenario by setting its folder name during compilation, e.g. -## STORAGE_INTEGRATION_TEST_INCLUDES=reachable -const scenario = getEnv("STORAGE_INTEGRATION_TEST_INCLUDES") -const only = - if scenario.len > 0: - "/" & scenario & "/" - else: - "" +## Limit which scenarios run with STORAGE_INTEGRATION_TEST_INCLUDES, listing test +## file paths, exactly as testIntegration does. +const includes = getEnv("STORAGE_INTEGRATION_TEST_INCLUDES") -importTests(currentSourcePath().parentDir() / "integration" / "nat", "", only) +when includes != "": + importAll(includes.split(",")) +else: + importTests(currentSourcePath().parentDir() / "integration" / "nat", "") {.warning[UnusedImport]: off.} diff --git a/tests/testStorage.nim b/tests/testStorage.nim index 55b22d4b..30549b32 100644 --- a/tests/testStorage.nim +++ b/tests/testStorage.nim @@ -1,6 +1,6 @@ import std/os import ./imports -importTests(currentSourcePath().parentDir() / "storage", "", "") +importTests(currentSourcePath().parentDir() / "storage", "") {.warning[UnusedImport]: off.} From 892e2f6963a3feab7d3d2181476f616967458045 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Mon, 15 Jun 2026 09:41:40 +0400 Subject: [PATCH 127/167] Compatibility with docker --- tests/integration/nat/composehelper.nim | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/tests/integration/nat/composehelper.nim b/tests/integration/nat/composehelper.nim index bf8e69a0..3d51222a 100644 --- a/tests/integration/nat/composehelper.nim +++ b/tests/integration/nat/composehelper.nim @@ -5,8 +5,22 @@ import std/[os, osproc] import ../utils +proc composeCmd(composeFile: string): string = + ## Match the engine the Makefile builds the image with (podman first), so the + ## compose tool sees that image. + let base = + if findExe("podman-compose") != "": + "podman-compose" + elif findExe("podman") != "": + "podman compose" + elif findExe("docker") != "": + "docker compose" + else: + raise newException(IOError, "neither podman nor docker found") + base & " -f \"" & composeFile & "\"" + proc compose*(composeFile, action: string) = - let cmd = "podman-compose -f \"" & composeFile & "\" " & action + let cmd = composeCmd(composeFile) & " " & action doAssert execShellCmd(cmd) == 0, "command failed: " & cmd proc saveContainerLogs*( @@ -19,7 +33,7 @@ proc saveContainerLogs*( try: let logFile = getLogFile("", startTime, suiteName, testName, service) - cmd = "podman-compose -f \"" & composeFile & "\" logs " & service + cmd = composeCmd(composeFile) & " logs " & service (output, code) = execCmdEx(cmd) if code != 0: echo "warning: '", cmd, "' exited ", code From 37d8904d67111beae5c65ad5a0fc6e0734050f63 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Mon, 15 Jun 2026 09:41:55 +0400 Subject: [PATCH 128/167] Cleanup --- tests/integration/nat/node-entrypoint.sh | 3 ++- .../nat/not-reachable/testnotreachable.nim | 17 ++++++----------- .../nat/reachable/testreachable.nim | 19 ++++++------------- 3 files changed, 14 insertions(+), 25 deletions(-) diff --git a/tests/integration/nat/node-entrypoint.sh b/tests/integration/nat/node-entrypoint.sh index b8b08ce4..d8279976 100644 --- a/tests/integration/nat/node-entrypoint.sh +++ b/tests/integration/nat/node-entrypoint.sh @@ -23,4 +23,5 @@ exec /app/build/storage \ --bootstrap-node="$spr" \ --nat-num-peers-to-ask=1 --nat-max-queue-size=1 \ --nat-min-confidence=1.0 --nat-schedule-interval=30s \ - --data-dir=/data --log-level=DEBUG + --data-dir=/data --log-level=DEBUG \ + ${EXTRA_STORAGE_ARGS:-} diff --git a/tests/integration/nat/not-reachable/testnotreachable.nim b/tests/integration/nat/not-reachable/testnotreachable.nim index 74b83f2c..eefd47ab 100644 --- a/tests/integration/nat/not-reachable/testnotreachable.nim +++ b/tests/integration/nat/not-reachable/testnotreachable.nim @@ -14,24 +14,19 @@ import ../../storageclient import ../composehelper const - composeFile = currentSourcePath.parentDir / "compose.yml" - nodeApiUrl = "http://127.0.0.1:18080/api/storage/v1" - suiteName = "NAT not reachable" - testName = "node behind NAT is NotReachable and falls back to relay" - services = ["router", "bootstrap", "node"] detectTimeout = 300_000 # ms pollInterval = 5_000 # ms proc announcesCircuitAddr(info: JsonNode): bool = info{"announceAddresses"}.getElems.anyIt("p2p-circuit" in it.getStr) -asyncchecksuite suiteName: +asyncchecksuite "NAT not reachable": let - composeFile = composeFile - nodeApiUrl = nodeApiUrl - suiteName = suiteName - testName = testName - services = services + composeFile = currentSourcePath.parentDir / "compose.yml" + nodeApiUrl = "http://127.0.0.1:18080/api/storage/v1" + suiteName = "NAT not reachable" + testName = "node behind NAT is NotReachable and falls back to relay" + services = ["router", "bootstrap", "node"] startTime = now().format("yyyy-MM-dd'_'HH:mm:ss") var client: StorageClient diff --git a/tests/integration/nat/reachable/testreachable.nim b/tests/integration/nat/reachable/testreachable.nim index 93e26bdb..adefe598 100644 --- a/tests/integration/nat/reachable/testreachable.nim +++ b/tests/integration/nat/reachable/testreachable.nim @@ -20,11 +20,6 @@ import ../../storageclient import ../composehelper const - composeFile = currentSourcePath.parentDir / "compose.yml" - nodeApiUrl = "http://127.0.0.1:18081/api/storage/v1" - suiteName = "NAT reachable" - testName = "node behind NAT with a forwarded port is Reachable" - services = ["router", "bootstrap", "node"] detectTimeout = 120_000 # ms pollInterval = 5_000 # ms @@ -32,15 +27,13 @@ proc announcesDirectAddr(info: JsonNode): bool = ## A reachable node announces at least one direct (non-circuit) address. info{"announceAddresses"}.getElems.anyIt("p2p-circuit" notin it.getStr) -asyncchecksuite suiteName: - # chronos' async setup/teardown cannot reference module-level GC'ed consts - # (strings/seqs), so rebind the ones they use to suite locals. +asyncchecksuite "NAT reachable": let - composeFile = composeFile - nodeApiUrl = nodeApiUrl - suiteName = suiteName - testName = testName - services = services + composeFile = currentSourcePath.parentDir / "compose.yml" + nodeApiUrl = "http://127.0.0.1:18081/api/storage/v1" + suiteName = "NAT reachable" + testName = "node behind NAT with a forwarded port is Reachable" + services = ["router", "bootstrap", "node"] startTime = now().format("yyyy-MM-dd'_'HH:mm:ss") var client: StorageClient From 1692ae7df297a367953e1df9291bb3e74a713ba8 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Mon, 15 Jun 2026 10:15:59 +0400 Subject: [PATCH 129/167] Add UPnP tests --- tests/integration/nat/Dockerfile | 27 ++- tests/integration/nat/miniupnpd_stub_rdr.c | 176 ------------------ tests/integration/nat/upnp/README.md | 64 +++++++ tests/integration/nat/upnp/compose.yml | 95 ++++++++++ .../integration/nat/upnp/router-entrypoint.sh | 49 +++++ tests/integration/nat/upnp/testupnp.nim | 70 +++++++ 6 files changed, 291 insertions(+), 190 deletions(-) delete mode 100644 tests/integration/nat/miniupnpd_stub_rdr.c create mode 100644 tests/integration/nat/upnp/README.md create mode 100644 tests/integration/nat/upnp/compose.yml create mode 100644 tests/integration/nat/upnp/router-entrypoint.sh create mode 100644 tests/integration/nat/upnp/testupnp.nim diff --git a/tests/integration/nat/Dockerfile b/tests/integration/nat/Dockerfile index 82c35d3c..4d297c65 100644 --- a/tests/integration/nat/Dockerfile +++ b/tests/integration/nat/Dockerfile @@ -1,5 +1,5 @@ # One image for every podman NAT scenario, built as localhost/storage-nat. -# Carries the storage binary + miniupnpd (for the upnp/pmp routers); scenarios +# Carries the storage binary + miniupnpd (for the upnp/pcp routers); scenarios # differ only in their entrypoint scripts, which compose mounts. # Build context = project root. FROM ubuntu:24.04 @@ -8,22 +8,21 @@ ARG NIM_VERSION=2.2.10 RUN apt-get update && apt-get install -y --no-install-recommends \ gcc g++ make cmake git curl ca-certificates xz-utils \ - libc-dev ccache \ - iproute2 iptables jq \ + libc-dev ccache pkg-config \ + iproute2 iptables nftables jq \ + libnftnl-dev libmnl-dev \ && rm -rf /var/lib/apt/lists/* -# miniupnpd with a stub firewall backend: the real backend needs kernel caps a -# container lacks, so the stub makes mapping requests succeed without touching -# the kernel. Only the upnp/pmp routers use it. -COPY tests/integration/nat/miniupnpd_stub_rdr.c /tmp/stub_rdr.c +# miniupnpd with the real nftables backend (the iptables backend no longer builds +# against modern libiptc), used by the upnp/pcp routers: its mapping requests +# install a genuine DNAT on the router, so AutoNAT's dial-back reaches the node. RUN git clone --depth=1 --branch miniupnpd_2_3_9 \ - https://github.com/miniupnp/miniupnp.git /tmp/miniupnp \ - && cd /tmp/miniupnp/miniupnpd \ - && ./configure \ - && cp /tmp/stub_rdr.c . \ - && make NETFILTEROBJS=stub_rdr.o miniupnpd \ - && install -m 755 miniupnpd /usr/local/sbin/miniupnpd \ - && rm -rf /tmp/miniupnp /tmp/stub_rdr.c + https://github.com/miniupnp/miniupnp.git /tmp/miniupnp-nft \ + && cd /tmp/miniupnp-nft/miniupnpd \ + && ./configure --firewall=nftables \ + && make miniupnpd \ + && install -m 755 miniupnpd /usr/local/sbin/miniupnpd-nft \ + && rm -rf /tmp/miniupnp-nft RUN curl -fsSL "https://nim-lang.org/download/nim-${NIM_VERSION}-linux_x64.tar.xz" \ | tar -xJ -C /opt diff --git a/tests/integration/nat/miniupnpd_stub_rdr.c b/tests/integration/nat/miniupnpd_stub_rdr.c deleted file mode 100644 index 1e9be2b0..00000000 --- a/tests/integration/nat/miniupnpd_stub_rdr.c +++ /dev/null @@ -1,176 +0,0 @@ -/* Stub firewall backend for miniupnpd used in Docker-based tests. - * - * miniupnpd normally calls iptables/nftables to install port forwarding rules - * when it processes a UPnP/PCP/NAT-PMP mapping request. In a container - * those calls fail because the container lacks the required kernel capabilities, - * causing every mapping request to return an error to the client. - * - * This file replaces iptcrdr.o + iptpinhole.o + nfct_get.o with no-ops that - * always return success, so miniupnpd responds correctly to mapping requests - * without touching the kernel. */ - -#include -#include - -/* commonrdr.h interface */ - -int init_redirect(void) { return 0; } -void shutdown_redirect(void) {} - -int get_redirect_rule_count(const char *ifname) -{ (void)ifname; return 0; } - -int get_redirect_rule(const char *ifname, unsigned short eport, int proto, - char *iaddr, int iaddrlen, unsigned short *iport, - char *desc, int desclen, - char *rhost, int rhostlen, - unsigned int *timestamp, - uint64_t *packets, uint64_t *bytes) -{ (void)ifname; (void)eport; (void)proto; (void)iaddr; (void)iaddrlen; - (void)iport; (void)desc; (void)desclen; (void)rhost; (void)rhostlen; - (void)timestamp; (void)packets; (void)bytes; return -1; } - -int get_redirect_rule_by_index(int index, - char *ifname, unsigned short *eport, - char *iaddr, int iaddrlen, unsigned short *iport, - int *proto, char *desc, int desclen, - char *rhost, int rhostlen, - unsigned int *timestamp, - uint64_t *packets, uint64_t *bytes) -{ (void)index; (void)ifname; (void)eport; (void)iaddr; (void)iaddrlen; - (void)iport; (void)proto; (void)desc; (void)desclen; (void)rhost; - (void)rhostlen; (void)timestamp; (void)packets; (void)bytes; return -1; } - -unsigned short *get_portmappings_in_range(unsigned short startport, - unsigned short endport, - int proto, unsigned int *number) -{ (void)startport; (void)endport; (void)proto; *number = 0; return 0; } - -int update_portmapping(const char *ifname, unsigned short eport, int proto, - unsigned short iport, const char *desc, - unsigned int timestamp) -{ (void)ifname; (void)eport; (void)proto; (void)iport; (void)desc; - (void)timestamp; return 0; } - -int update_portmapping_desc_timestamp(const char *ifname, - unsigned short eport, int proto, - const char *desc, unsigned int timestamp) -{ (void)ifname; (void)eport; (void)proto; (void)desc; (void)timestamp; - return 0; } - -/* iptcrdr.h interface */ - -int add_redirect_rule2(const char *ifname, - const char *rhost, unsigned short eport, - const char *iaddr, unsigned short iport, int proto, - const char *desc, unsigned int timestamp) -{ (void)ifname; (void)rhost; (void)eport; (void)iaddr; (void)iport; - (void)proto; (void)desc; (void)timestamp; return 0; } - -int add_peer_redirect_rule2(const char *ifname, - const char *rhost, unsigned short rport, - const char *eaddr, unsigned short eport, - const char *iaddr, unsigned short iport, int proto, - const char *desc, unsigned int timestamp) -{ (void)ifname; (void)rhost; (void)rport; (void)eaddr; (void)eport; - (void)iaddr; (void)iport; (void)proto; (void)desc; (void)timestamp; - return 0; } - -int add_filter_rule2(const char *ifname, - const char *rhost, const char *iaddr, - unsigned short eport, unsigned short iport, - int proto, const char *desc) -{ (void)ifname; (void)rhost; (void)iaddr; (void)eport; (void)iport; - (void)proto; (void)desc; return 0; } - -int delete_redirect_and_filter_rules(unsigned short eport, int proto) -{ (void)eport; (void)proto; return 0; } - -int delete_filter_rule(const char *ifname, unsigned short port, int proto) -{ (void)ifname; (void)port; (void)proto; return 0; } - -int add_peer_dscp_rule2(const char *ifname, - const char *rhost, unsigned short rport, - unsigned char dscp, - const char *iaddr, unsigned short iport, int proto, - const char *desc, unsigned int timestamp) -{ (void)ifname; (void)rhost; (void)rport; (void)dscp; (void)iaddr; - (void)iport; (void)proto; (void)desc; (void)timestamp; return 0; } - -int get_peer_rule_by_index(int index, - char *ifname, unsigned short *eport, - char *iaddr, int iaddrlen, unsigned short *iport, - int *proto, char *desc, int desclen, - char *rhost, int rhostlen, unsigned short *rport, - unsigned int *timestamp, - uint64_t *packets, uint64_t *bytes) -{ (void)index; (void)ifname; (void)eport; (void)iaddr; (void)iaddrlen; - (void)iport; (void)proto; (void)desc; (void)desclen; (void)rhost; - (void)rhostlen; (void)rport; (void)timestamp; (void)packets; (void)bytes; - return -1; } - -int get_nat_redirect_rule(const char *nat_chain_name, const char *ifname, - unsigned short eport, int proto, - char *iaddr, int iaddrlen, unsigned short *iport, - char *desc, int desclen, - char *rhost, int rhostlen, - unsigned int *timestamp, - uint64_t *packets, uint64_t *bytes) -{ (void)nat_chain_name; (void)ifname; (void)eport; (void)proto; (void)iaddr; - (void)iaddrlen; (void)iport; (void)desc; (void)desclen; (void)rhost; - (void)rhostlen; (void)timestamp; (void)packets; (void)bytes; return -1; } - -int list_redirect_rule(const char *ifname) -{ (void)ifname; return 0; } - -/* commonrdr.h USE_NETFILTER interface */ - -int set_rdr_name(int param, const char *string) -{ (void)param; (void)string; return 0; } - -/* nfct_get.c interface */ - -int get_nat_ext_addr(struct sockaddr *src, struct sockaddr *dst, uint8_t proto, - struct sockaddr *ret_ext) -{ (void)src; (void)dst; (void)proto; (void)ret_ext; return -1; } - -/* iptpinhole.h interface */ - -int find_pinhole(const char *ifname, - const char *rem_host, unsigned short rem_port, - const char *int_client, unsigned short int_port, - int proto, char *desc, int desc_len, unsigned int *timestamp) -{ (void)ifname; (void)rem_host; (void)rem_port; (void)int_client; - (void)int_port; (void)proto; (void)desc; (void)desc_len; (void)timestamp; - return -1; } - -int add_pinhole(const char *ifname, - const char *rem_host, unsigned short rem_port, - const char *int_client, unsigned short int_port, - int proto, const char *desc, unsigned int timestamp) -{ (void)ifname; (void)rem_host; (void)rem_port; (void)int_client; - (void)int_port; (void)proto; (void)desc; (void)timestamp; return 0; } - -int update_pinhole(unsigned short uid, unsigned int timestamp) -{ (void)uid; (void)timestamp; return 0; } - -int delete_pinhole(unsigned short uid) -{ (void)uid; return 0; } - -int get_pinhole_info(unsigned short uid, - char *rem_host, int rem_hostlen, unsigned short *rem_port, - char *int_client, int int_clientlen, - unsigned short *int_port, - int *proto, char *desc, int desclen, - unsigned int *timestamp, - uint64_t *packets, uint64_t *bytes) -{ (void)uid; (void)rem_host; (void)rem_hostlen; (void)rem_port; - (void)int_client; (void)int_clientlen; (void)int_port; (void)proto; - (void)desc; (void)desclen; (void)timestamp; (void)packets; (void)bytes; - return -1; } - -int get_pinhole_uid_by_index(int index) -{ (void)index; return -1; } - -int clean_pinhole_list(unsigned int *next_timestamp) -{ (void)next_timestamp; return 0; } diff --git a/tests/integration/nat/upnp/README.md b/tests/integration/nat/upnp/README.md new file mode 100644 index 00000000..083316bf --- /dev/null +++ b/tests/integration/nat/upnp/README.md @@ -0,0 +1,64 @@ +# NAT upnp scenario + +## Scenario + +A node behind a NAT becomes `Reachable` by mapping its port over UPnP — the +router forwards nothing on its own, the node asks for the mapping and no relay is +needed. + +## Topology + +``` +node B ──── lan ──── router (NAT + miniupnpd) ──── wan ──── bootstrap A +``` + +- **bootstrap A** — public node on the wan, runs the relay + autonat server. +- **router** — `lan -> wan` masquerade and *no* static forward. It runs + `miniupnpd` (real nftables backend) as the UPnP gateway, with PCP/NAT-PMP + disabled so libplum falls back to UPnP. +- **node B** — `nat=auto`, on the lan. First detected `NotReachable`, it maps its + TCP listen (8070) and UDP disc (8090) ports over UPnP; the resulting DNAT lets + A's dial-back reach it, so the next AutoNAT round flips it to `Reachable`. + +The wan public range and `internal` flag work as in +[not-reachable](../not-reachable/README.md); the public wan IP also keeps +miniupnpd from treating the setup as double-NAT and refusing to forward. + +## Run + +Every NAT scenario: + +```bash +make testNatIntegration +``` + +Just this one — same `STORAGE_INTEGRATION_TEST_INCLUDES` filter as testIntegration, +with the test file path: + +```bash +make testNatIntegration \ + STORAGE_INTEGRATION_TEST_INCLUDES=tests/integration/nat/upnp/testupnp.nim +``` + +Builds the shared image and brings the compose topology up and down. Rootless, but +needs the host netfilter modules — if the router fails on iptables: +`sudo modprobe iptable_nat nf_conntrack`. + +## Expected result + +B ends up `Reachable`, the relay not running, announcing its direct address with +an active UPnP mapping. Its `debug/info`: + +```json +{ + "nat": { + "reachability": "Reachable", + "clientMode": false, + "relayRunning": false, + "portMapping": "upnp" + } +} +``` + +Per-run container logs (router, bootstrap, node) are written before teardown to +`tests/integration/logs/__NAT_upnp//.log`. diff --git a/tests/integration/nat/upnp/compose.yml b/tests/integration/nat/upnp/compose.yml new file mode 100644 index 00000000..50ca1df8 --- /dev/null +++ b/tests/integration/nat/upnp/compose.yml @@ -0,0 +1,95 @@ +# Same NAT topology as not-reachable, but the router runs miniupnpd. The node +# maps its port over UPnP, which installs a real DNAT on the router, so AutoNAT's +# dial-back reaches it and it is detected Reachable — no relay. Run via testupnp.nim. +# +# node B ──── lan ──── router (NAT + miniupnpd) ──── wan ──── bootstrap A +name: nat-upnp + +# Topology addresses, named for their role (defined once, referenced below). +x-addresses: + # fake public internet; a routable range so B looks public to A + wan_subnet: &wan_subnet 7.7.7.0/24 + # private network behind the NAT + lan_subnet: &lan_subnet 10.99.0.0/24 + # A: public bootstrap, relay + autonat server + bootstrap_ip: &bootstrap_ip 7.7.7.10 + # router's public face + router_wan_ip: &router_wan_ip 7.7.7.2 + # router's private face = B's gateway, and the UPnP control point + router_lan_ip: &router_lan_ip 10.99.0.2 + # B, behind the NAT + node_ip: &node_ip 10.99.0.10 + +networks: + wan: + # Keep the fake public range private, not exposed to the host + internal: true + ipam: + config: + - subnet: *wan_subnet + lan: + ipam: + config: + - subnet: *lan_subnet + +services: + router: + image: localhost/storage-nat + cap_add: [NET_ADMIN] + sysctls: + net.ipv4.ip_forward: 1 + networks: + wan: + ipv4_address: *router_wan_ip + lan: + ipv4_address: *router_lan_ip + environment: + ROUTER_WAN_IP: *router_wan_ip + ROUTER_LAN_IP: *router_lan_ip + LAN_SUBNET: *lan_subnet + # scripts mounted, so editing them needs no image rebuild + volumes: + - ../router-common.sh:/scripts/router-common.sh:ro,z + - ./router-entrypoint.sh:/scripts/router-entrypoint.sh:ro,z + entrypoint: ["bash", "/scripts/router-entrypoint.sh"] + + bootstrap: + image: localhost/storage-nat + networks: + wan: + ipv4_address: *bootstrap_ip + entrypoint: ["/app/build/storage"] + command: + - --listen-ip=0.0.0.0 + - --api-bindaddr=0.0.0.0 + - --listen-port=8070 + - --disc-port=8090 + - --api-port=8080 + # bootstrap_ip (anchors can't go inside a string) + - --nat=extip:7.7.7.10 + - --relay-server + - --autonat-server + - --no-bootstrap-node + - --data-dir=/data + - --log-level=DEBUG + + node: + image: localhost/storage-nat + cap_add: [NET_ADMIN] + depends_on: [router, bootstrap] + networks: + lan: + ipv4_address: *node_ip + # B's API, published so the test can poll it + ports: + - "127.0.0.1:18082:8080" + environment: + ROUTER_LAN_IP: *router_lan_ip + # B fetches A's SPR from this API at startup to join the network (bootstrap_ip) + BOOTSTRAP_API: http://7.7.7.10:8080 + EXTRA_STORAGE_ARGS: >- + --nat-port-mapping-discover-timeout=5000 + --nat-port-mapping-timeout=5000 + volumes: + - ../node-entrypoint.sh:/scripts/node-entrypoint.sh:ro,z + entrypoint: ["bash", "/scripts/node-entrypoint.sh"] diff --git a/tests/integration/nat/upnp/router-entrypoint.sh b/tests/integration/nat/upnp/router-entrypoint.sh new file mode 100644 index 00000000..64b66598 --- /dev/null +++ b/tests/integration/nat/upnp/router-entrypoint.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +source "$(dirname "$0")/router-common.sh" + +# miniupnpd listens for UPnP (SSDP) on the lan face, so find it by its IP like +# router-common.sh finds the wan one. +lanif=$(ip -o -4 addr show | awk -v ip="$ROUTER_LAN_IP" '$0 ~ ip {print $2; exit}') + +# Reuse miniupnpd's chains (as nft_init.sh sets them up) without its forward drop +# policy. +nft -f - <<'EOF' +table inet filter { + chain prerouting_miniupnpd {} + chain postrouting_miniupnpd {} + chain miniupnpd {} + chain prerouting { + type nat hook prerouting priority -100; policy accept; + jump prerouting_miniupnpd + } + chain postrouting { + type nat hook postrouting priority 100; policy accept; + jump postrouting_miniupnpd + } +} +EOF + +conf=/tmp/miniupnpd.conf +cat > "$conf" </dev/null \ + || { echo "ERROR: miniupnpd failed to start" >&2; exit 1; } + +echo "router ready (wan iface $wanif, miniupnpd on $lanif)" + +hold_until_stopped diff --git a/tests/integration/nat/upnp/testupnp.nim b/tests/integration/nat/upnp/testupnp.nim new file mode 100644 index 00000000..e22e812d --- /dev/null +++ b/tests/integration/nat/upnp/testupnp.nim @@ -0,0 +1,70 @@ +## NAT upnp scenario — node behind a real NAT becomes Reachable by mapping its +## port over UPnP. +## +## Same shape as the reachable test, but the router opens no port itself: it runs +## miniupnpd and the node maps its TCP/UDP ports via UPnP, which installs a real +## DNAT on the router. AutoNAT's dial-back then reaches the node, so it is +## detected Reachable with an active UPnP mapping — no relay. +## +## Requires podman-compose and the scenario image: +## podman build -t localhost/storage-nat -f tests/integration/nat/Dockerfile . + +import std/[json, os, sequtils, strutils, times] +import pkg/chronos +import pkg/questionable/results + +import ../../../asynctest +import ../../../checktest +import ../../storageclient +import ../composehelper + +const + detectTimeout = 300_000 # ms + pollInterval = 5_000 # ms + +proc announcesDirectAddr(info: JsonNode): bool = + ## A reachable node announces at least one direct (non-circuit) address. + info{"announceAddresses"}.getElems.anyIt("p2p-circuit" notin it.getStr) + +asyncchecksuite "NAT upnp": + let + composeFile = currentSourcePath.parentDir / "compose.yml" + nodeApiUrl = "http://127.0.0.1:18082/api/storage/v1" + suiteName = "NAT upnp" + testName = "node behind NAT maps its port over UPnP and is Reachable" + services = ["router", "bootstrap", "node"] + startTime = now().format("yyyy-MM-dd'_'HH:mm:ss") + var client: StorageClient + + setup: + compose(composeFile, "up -d") + client = StorageClient.new(nodeApiUrl) + + teardown: + await client.close() + saveContainerLogs(composeFile, suiteName, testName, startTime, services) + compose(composeFile, "down -v") + + test testName: + # Reachable is the settling signal: wait for it, then assert each expected + # property separately so a failure points at the exact condition. + check eventuallySafe( + block: + var reachable = false + try: + let info = await client.info() + reachable = + info.isOk and info.get{"nat"}{"reachability"}.getStr == "Reachable" + except HttpError: + discard # B's API is not up yet, keep polling + reachable, + timeout = detectTimeout, + pollInterval = pollInterval, + ) + + let info = (await client.info()).get + let nat = info{"nat"} + check nat{"reachability"}.getStr == "Reachable" + check nat{"relayRunning"}.getBool == false + check nat{"portMapping"}.getStr == "upnp" + check info.announcesDirectAddr() From cc6135b1a11016398be3debbce40f526828a80ac Mon Sep 17 00:00:00 2001 From: Arnaud Date: Mon, 15 Jun 2026 10:16:08 +0400 Subject: [PATCH 130/167] Add PCP tests --- tests/integration/nat/pcp/README.md | 64 +++++++++++++ tests/integration/nat/pcp/compose.yml | 96 +++++++++++++++++++ .../integration/nat/pcp/router-entrypoint.sh | 48 ++++++++++ tests/integration/nat/pcp/testpcp.nim | 70 ++++++++++++++ 4 files changed, 278 insertions(+) create mode 100644 tests/integration/nat/pcp/README.md create mode 100644 tests/integration/nat/pcp/compose.yml create mode 100644 tests/integration/nat/pcp/router-entrypoint.sh create mode 100644 tests/integration/nat/pcp/testpcp.nim diff --git a/tests/integration/nat/pcp/README.md b/tests/integration/nat/pcp/README.md new file mode 100644 index 00000000..848156e4 --- /dev/null +++ b/tests/integration/nat/pcp/README.md @@ -0,0 +1,64 @@ +# NAT pcp scenario + +## Scenario + +A node behind a NAT becomes `Reachable` by mapping its port over PCP — the router +forwards nothing on its own, the node asks for the mapping and no relay is needed. + +## Topology + +``` +node B ──── lan ──── router (NAT + miniupnpd/PCP) ──── wan ──── bootstrap A +``` + +- **bootstrap A** — public node on the wan, runs the relay + autonat server. +- **router** — `lan -> wan` masquerade and *no* static forward. It runs + `miniupnpd` (real nftables backend) with PCP/NAT-PMP enabled. libplum tries PCP + first, so the mapping request goes over PCP and installs a real DNAT into the + nft chains the entrypoint pre-creates. +- **node B** — `nat=auto`, on the lan. First detected `NotReachable`, it maps its + TCP listen (8070) and UDP disc (8090) ports over PCP; the resulting DNAT lets + A's dial-back reach it, so the next AutoNAT round flips it to `Reachable`. + +The wan public range and `internal` flag work as in +[not-reachable](../not-reachable/README.md); the public wan IP also keeps +miniupnpd from refusing PCP/NAT-PMP as double-NAT. + +## Run + +Every NAT scenario: + +```bash +make testNatIntegration +``` + +Just this one — same `STORAGE_INTEGRATION_TEST_INCLUDES` filter as testIntegration, +with the test file path: + +```bash +make testNatIntegration \ + STORAGE_INTEGRATION_TEST_INCLUDES=tests/integration/nat/pcp/testpcp.nim +``` + +Builds the shared image and brings the compose topology up and down. Rootless, but +needs the host netfilter modules — if the router fails on iptables: +`sudo modprobe iptable_nat nf_conntrack`. + +## Expected result + +B ends up `Reachable`, the relay not running, announcing its direct address with +an active PCP mapping. Its `debug/info`: + +```json +{ + "nat": { + "reachability": "Reachable", + "clientMode": false, + "relayRunning": false, + "portMapping": "pcp" + } +} +``` + +Per-run container logs (router, bootstrap, node) are written before teardown to +`tests/integration/logs/__NAT_pcp//.log`. diff --git a/tests/integration/nat/pcp/compose.yml b/tests/integration/nat/pcp/compose.yml new file mode 100644 index 00000000..1cf8d43e --- /dev/null +++ b/tests/integration/nat/pcp/compose.yml @@ -0,0 +1,96 @@ +# Same NAT topology as upnp, but miniupnpd has PCP enabled and the node maps its +# port over PCP (libplum's preferred protocol), which installs a real DNAT on the +# router, so AutoNAT's dial-back reaches it and it is detected Reachable — no +# relay. Run via testpcp.nim. +# +# node B ──── lan ──── router (NAT + miniupnpd/PCP) ──── wan ──── bootstrap A +name: nat-pcp + +# Topology addresses, named for their role (defined once, referenced below). +x-addresses: + # fake public internet; a routable range so B looks public to A + wan_subnet: &wan_subnet 7.7.7.0/24 + # private network behind the NAT + lan_subnet: &lan_subnet 10.99.0.0/24 + # A: public bootstrap, relay + autonat server + bootstrap_ip: &bootstrap_ip 7.7.7.10 + # router's public face + router_wan_ip: &router_wan_ip 7.7.7.2 + # router's private face = B's gateway, and the PCP/NAT-PMP control point + router_lan_ip: &router_lan_ip 10.99.0.2 + # B, behind the NAT + node_ip: &node_ip 10.99.0.10 + +networks: + wan: + # Keep the fake public range private, not exposed to the host + internal: true + ipam: + config: + - subnet: *wan_subnet + lan: + ipam: + config: + - subnet: *lan_subnet + +services: + router: + image: localhost/storage-nat + cap_add: [NET_ADMIN] + sysctls: + net.ipv4.ip_forward: 1 + networks: + wan: + ipv4_address: *router_wan_ip + lan: + ipv4_address: *router_lan_ip + environment: + ROUTER_WAN_IP: *router_wan_ip + ROUTER_LAN_IP: *router_lan_ip + LAN_SUBNET: *lan_subnet + # scripts mounted, not baked, so editing them needs no image rebuild + volumes: + - ../router-common.sh:/scripts/router-common.sh:ro,z + - ./router-entrypoint.sh:/scripts/router-entrypoint.sh:ro,z + entrypoint: ["bash", "/scripts/router-entrypoint.sh"] + + bootstrap: + image: localhost/storage-nat + networks: + wan: + ipv4_address: *bootstrap_ip + entrypoint: ["/app/build/storage"] + command: + - --listen-ip=0.0.0.0 + - --api-bindaddr=0.0.0.0 + - --listen-port=8070 + - --disc-port=8090 + - --api-port=8080 + # bootstrap_ip (anchors can't go inside a string) + - --nat=extip:7.7.7.10 + - --relay-server + - --autonat-server + - --no-bootstrap-node + - --data-dir=/data + - --log-level=DEBUG + + node: + image: localhost/storage-nat + cap_add: [NET_ADMIN] + depends_on: [router, bootstrap] + networks: + lan: + ipv4_address: *node_ip + # B's API, published so the test can poll it + ports: + - "127.0.0.1:18083:8080" + environment: + ROUTER_LAN_IP: *router_lan_ip + # B fetches A's SPR from this API at startup to join the network (bootstrap_ip) + BOOTSTRAP_API: http://7.7.7.10:8080 + EXTRA_STORAGE_ARGS: >- + --nat-port-mapping-discover-timeout=5000 + --nat-port-mapping-timeout=5000 + volumes: + - ../node-entrypoint.sh:/scripts/node-entrypoint.sh:ro,z + entrypoint: ["bash", "/scripts/node-entrypoint.sh"] diff --git a/tests/integration/nat/pcp/router-entrypoint.sh b/tests/integration/nat/pcp/router-entrypoint.sh new file mode 100644 index 00000000..257ce6cb --- /dev/null +++ b/tests/integration/nat/pcp/router-entrypoint.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + +source "$(dirname "$0")/router-common.sh" + +# miniupnpd serves PCP/NAT-PMP (UDP 5351) on the lan face, so find it by its IP +# like router-common.sh finds the wan one. +lanif=$(ip -o -4 addr show | awk -v ip="$ROUTER_LAN_IP" '$0 ~ ip {print $2; exit}') + +# Reuse miniupnpd's chains (as nft_init.sh sets them up) without its forward drop +# policy. +nft -f - <<'EOF' +table inet filter { + chain prerouting_miniupnpd {} + chain postrouting_miniupnpd {} + chain miniupnpd {} + chain prerouting { + type nat hook prerouting priority -100; policy accept; + jump prerouting_miniupnpd + } + chain postrouting { + type nat hook postrouting priority 100; policy accept; + jump postrouting_miniupnpd + } +} +EOF + +conf=/tmp/miniupnpd.conf +cat > "$conf" </dev/null \ + || { echo "ERROR: miniupnpd failed to start" >&2; exit 1; } + +echo "router ready (wan iface $wanif, miniupnpd on $lanif)" + +hold_until_stopped diff --git a/tests/integration/nat/pcp/testpcp.nim b/tests/integration/nat/pcp/testpcp.nim new file mode 100644 index 00000000..2785f6d6 --- /dev/null +++ b/tests/integration/nat/pcp/testpcp.nim @@ -0,0 +1,70 @@ +## NAT pcp scenario — node behind a real NAT becomes Reachable by mapping its +## port over PCP. +## +## Same shape as the upnp test, but miniupnpd has PCP enabled and the node maps +## its TCP/UDP ports via PCP (libplum's preferred protocol), which installs a real +## DNAT on the router. AutoNAT's dial-back then reaches the node, so it is +## detected Reachable with an active PCP mapping — no relay. +## +## Requires podman-compose and the scenario image: +## podman build -t localhost/storage-nat -f tests/integration/nat/Dockerfile . + +import std/[json, os, sequtils, strutils, times] +import pkg/chronos +import pkg/questionable/results + +import ../../../asynctest +import ../../../checktest +import ../../storageclient +import ../composehelper + +const + detectTimeout = 300_000 # ms + pollInterval = 5_000 # ms + +proc announcesDirectAddr(info: JsonNode): bool = + ## A reachable node announces at least one direct (non-circuit) address. + info{"announceAddresses"}.getElems.anyIt("p2p-circuit" notin it.getStr) + +asyncchecksuite "NAT pcp": + let + composeFile = currentSourcePath.parentDir / "compose.yml" + nodeApiUrl = "http://127.0.0.1:18083/api/storage/v1" + suiteName = "NAT pcp" + testName = "node behind NAT maps its port over PCP and is Reachable" + services = ["router", "bootstrap", "node"] + startTime = now().format("yyyy-MM-dd'_'HH:mm:ss") + var client: StorageClient + + setup: + compose(composeFile, "up -d") + client = StorageClient.new(nodeApiUrl) + + teardown: + await client.close() + saveContainerLogs(composeFile, suiteName, testName, startTime, services) + compose(composeFile, "down -v") + + test testName: + # Reachable is the settling signal: wait for it, then assert each expected + # property separately so a failure points at the exact condition. + check eventuallySafe( + block: + var reachable = false + try: + let info = await client.info() + reachable = + info.isOk and info.get{"nat"}{"reachability"}.getStr == "Reachable" + except HttpError: + discard # B's API is not up yet, keep polling + reachable, + timeout = detectTimeout, + pollInterval = pollInterval, + ) + + let info = (await client.info()).get + let nat = info{"nat"} + check nat{"reachability"}.getStr == "Reachable" + check nat{"relayRunning"}.getBool == false + check nat{"portMapping"}.getStr == "pcp" + check info.announcesDirectAddr() From b7ee215c326e12664e0e56d91c7d4410455361fb Mon Sep 17 00:00:00 2001 From: Arnaud Date: Mon, 15 Jun 2026 11:32:26 +0400 Subject: [PATCH 131/167] Fix duplicate addresses in libp2p --- storage/nat.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/nat.nim b/storage/nat.nim index da65e8b5..ac7495ba 100644 --- a/storage/nat.nim +++ b/storage/nat.nim @@ -138,7 +138,7 @@ proc announcePeerInfoAddrs*(discovery: Discovery, peerInfo: PeerInfo, udpPort: P ## they are announced via onReservation and must not enter the DHT routing ## record. No-op when the addresses are already announced, so peerInfo ## updates that only touch filtered-out addresses do not re-announce. - let addrs = peerInfo.addrs.filterIt(not it.isCircuitRelayMA()) + let addrs = peerInfo.addrs.filterIt(not it.isCircuitRelayMA()).deduplicate() if addrs.len == 0 or addrs == discovery.announceAddrs: return discovery.announceDirectAddrs(addrs, udpPort = udpPort) From 1ec065113952148205ce1eadaafd0c40366e8368 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Mon, 15 Jun 2026 11:36:56 +0400 Subject: [PATCH 132/167] Retry port mapping on NotReachable --- storage/nat.nim | 7 ------- 1 file changed, 7 deletions(-) diff --git a/storage/nat.nim b/storage/nat.nim index ac7495ba..d7c1be54 100644 --- a/storage/nat.nim +++ b/storage/nat.nim @@ -211,13 +211,6 @@ method handleNatStatus*( # We remove the announced records. # Eventually, it will we updated by the relay when it started discovery.announceDirectAddrs(@[], udpPort = discoveryPort) - elif autoRelayService.isRunning: - # The mapping was already tried and did not make the node reachable. - # If the relay is running, there is nothing to do. - # We do not want to retry the port mapping if it failed already, - # it would stop the relay service while there is little chance to have - # a Reachable status after it was detected Not Reachable the first time. - discard else: debug "Node is not reachable trying port mapping now" From fdf5396e60a5716086961290871f91112e1d267a Mon Sep 17 00:00:00 2001 From: Arnaud Date: Mon, 15 Jun 2026 14:55:04 +0400 Subject: [PATCH 133/167] Add custom address mapper to remap the port mapping by UPnP / PCP --- storage/nat.nim | 28 ++++++++++++ storage/storage.nim | 2 + tests/storage/testnatdetection.nim | 71 ++++++++++++++++++++++++++++++ tests/storage/testnatreaction.nim | 34 ++++++++++++++ 4 files changed, 135 insertions(+) diff --git a/storage/nat.nim b/storage/nat.nim index d7c1be54..1c65dbc7 100644 --- a/storage/nat.nim +++ b/storage/nat.nim @@ -165,6 +165,34 @@ proc setupPeerInfoObserver*( switch.peerInfo.addObserver(observer) observer +proc setupMappedAddrMapper*(switch: Switch, natMapper: NatPortMapper) = + ## We define a custom mapper that adds the external port to peerInfo.addrs when + ## a port mapping is active, so AutoNAT tests that port. + ## PCP/NAT-PMP may grant an external port different from the listen port. + let mapper: AddressMapper = proc( + addrs: seq[MultiAddress] + ): Future[seq[MultiAddress]] {.gcsafe, async: (raises: [CancelledError]).} = + result = addrs + + if natMapper.activeTcpPort.isNone: + return result + + let mappedPort = natMapper.activeTcpPort.get + for listenAddr in switch.peerInfo.listenAddrs: + # Dialable IP (observed public, or the listen IP if already public) + # used with the mapped port. + let mappedAddr = switch.peerStore.guessDialableAddr(listenAddr).remapAddr( + port = some(mappedPort) + ) + if mappedAddr.isPublicMA(): + # Insert first so AutoNAT dials it before the listen-port candidate (the + # server tests only the first dialable address). + result.insert(mappedAddr, 0) + + return result.deduplicate() + + switch.peerInfo.addressMappers.add(mapper) + method handleNatStatus*( m: NatPortMapper, networkReachability: NetworkReachability, diff --git a/storage/storage.nim b/storage/storage.nim index aa0ddc84..8b623de5 100644 --- a/storage/storage.nim +++ b/storage/storage.nim @@ -490,6 +490,8 @@ proc new*( peerInfoObserver = some(setupPeerInfoObserver(switch, autonatService.get, discovery, natMapper.get)) + setupMappedAddrMapper(switch, natMapper.get) + autonatService.get.setStatusAndConfidenceHandler( proc( networkReachability: NetworkReachability, diff --git a/tests/storage/testnatdetection.nim b/tests/storage/testnatdetection.nim index e6c73d89..0673be44 100644 --- a/tests/storage/testnatdetection.nim +++ b/tests/storage/testnatdetection.nim @@ -30,6 +30,8 @@ const discoveryPort = Port(8090) # ms — AutoNAT probe + confidence + reaction detectTimeout = 20000 + mockMappedTcpPort = Port(40000) + mockMappedUdpPort = Port(40001) type MockNatPortMapper = ref object of NatPortMapper @@ -40,6 +42,19 @@ method mapNatPorts*( .} = none((Port, Port, MappingProtocol)) +# Simulates a successful PCP mapping +type MockMappingNatPortMapper = ref object of NatPortMapper + +method mapNatPorts*( + m: MockMappingNatPortMapper +): Future[Option[(Port, Port, MappingProtocol)]] {. + async: (raises: [CancelledError]), gcsafe +.} = + m.activeTcpPort = some(mockMappedTcpPort) + m.activeUdpPort = some(mockMappedUdpPort) + m.activeMappingProtocol = some(MappingProtocol.PCP) + some((mockMappedTcpPort, mockMappedUdpPort, MappingProtocol.PCP)) + # Captures the candidate addresses the service sends and answers Reachable, so # the service flips to reachable and runs its address mapper — without dialing. type MockAutonatV2Client = ref object of AutonatV2Client @@ -241,3 +256,59 @@ asyncchecksuite "NAT detection - dial request candidates": await autonat.stop(sw) await sw2.stop() + + test "after a port mapping, the mapped address is AutoNAT's first dial candidate": + let mapper = MockMappingNatPortMapper() + + setupMappedAddrMapper(sw, mapper) + + # Reach the observation quorum so guessDialableAddr trusts 8.8.8.8 + let observed = MultiAddress.init("/ip4/8.8.8.8/tcp/4001").expect("valid") + let quorum = 3 + for _ in 0 ..< quorum: + discard sw.peerStore.identify.observedAddrManager.addObservation(observed) + + # Setup AutoRelayService + let relay = AutoRelayService.new( + 1, relayClientModule.RelayClient.new(), nil, Rng.instance().libp2pRng + ) + autorelayservice.setup(relay, sw) + + # Define our handleNatStatus callback + let disc = Discovery.new( + PrivateKey.random(Rng.instance().libp2pRng).get(), announceAddrs = @[] + ) + let dialBack = MultiAddress.init("/ip4/8.8.8.8/tcp/8080").expect("valid") + await mapper.handleNatStatus( + NotReachable, Opt.some(dialBack), discoveryPort, disc, sw, relay + ) + + # Define our AutonatV2Service + let mockClient = MockAutonatV2Client() + let autonat = AutonatV2Service.new( + Rng.instance().libp2pRng, + mockClient, + AutonatV2ServiceConfig.new( + enableDialableCandidates = true, maxQueueSize = 1, minConfidence = 0.5 + ), + ) + service.setup(autonat, sw) + await autonat.start(sw) + + # Connect to a second switch to test NAT detection + let sw2 = newStandardSwitch() + await sw2.start() + await sw.connect(sw2.peerInfo.peerId, sw2.peerInfo.addrs) + + # The expected mapped address should be the guessDialableAddr (8.8.8.8) + # using the mapping mocked port (40000) because a mapping was created. + let mapped = + MultiAddress.init("/ip4/8.8.8.8/tcp/" & $mockMappedTcpPort).expect("valid") + check eventually(mapped in mockClient.reqAddrs) + # Ensute that it comes first (because AutonatV2 test only the first candidate) + check mockClient.reqAddrs[0] == mapped + + await autonat.stop(sw) + await sw2.stop() + if relay.isRunning: + await relay.stop(sw) diff --git a/tests/storage/testnatreaction.nim b/tests/storage/testnatreaction.nim index 6607c1a0..468a3267 100644 --- a/tests/storage/testnatreaction.nim +++ b/tests/storage/testnatreaction.nim @@ -5,6 +5,7 @@ import pkg/libp2p/protocols/connectivity/autonat/types import pkg/libp2p/protocols/connectivity/autonatv2/service except setup import pkg/libp2p/protocols/connectivity/relay/client as relayClientModule import pkg/libp2p/services/autorelayservice except setup +import pkg/libp2p/observedaddrmanager import pkg/results import ./helpers @@ -223,3 +224,36 @@ asyncchecksuite "NAT reaction - address announcing": await sw.peerInfo.update() check disc.announceAddrs == newSeq[MultiAddress]() + + test "mapped-addr mapper injects the mapped port as the first candidate": + const mockMappedTcpPort = 40000 + + setupMappedAddrMapper( + sw, NatPortMapper(activeTcpPort: some(Port(mockMappedTcpPort))) + ) + + # Reach the observation quorum so guessDialableAddr trusts 8.8.8.8 + let observed = MultiAddress.init("/ip4/8.8.8.8/tcp/4001").expect("valid") + let quorum = 3 + for _ in 0 ..< quorum: + discard sw.peerStore.identify.observedAddrManager.addObservation(observed) + + await sw.peerInfo.update() + + # Ensure that the address mapper injects the mapped port as the first candidate + # after peer info update + check sw.peerInfo.addrs[0] == + MultiAddress.init("/ip4/8.8.8.8/tcp/" & $mockMappedTcpPort).expect("valid") + + test "mapped-addr mapper is a no-op without an active mapping": + setupMappedAddrMapper(sw, NatPortMapper()) + + let observed = MultiAddress.init("/ip4/8.8.8.8/tcp/4001").expect("valid") + let quorum = 3 + for _ in 0 ..< quorum: + discard sw.peerStore.identify.observedAddrManager.addObservation(observed) + + await sw.peerInfo.update() + + # Ensure that nothing is injected because there is no active mapping + check sw.peerInfo.addrs == sw.peerInfo.listenAddrs From 37c4ee5e874beaa6c926058e817341ed05843024 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Mon, 15 Jun 2026 15:07:08 +0400 Subject: [PATCH 134/167] Cleanup --- storage/nat.nim | 25 +++++++------------------ 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/storage/nat.nim b/storage/nat.nim index 1c65dbc7..11aa4961 100644 --- a/storage/nat.nim +++ b/storage/nat.nim @@ -222,23 +222,22 @@ method handleNatStatus*( discovery.protocol.clientMode = true + if not autoRelayService.isRunning: + # Remove any announced addresses, they will be replaced. + # If the relay is running, the addresses will be updated on reservation. + discovery.announceDirectAddrs(@[], udpPort = discoveryPort) + if dialBackAddr.isNone: warn "Got empty dialback address in AutoNat when node is NotReachable" if m.hasMappingIds(): m.close() - - discovery.announceDirectAddrs(@[], udpPort = discoveryPort) elif m.hasMappingIds(): warn "Not Reachable with active port mapping. The port mapping will be deleted and relay will start." # The mapping was created the the node is still not reachable. # In that case, we delete the mapping and relay will start. m.close() - - # We remove the announced records. - # Eventually, it will we updated by the relay when it started - discovery.announceDirectAddrs(@[], udpPort = discoveryPort) else: debug "Node is not reachable trying port mapping now" @@ -249,19 +248,9 @@ method handleNatStatus*( info "Port mapping created successfully", tcpPort, udpPort, protocol - let announceAddress = dialBackAddr.get.remapAddr(port = some(tcpPort)) + # The address mapper will use the mapped ports map the addresses for + # libp2p. - if autoRelayService.isRunning: - # Here we stop the relay because the node *should* be reachable - await autoRelayService.stop(switch) - debug "AutoRelayService stopped" - - # Note that we update the DHT records but we don't set the client mode - # to false because we are not sure the node is reachable. - # The client mode will be updated on the next iteration of autonat. - # Trying to check manually that the node is reachable is not trivial, - # this is exactly what Autonat is for. - discovery.announceDirectAddrs(@[announceAddress], udpPort = udpPort) hasPortMapping = true else: # In case of failure, close the port mapping in order to rerun discover From d33878e214e7d55c30bb45fabee74f24e4484baa Mon Sep 17 00:00:00 2001 From: Arnaud Date: Mon, 15 Jun 2026 17:39:49 +0400 Subject: [PATCH 135/167] Simplify the address announcement --- storage/nat.nim | 40 +++-------------- storage/storage.nim | 9 ---- tests/storage/testnatreaction.nim | 73 ++++++++++--------------------- 3 files changed, 28 insertions(+), 94 deletions(-) diff --git a/storage/nat.nim b/storage/nat.nim index 11aa4961..a7e76389 100644 --- a/storage/nat.nim +++ b/storage/nat.nim @@ -133,38 +133,6 @@ method hasMappingIds*(m: NatPortMapper): bool {.base, gcsafe.} = # (use hasMapping() for liveness check). m.tcpMappingId.isSome and m.udpMappingId.isSome -proc announcePeerInfoAddrs*(discovery: Discovery, peerInfo: PeerInfo, udpPort: Port) = - ## Announces peerInfo.addrs to the DHT, excluding relay circuit addresses: - ## they are announced via onReservation and must not enter the DHT routing - ## record. No-op when the addresses are already announced, so peerInfo - ## updates that only touch filtered-out addresses do not re-announce. - let addrs = peerInfo.addrs.filterIt(not it.isCircuitRelayMA()).deduplicate() - if addrs.len == 0 or addrs == discovery.announceAddrs: - return - discovery.announceDirectAddrs(addrs, udpPort = udpPort) - -proc setupPeerInfoObserver*( - switch: Switch, - autonat: AutonatV2Service, - discovery: Discovery, - natMapper: NatPortMapper, -): PeerInfoObserver = - ## AutoNAT's address mapper resolves peerInfo.addrs into public addresses - ## once the node is Reachable; peerInfo.update() then notifies observers. - ## Keep the DHT records in sync with what libp2p announces. - let observer: PeerInfoObserver = proc(peerInfo: PeerInfo) {.gcsafe, raises: [].} = - info "PeerInfo updated", - addrs = peerInfo.addrs, reachability = autonat.networkReachability - if autonat.networkReachability != NetworkReachability.Reachable: - return - # When a NAT mapping is active, announce its external UDP port: the router - # may have assigned a different port than the requested discoveryPort. - let udpPort = natMapper.activeUdpPort.get(natMapper.discoveryPort) - announcePeerInfoAddrs(discovery, peerInfo, udpPort) - - switch.peerInfo.addObserver(observer) - observer - proc setupMappedAddrMapper*(switch: Switch, natMapper: NatPortMapper) = ## We define a custom mapper that adds the external port to peerInfo.addrs when ## a port mapping is active, so AutoNAT tests that port. @@ -213,10 +181,12 @@ method handleNatStatus*( await autoRelayService.stop(switch) debug "AutoRelayService stopped" - # No announce here: AutoNAT refreshes peerInfo right after this handler, - # its address mapper (active now that the node is Reachable) resolves the - # public addresses and the peerInfo observer announces them. discovery.protocol.clientMode = false + + if dialBackAddr.isSome: + discovery.announceDirectAddrs( + @[dialBackAddr.get], udpPort = m.activeUdpPort.get(discoveryPort) + ) of NotReachable: var hasPortMapping = false diff --git a/storage/storage.nim b/storage/storage.nim index 8b623de5..a57705ea 100644 --- a/storage/storage.nim +++ b/storage/storage.nim @@ -67,7 +67,6 @@ type autoRelayService*: Option[AutoRelayService] natMapper*: Option[NatPortMapper] holePunchHandler: Option[connmanager.PeerEventHandler] - peerInfoObserver: Option[PeerInfoObserver] bootstrapNodes: seq[SignedPeerRecord] isStarted: bool @@ -170,9 +169,6 @@ proc stop*(s: StorageServer) {.async.} = s.holePunchHandler.get, PeerEventKind.Joined ) - if s.peerInfoObserver.isSome: - s.storageNode.switch.peerInfo.removeObserver(s.peerInfoObserver.get) - var futures = @[ s.storageNode.switch.stop(), s.storageNode.stop(), @@ -451,7 +447,6 @@ proc new*( var natMapper: Option[NatPortMapper] var autoRelayService: Option[AutoRelayService] var holePunchHandler: Option[connmanager.PeerEventHandler] - var peerInfoObserver: Option[PeerInfoObserver] if autonatService.isSome: let relayService = AutoRelayService.new( @@ -487,9 +482,6 @@ proc new*( ) ) - peerInfoObserver = - some(setupPeerInfoObserver(switch, autonatService.get, discovery, natMapper.get)) - setupMappedAddrMapper(switch, natMapper.get) autonatService.get.setStatusAndConfidenceHandler( @@ -535,6 +527,5 @@ proc new*( autoRelayService: autoRelayService, natMapper: natMapper, holePunchHandler: holePunchHandler, - peerInfoObserver: peerInfoObserver, bootstrapNodes: bootstrapNodes, ) diff --git a/tests/storage/testnatreaction.nim b/tests/storage/testnatreaction.nim index 468a3267..fe265285 100644 --- a/tests/storage/testnatreaction.nim +++ b/tests/storage/testnatreaction.nim @@ -2,7 +2,6 @@ import std/[net] import pkg/chronos import pkg/libp2p/[multiaddress, multihash, multicodec] import pkg/libp2p/protocols/connectivity/autonat/types -import pkg/libp2p/protocols/connectivity/autonatv2/service except setup import pkg/libp2p/protocols/connectivity/relay/client as relayClientModule import pkg/libp2p/services/autorelayservice except setup import pkg/libp2p/observedaddrmanager @@ -53,7 +52,7 @@ asyncchecksuite "NAT reaction - port mapping": let discoveryPort = Port(8090) - test "handleNatStatus announces mapped address when NotReachable and UPnP succeeds": + test "handleNatStatus keeps relay off when NotReachable and mapping succeeds": let dialBack = MultiAddress.init("/ip4/1.2.3.4/tcp/8080").expect("valid") let mapper = MockNatPortMapper( mappedPorts: some((Port(9000), Port(9001), MappingProtocol.UPnP)) @@ -64,8 +63,8 @@ asyncchecksuite "NAT reaction - port mapping": NotReachable, Opt.some(dialBack), discoveryPort, disc, sw, autoRelay ) - check disc.announceAddrs == - @[MultiAddress.init("/ip4/1.2.3.4/tcp/9000").expect("valid")] + # A mapping doesn't guarantee reachability, so the relay stays off until + # AutoNAT confirms Reachable. check not autoRelay.isRunning check disc.protocol.clientMode @@ -149,8 +148,12 @@ asyncchecksuite "NAT reaction - address announcing": var sw: Switch var key: PrivateKey var disc: Discovery + var autoRelay: AutoRelayService setup: + autoRelay = AutoRelayService.new( + 1, relayClientModule.RelayClient.new(), nil, Rng.instance().libp2pRng + ) key = PrivateKey.random(Rng.instance().libp2pRng).get() disc = Discovery.new(key, announceAddrs = @[]) sw = newStandardSwitch() @@ -158,70 +161,40 @@ asyncchecksuite "NAT reaction - address announcing": teardown: await sw.stop() + if autoRelay.isRunning: + await autoRelay.stop(sw) let discoveryPort = Port(8090) - test "announcePeerInfoAddrs excludes relay circuit addresses": - let circuitAddr = MultiAddress - .init("/ip4/1.2.3.4/tcp/4040/p2p/" & $sw.peerInfo.peerId & "/p2p-circuit") - .expect("valid") - sw.peerInfo.addrs.add(circuitAddr) + test "handleNatStatus announces the dial-back address when Reachable": + let dialBack = MultiAddress.init("/ip4/1.2.3.4/tcp/9000").expect("valid") - announcePeerInfoAddrs(disc, sw.peerInfo, discoveryPort) - - check circuitAddr notin disc.announceAddrs - check disc.announceAddrs == sw.peerInfo.addrs.filterIt(it != circuitAddr) - - test "announcePeerInfoAddrs does nothing when addresses are already announced": - announcePeerInfoAddrs(disc, sw.peerInfo, discoveryPort) - let seqNo = disc.getSpr().data.seqNo - - announcePeerInfoAddrs(disc, sw.peerInfo, discoveryPort) - - check disc.getSpr().data.seqNo == seqNo - - test "peerInfo observer announces addresses when Reachable": - let autonat = AutonatV2Service.new(Rng.instance().libp2pRng) - discard setupPeerInfoObserver( - sw, autonat, disc, NatPortMapper(discoveryPort: discoveryPort) + let mapper = NatPortMapper(discoveryPort: discoveryPort) + await mapper.handleNatStatus( + Reachable, Opt.some(dialBack), discoveryPort, disc, sw, autoRelay ) - autonat.networkReachability = Reachable - sw.peerInfo.listenAddrs.add( - MultiAddress.init("/ip4/1.2.3.4/tcp/9999").expect("valid") - ) - await sw.peerInfo.update() + check disc.announceAddrs == @[dialBack] - check disc.announceAddrs == sw.peerInfo.addrs + test "handleNatStatus announces the mapped external UDP port when a mapping is active": + let dialBack = MultiAddress.init("/ip4/1.2.3.4/tcp/9000").expect("valid") - test "peerInfo observer announces the mapped external UDP port when a mapping is active": - let autonat = AutonatV2Service.new(Rng.instance().libp2pRng) let mapper = NatPortMapper(discoveryPort: discoveryPort, activeUdpPort: some(Port(40001))) - discard setupPeerInfoObserver(sw, autonat, disc, mapper) - autonat.networkReachability = Reachable - - sw.peerInfo.listenAddrs.add( - MultiAddress.init("/ip4/1.2.3.4/tcp/9999").expect("valid") + await mapper.handleNatStatus( + Reachable, Opt.some(dialBack), discoveryPort, disc, sw, autoRelay ) - await sw.peerInfo.update() let sprAddrs = disc.getSpr().data.addresses.mapIt(it.address) check MultiAddress.init("/ip4/1.2.3.4/udp/40001").expect("valid") in sprAddrs check MultiAddress.init("/ip4/1.2.3.4/udp/" & $discoveryPort).expect("valid") notin sprAddrs - test "peerInfo observer does not announce when the node is not Reachable": - let autonat = AutonatV2Service.new(Rng.instance().libp2pRng) - discard setupPeerInfoObserver( - sw, autonat, disc, NatPortMapper(discoveryPort: discoveryPort) + test "handleNatStatus does not announce when Reachable without a dial-back address": + let mapper = NatPortMapper(discoveryPort: discoveryPort) + await mapper.handleNatStatus( + Reachable, Opt.none(MultiAddress), discoveryPort, disc, sw, autoRelay ) - autonat.networkReachability = NotReachable - - sw.peerInfo.listenAddrs.add( - MultiAddress.init("/ip4/1.2.3.4/tcp/9999").expect("valid") - ) - await sw.peerInfo.update() check disc.announceAddrs == newSeq[MultiAddress]() From ec24e0ffdcff63d35a16ec277347a077e805109e Mon Sep 17 00:00:00 2001 From: Arnaud Date: Mon, 15 Jun 2026 18:01:14 +0400 Subject: [PATCH 136/167] Cleanup --- storage/nat.nim | 10 +++++----- tests/integration/nat/composehelper.nim | 3 +-- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/storage/nat.nim b/storage/nat.nim index a7e76389..31be9c75 100644 --- a/storage/nat.nim +++ b/storage/nat.nim @@ -134,8 +134,8 @@ method hasMappingIds*(m: NatPortMapper): bool {.base, gcsafe.} = m.tcpMappingId.isSome and m.udpMappingId.isSome proc setupMappedAddrMapper*(switch: Switch, natMapper: NatPortMapper) = - ## We define a custom mapper that adds the external port to peerInfo.addrs when - ## a port mapping is active, so AutoNAT tests that port. + ## We define a custom mapper that adds the externally-mapped address to + ## peerInfo.addrs when a port mapping is active, so AutoNAT tests that port. ## PCP/NAT-PMP may grant an external port different from the listen port. let mapper: AddressMapper = proc( addrs: seq[MultiAddress] @@ -192,7 +192,7 @@ method handleNatStatus*( discovery.protocol.clientMode = true - if not autoRelayService.isRunning: + if not autoRelayService.isRunning and discovery.announceAddrs.len > 0: # Remove any announced addresses, they will be replaced. # If the relay is running, the addresses will be updated on reservation. discovery.announceDirectAddrs(@[], udpPort = discoveryPort) @@ -218,8 +218,8 @@ method handleNatStatus*( info "Port mapping created successfully", tcpPort, udpPort, protocol - # The address mapper will use the mapped ports map the addresses for - # libp2p. + # The address mapper uses the mapped port to build the candidate address + # for AutoNAT; the announce happens once AutoNAT confirms Reachable. hasPortMapping = true else: diff --git a/tests/integration/nat/composehelper.nim b/tests/integration/nat/composehelper.nim index 3d51222a..db5f24ec 100644 --- a/tests/integration/nat/composehelper.nim +++ b/tests/integration/nat/composehelper.nim @@ -6,8 +6,7 @@ import std/[os, osproc] import ../utils proc composeCmd(composeFile: string): string = - ## Match the engine the Makefile builds the image with (podman first), so the - ## compose tool sees that image. + ## Prefer podman (where the Makefile builds the image), fall back to docker. let base = if findExe("podman-compose") != "": "podman-compose" From 43db403b849e99526bf284d75b6e8bb23034fcc1 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Mon, 15 Jun 2026 19:45:58 +0400 Subject: [PATCH 137/167] Add warning --- storage/nat.nim | 2 ++ 1 file changed, 2 insertions(+) diff --git a/storage/nat.nim b/storage/nat.nim index 31be9c75..2f87693f 100644 --- a/storage/nat.nim +++ b/storage/nat.nim @@ -187,6 +187,8 @@ method handleNatStatus*( discovery.announceDirectAddrs( @[dialBackAddr.get], udpPort = m.activeUdpPort.get(discoveryPort) ) + else: + warn "Empty dialback address in AutoNat when node is Reachable" of NotReachable: var hasPortMapping = false From bdbd9ed543e583e52beb2694787afad20c7dd40d Mon Sep 17 00:00:00 2001 From: Arnaud Date: Mon, 15 Jun 2026 20:14:35 +0400 Subject: [PATCH 138/167] Add not downloadable test --- tests/integration/nat/node-entrypoint.sh | 9 +- .../nat/not-downloadable/README.md | 42 +++++++ .../nat/not-downloadable/compose.yml | 112 ++++++++++++++++++ .../nat/not-downloadable/router-entrypoint.sh | 7 ++ .../not-downloadable/testnotdownloadable.nim | 96 +++++++++++++++ 5 files changed, 263 insertions(+), 3 deletions(-) create mode 100644 tests/integration/nat/not-downloadable/README.md create mode 100644 tests/integration/nat/not-downloadable/compose.yml create mode 100755 tests/integration/nat/not-downloadable/router-entrypoint.sh create mode 100644 tests/integration/nat/not-downloadable/testnotdownloadable.nim diff --git a/tests/integration/nat/node-entrypoint.sh b/tests/integration/nat/node-entrypoint.sh index d8279976..118bca81 100644 --- a/tests/integration/nat/node-entrypoint.sh +++ b/tests/integration/nat/node-entrypoint.sh @@ -2,9 +2,12 @@ set -euo pipefail -# Redirect the traffic to our router instead -# of podman's own gateway to put B behind the NAT. -ip route replace default via "$ROUTER_LAN_IP" +# Redirect the traffic to our router instead of podman's own gateway to put the +# node behind the NAT. A node on the wan (reachable) leaves ROUTER_LAN_IP unset +# and keeps its default route. +if [[ -n "${ROUTER_LAN_IP:-}" ]]; then + ip route replace default via "$ROUTER_LAN_IP" +fi # Fetch the bootstrap SPR (retry: the bootstrap may still be starting). echo "fetching bootstrap SPR from $BOOTSTRAP_API ..." diff --git a/tests/integration/nat/not-downloadable/README.md b/tests/integration/nat/not-downloadable/README.md new file mode 100644 index 00000000..5581a52a --- /dev/null +++ b/tests/integration/nat/not-downloadable/README.md @@ -0,0 +1,42 @@ +# NAT not-downloadable scenario + +## Scenario + +A node behind a NAT with no relay is `NotReachable` and announces no dialable +address. A remote peer can never dial it, so a download from it fails. + +## Topology + +``` +node B ──── lan ──── router (NAT) ──── wan ──── bootstrap A + └────── node C (reachable) +``` + +- **bootstrap A** — public node on the wan, autonat server, started with + `--nat=extip`. Unlike not-reachable, it runs *without* `--relay-server`, so B + has no relay to fall back to. +- **router** — `lan -> wan` masquerade and *no* inbound forward, so B can dial + out but nothing can dial back in. +- **node B** — `nat=auto`, on the lan. It joins via A, AutoNAT finds it + unreachable, and with no relay it ends up announcing nothing dialable. +- **node C** — `nat=auto`, directly on the wan, so AutoNAT finds it + `Reachable`. It is the peer that tries (and fails) to download from B. + +## Run + +```bash +make testNatIntegration \ + STORAGE_INTEGRATION_TEST_INCLUDES=tests/integration/nat/not-downloadable/testnotdownloadable.nim +``` + +Builds the shared image and brings the compose topology up and down. Rootless, but +needs the host netfilter modules — if the router fails on iptables: +`sudo modprobe iptable_nat nf_conntrack`. + +## Expected result + +B is `NotReachable` and announces no address, while C is `Reachable`. B uploads +a file, then C tries to fetch its manifest over the network and fails. + +Per-run container logs (router, bootstrap, client, node) are written before teardown to +`tests/integration/logs/__NAT_not_downloadable//.log`. diff --git a/tests/integration/nat/not-downloadable/compose.yml b/tests/integration/nat/not-downloadable/compose.yml new file mode 100644 index 00000000..eb29d5f2 --- /dev/null +++ b/tests/integration/nat/not-downloadable/compose.yml @@ -0,0 +1,112 @@ +# A node behind a NAT with no relay can't be reached from outside, so it can't +# be downloaded from: it announces no dialable address, the reachable node C +# finds it as a provider but never dials it. Same real iptables NAT as +# not-reachable, but bootstrap A runs *without* the relay server. Run via +# testnotdownloadable.nim. +# +# node B ──── lan ──── router (NAT) ──── wan ──── bootstrap A +# └────── node C (reachable) +name: nat-not-downloadable + +# Topology addresses, named for their role (defined once, referenced below). +x-addresses: + # fake public internet; a routable range so B looks public to A + wan_subnet: &wan_subnet 7.7.7.0/24 + # private network behind the NAT + lan_subnet: &lan_subnet 10.99.0.0/24 + # A: public bootstrap, autonat server (no relay) + bootstrap_ip: &bootstrap_ip 7.7.7.10 + # C: public node on the wan, reachable, the one that tries to download from B + client_ip: &client_ip 7.7.7.20 + # router's public face + router_wan_ip: &router_wan_ip 7.7.7.2 + # router's private face = B's gateway + router_lan_ip: &router_lan_ip 10.99.0.2 + # B, behind the NAT + node_ip: &node_ip 10.99.0.10 + +networks: + wan: + # Keep the fake public range private, not exposed to the host + internal: true + ipam: + config: + - subnet: *wan_subnet + lan: + ipam: + config: + - subnet: *lan_subnet + +services: + router: + image: localhost/storage-nat + cap_add: [NET_ADMIN] + sysctls: + net.ipv4.ip_forward: 1 + networks: + wan: + ipv4_address: *router_wan_ip + lan: + ipv4_address: *router_lan_ip + environment: + ROUTER_WAN_IP: *router_wan_ip + LAN_SUBNET: *lan_subnet + # scripts mounted, so editing them needs no image rebuild + volumes: + - ../router-common.sh:/scripts/router-common.sh:ro,z + - ./router-entrypoint.sh:/scripts/router-entrypoint.sh:ro,z + entrypoint: ["bash", "/scripts/router-entrypoint.sh"] + + bootstrap: + image: localhost/storage-nat + networks: + wan: + ipv4_address: *bootstrap_ip + entrypoint: ["/app/build/storage"] + command: + - --listen-ip=0.0.0.0 + - --api-bindaddr=0.0.0.0 + - --listen-port=8070 + - --disc-port=8090 + - --api-port=8080 + # bootstrap_ip (anchors can't go inside a string) + - --nat=extip:7.7.7.10 + - --autonat-server + - --no-bootstrap-node + - --data-dir=/data + - --log-level=DEBUG + + # C sits on the wan, directly reachable + client: + image: localhost/storage-nat + depends_on: [bootstrap] + networks: + wan: + ipv4_address: *client_ip + # C's API, published so the test can drive the download from C and poll it + ports: + - "127.0.0.1:18085:8080" + environment: + # C fetches A's SPR from this API at startup to join the network (bootstrap_ip) + BOOTSTRAP_API: http://7.7.7.10:8080 + volumes: + - ../node-entrypoint.sh:/scripts/node-entrypoint.sh:ro,z + entrypoint: ["bash", "/scripts/node-entrypoint.sh"] + + node: + image: localhost/storage-nat + cap_add: [NET_ADMIN] + depends_on: [router, bootstrap] + networks: + lan: + ipv4_address: *node_ip + # B's API, published so the test can upload to it and poll it + ports: + - "127.0.0.1:18084:8080" + environment: + ROUTER_LAN_IP: *router_lan_ip + # B fetches A's SPR from this API at startup to join the network (bootstrap_ip) + BOOTSTRAP_API: http://7.7.7.10:8080 + volumes: + - ../node-entrypoint.sh:/scripts/node-entrypoint.sh:ro,z + entrypoint: ["bash", "/scripts/node-entrypoint.sh"] diff --git a/tests/integration/nat/not-downloadable/router-entrypoint.sh b/tests/integration/nat/not-downloadable/router-entrypoint.sh new file mode 100755 index 00000000..21a8ef3b --- /dev/null +++ b/tests/integration/nat/not-downloadable/router-entrypoint.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +source "$(dirname "$0")/router-common.sh" + +echo "router ready (wan iface $wanif)" + +hold_until_stopped diff --git a/tests/integration/nat/not-downloadable/testnotdownloadable.nim b/tests/integration/nat/not-downloadable/testnotdownloadable.nim new file mode 100644 index 00000000..393ddabb --- /dev/null +++ b/tests/integration/nat/not-downloadable/testnotdownloadable.nim @@ -0,0 +1,96 @@ +## NAT not-downloadable scenario — a node behind a NAT with no relay cannot be +## downloaded from. +## +## Same shape as the not-reachable test: compose.yml brings up a real NAT +## topology, but bootstrap A runs without the relay server. B stays NotReachable +## and announces no dialable address, so a reachable peer C finds it as a +## provider but can never dial it — the manifest fetch fails. +## +## Requires podman-compose and the scenario image: +## podman build -t localhost/storage-nat \ +## -f tests/integration/nat/Dockerfile . + +import std/[json, os, times] +import pkg/chronos +import pkg/questionable/results + +import ../../../asynctest +import ../../../checktest +import ../../storageclient +import ../composehelper + +const + detectTimeout = 300_000 # ms + pollInterval = 5_000 # ms + +proc announcesNothing(info: JsonNode): bool = + ## An unreachable node with no relay has no dialable address to announce. + info{"announceAddresses"}.getElems.len == 0 + +asyncchecksuite "NAT not downloadable": + let + composeFile = currentSourcePath.parentDir / "compose.yml" + nodeApiUrl = "http://127.0.0.1:18084/api/storage/v1" + clientApiUrl = "http://127.0.0.1:18085/api/storage/v1" + suiteName = "NAT not downloadable" + testName = "a NAT'd node without relay cannot be downloaded from" + services = ["router", "bootstrap", "client", "node"] + startTime = now().format("yyyy-MM-dd'_'HH:mm:ss") + var + nodeClient: StorageClient + clientC: StorageClient + + setup: + compose(composeFile, "up -d") + nodeClient = StorageClient.new(nodeApiUrl) + clientC = StorageClient.new(clientApiUrl) + + teardown: + await nodeClient.close() + await clientC.close() + saveContainerLogs(composeFile, suiteName, testName, startTime, services) + compose(composeFile, "down -v") + + test testName: + # Make sure nodeClient is not reachable + check eventuallySafe( + block: + var settled = false + try: + let info = await nodeClient.info() + settled = + info.isOk and info.get{"nat"}{"reachability"}.getStr == "NotReachable" and + info.get.announcesNothing() + except HttpError: + discard + settled, + timeout = detectTimeout, + pollInterval = pollInterval, + ) + + let info = (await nodeClient.info()).get + # Double check to make sure nodeClient is not reachable and has + # nothing to announce + check info.announcesNothing() + + # C is reachable + check eventuallySafe( + block: + var reachable = false + try: + let cInfo = await clientC.info() + reachable = + cInfo.isOk and cInfo.get{"nat"}{"reachability"}.getStr == "Reachable" + except HttpError: + discard + reachable, + timeout = detectTimeout, + pollInterval = pollInterval, + ) + + # B uploads a file + let cid = (await nodeClient.upload("hello from behind the NAT")).get + + # C cannot download the manifest, as B is not reachable + let res = await clientC.downloadManifestOnly(cid) + check res.isErr From df40f4c6e3806977b4e09388e7041a1f21d25cf7 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Mon, 15 Jun 2026 22:26:36 +0400 Subject: [PATCH 139/167] Refactoring --- .../not-downloadable/testnotdownloadable.nim | 34 +++---------------- .../nat/not-reachable/testnotreachable.nim | 18 +--------- tests/integration/nat/pcp/testpcp.nim | 18 +--------- .../nat/reachable/testreachable.nim | 18 +--------- tests/integration/nat/upnp/testupnp.nim | 18 +--------- 5 files changed, 8 insertions(+), 98 deletions(-) diff --git a/tests/integration/nat/not-downloadable/testnotdownloadable.nim b/tests/integration/nat/not-downloadable/testnotdownloadable.nim index 393ddabb..d6a4fb30 100644 --- a/tests/integration/nat/not-downloadable/testnotdownloadable.nim +++ b/tests/integration/nat/not-downloadable/testnotdownloadable.nim @@ -19,10 +19,6 @@ import ../../../checktest import ../../storageclient import ../composehelper -const - detectTimeout = 300_000 # ms - pollInterval = 5_000 # ms - proc announcesNothing(info: JsonNode): bool = ## An unreachable node with no relay has no dialable address to announce. info{"announceAddresses"}.getElems.len == 0 @@ -53,19 +49,9 @@ asyncchecksuite "NAT not downloadable": test testName: # Make sure nodeClient is not reachable - check eventuallySafe( - block: - var settled = false - try: - let info = await nodeClient.info() - settled = - info.isOk and info.get{"nat"}{"reachability"}.getStr == "NotReachable" and - info.get.announcesNothing() - except HttpError: - discard - settled, - timeout = detectTimeout, - pollInterval = pollInterval, + check eventuallyInfo( + nodeClient, + info{"nat"}{"reachability"}.getStr == "NotReachable" and info.announcesNothing(), ) let info = (await nodeClient.info()).get @@ -74,19 +60,7 @@ asyncchecksuite "NAT not downloadable": check info.announcesNothing() # C is reachable - check eventuallySafe( - block: - var reachable = false - try: - let cInfo = await clientC.info() - reachable = - cInfo.isOk and cInfo.get{"nat"}{"reachability"}.getStr == "Reachable" - except HttpError: - discard - reachable, - timeout = detectTimeout, - pollInterval = pollInterval, - ) + check eventuallyInfo(clientC, info{"nat"}{"reachability"}.getStr == "Reachable") # B uploads a file let cid = (await nodeClient.upload("hello from behind the NAT")).get diff --git a/tests/integration/nat/not-reachable/testnotreachable.nim b/tests/integration/nat/not-reachable/testnotreachable.nim index eefd47ab..33f4b510 100644 --- a/tests/integration/nat/not-reachable/testnotreachable.nim +++ b/tests/integration/nat/not-reachable/testnotreachable.nim @@ -13,10 +13,6 @@ import ../../../checktest import ../../storageclient import ../composehelper -const - detectTimeout = 300_000 # ms - pollInterval = 5_000 # ms - proc announcesCircuitAddr(info: JsonNode): bool = info{"announceAddresses"}.getElems.anyIt("p2p-circuit" in it.getStr) @@ -41,19 +37,7 @@ asyncchecksuite "NAT not reachable": test testName: # Wait for the announcements, after the relay reservation is created. - check eventuallySafe( - block: - var settled = false - try: - let info = await client.info() - settled = info.isOk and info.get.announcesCircuitAddr() - except HttpError: - # B's API is not up yet, keep polling - discard - settled, - timeout = detectTimeout, - pollInterval = pollInterval, - ) + check eventuallyInfo(client, info.announcesCircuitAddr()) let info = (await client.info()).get let nat = info{"nat"} diff --git a/tests/integration/nat/pcp/testpcp.nim b/tests/integration/nat/pcp/testpcp.nim index 2785f6d6..6300bc3e 100644 --- a/tests/integration/nat/pcp/testpcp.nim +++ b/tests/integration/nat/pcp/testpcp.nim @@ -18,10 +18,6 @@ import ../../../checktest import ../../storageclient import ../composehelper -const - detectTimeout = 300_000 # ms - pollInterval = 5_000 # ms - proc announcesDirectAddr(info: JsonNode): bool = ## A reachable node announces at least one direct (non-circuit) address. info{"announceAddresses"}.getElems.anyIt("p2p-circuit" notin it.getStr) @@ -48,19 +44,7 @@ asyncchecksuite "NAT pcp": test testName: # Reachable is the settling signal: wait for it, then assert each expected # property separately so a failure points at the exact condition. - check eventuallySafe( - block: - var reachable = false - try: - let info = await client.info() - reachable = - info.isOk and info.get{"nat"}{"reachability"}.getStr == "Reachable" - except HttpError: - discard # B's API is not up yet, keep polling - reachable, - timeout = detectTimeout, - pollInterval = pollInterval, - ) + check eventuallyInfo(client, info{"nat"}{"reachability"}.getStr == "Reachable") let info = (await client.info()).get let nat = info{"nat"} diff --git a/tests/integration/nat/reachable/testreachable.nim b/tests/integration/nat/reachable/testreachable.nim index adefe598..7d5b1110 100644 --- a/tests/integration/nat/reachable/testreachable.nim +++ b/tests/integration/nat/reachable/testreachable.nim @@ -19,10 +19,6 @@ import ../../../checktest import ../../storageclient import ../composehelper -const - detectTimeout = 120_000 # ms - pollInterval = 5_000 # ms - proc announcesDirectAddr(info: JsonNode): bool = ## A reachable node announces at least one direct (non-circuit) address. info{"announceAddresses"}.getElems.anyIt("p2p-circuit" notin it.getStr) @@ -49,19 +45,7 @@ asyncchecksuite "NAT reachable": test testName: # Reachable is the settling signal: wait for it, then assert each expected # property separately so a failure points at the exact condition. - check eventuallySafe( - block: - var reachable = false - try: - let info = await client.info() - reachable = - info.isOk and info.get{"nat"}{"reachability"}.getStr == "Reachable" - except HttpError: - discard # B's API is not up yet, keep polling - reachable, - timeout = detectTimeout, - pollInterval = pollInterval, - ) + check eventuallyInfo(client, info{"nat"}{"reachability"}.getStr == "Reachable") let info = (await client.info()).get let nat = info{"nat"} diff --git a/tests/integration/nat/upnp/testupnp.nim b/tests/integration/nat/upnp/testupnp.nim index e22e812d..7f69facc 100644 --- a/tests/integration/nat/upnp/testupnp.nim +++ b/tests/integration/nat/upnp/testupnp.nim @@ -18,10 +18,6 @@ import ../../../checktest import ../../storageclient import ../composehelper -const - detectTimeout = 300_000 # ms - pollInterval = 5_000 # ms - proc announcesDirectAddr(info: JsonNode): bool = ## A reachable node announces at least one direct (non-circuit) address. info{"announceAddresses"}.getElems.anyIt("p2p-circuit" notin it.getStr) @@ -48,19 +44,7 @@ asyncchecksuite "NAT upnp": test testName: # Reachable is the settling signal: wait for it, then assert each expected # property separately so a failure points at the exact condition. - check eventuallySafe( - block: - var reachable = false - try: - let info = await client.info() - reachable = - info.isOk and info.get{"nat"}{"reachability"}.getStr == "Reachable" - except HttpError: - discard # B's API is not up yet, keep polling - reachable, - timeout = detectTimeout, - pollInterval = pollInterval, - ) + check eventuallyInfo(client, info{"nat"}{"reachability"}.getStr == "Reachable") let info = (await client.info()).get let nat = info{"nat"} From 7edb8bf664aae2448d587b48442c8144ac98a1d8 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Mon, 15 Jun 2026 22:27:15 +0400 Subject: [PATCH 140/167] Add relay download test --- tests/integration/nat/composehelper.nim | 41 +++++-- .../integration/nat/relay-download/README.md | 43 +++++++ .../nat/relay-download/compose.yml | 113 ++++++++++++++++++ .../nat/relay-download/router-entrypoint.sh | 7 ++ .../nat/relay-download/testrelaydownload.nim | 72 +++++++++++ 5 files changed, 269 insertions(+), 7 deletions(-) create mode 100644 tests/integration/nat/relay-download/README.md create mode 100644 tests/integration/nat/relay-download/compose.yml create mode 100755 tests/integration/nat/relay-download/router-entrypoint.sh create mode 100644 tests/integration/nat/relay-download/testrelaydownload.nim diff --git a/tests/integration/nat/composehelper.nim b/tests/integration/nat/composehelper.nim index db5f24ec..4996de7a 100644 --- a/tests/integration/nat/composehelper.nim +++ b/tests/integration/nat/composehelper.nim @@ -22,6 +22,19 @@ proc compose*(composeFile, action: string) = let cmd = composeCmd(composeFile) & " " & action doAssert execShellCmd(cmd) == 0, "command failed: " & cmd +proc serviceLogs*(composeFile, service: string): string = + ## Current logs (stdout+stderr) of a compose service, or "" on error. + try: + let + cmd = composeCmd(composeFile) & " logs " & service + (output, code) = execCmdEx(cmd) + if code != 0: + echo "warning: '", cmd, "' exited ", code + output + except CatchableError as e: + echo "could not read logs for ", service, ": ", e.msg + "" + proc saveContainerLogs*( composeFile, suiteName, testName, startTime: string, services: openArray[string] ) = @@ -30,12 +43,26 @@ proc saveContainerLogs*( ## /.log. Must run before `down` destroys the containers. for service in services: try: - let - logFile = getLogFile("", startTime, suiteName, testName, service) - cmd = composeCmd(composeFile) & " logs " & service - (output, code) = execCmdEx(cmd) - if code != 0: - echo "warning: '", cmd, "' exited ", code - writeFile(logFile, output) + let logFile = getLogFile("", startTime, suiteName, testName, service) + writeFile(logFile, serviceLogs(composeFile, service)) except CatchableError as e: echo "could not save logs for ", service, ": ", e.msg + +template eventuallyInfo*(client, predicate: untyped): bool = + ## Poll `client.info()` until `predicate` holds, swallowing HttpError while the + ## node's API is still starting. The decoded info JsonNode is exposed as `info` + ## inside `predicate`. + eventuallySafe( + block: + var satisfied = false + try: + let res = await client.info() + if res.isOk: + let info {.inject.} = res.get + satisfied = predicate + except HttpError: + discard + satisfied, + timeout = 300000, + pollInterval = 5000, + ) diff --git a/tests/integration/nat/relay-download/README.md b/tests/integration/nat/relay-download/README.md new file mode 100644 index 00000000..dfa9048a --- /dev/null +++ b/tests/integration/nat/relay-download/README.md @@ -0,0 +1,43 @@ +# NAT relay-download scenario + +## Scenario + +A node behind a NAT falls back to bootstrap A's relay and announces its circuit +address. A reachable node C finds it as a provider and downloads its data +through the relay. + +## Topology + +``` +node B ──── lan ──── router (NAT) ──── wan ──── bootstrap A (relay) + └────── node C (reachable) +``` + +- **bootstrap A** — public node on the wan, autonat + relay server, started with + `--nat=extip`. +- **router** — `lan -> wan` masquerade and *no* inbound forward, so B can dial + out but nothing can dial back in. +- **node B** — `nat=auto`, on the lan. AutoNAT finds it unreachable, so it takes + a relay reservation on A and announces its circuit address. +- **node C** — `nat=auto`, directly on the wan, so AutoNAT finds it + `Reachable`. It is the peer that downloads from B through the relay. + +## Run + +```bash +make testNatIntegration \ + STORAGE_INTEGRATION_TEST_INCLUDES=tests/integration/nat/relay-download/testrelaydownload.nim +``` + +Builds the shared image and brings the compose topology up and down. Rootless, but +needs the host netfilter modules — if the router fails on iptables: +`sudo modprobe iptable_nat nf_conntrack`. + +## Expected result + +B is `NotReachable` and announces its circuit address, while C is `Reachable`. +B uploads a file, then C fetches it over the network through the relay and gets +the same content back. + +Per-run container logs (router, bootstrap, client, node) are written before teardown to +`tests/integration/logs/__NAT_relay_download//.log`. diff --git a/tests/integration/nat/relay-download/compose.yml b/tests/integration/nat/relay-download/compose.yml new file mode 100644 index 00000000..d079d587 --- /dev/null +++ b/tests/integration/nat/relay-download/compose.yml @@ -0,0 +1,113 @@ +# A node behind a NAT falls back to bootstrap A's relay and announces its +# circuit address, so a reachable node C can download from it through the relay. +# Same real iptables NAT as not-reachable, with C added as the downloader. Run +# via testrelaydownload.nim. +# +# node B ──── lan ──── router (NAT) ──── wan ──── bootstrap A (relay) +# └────── node C (reachable) +name: nat-relay-download + +# Topology addresses, named for their role (defined once, referenced below). +x-addresses: + # fake public internet; a routable range so B looks public to A + wan_subnet: &wan_subnet 7.7.7.0/24 + # private network behind the NAT + lan_subnet: &lan_subnet 10.99.0.0/24 + # A: public bootstrap, autonat + relay server + bootstrap_ip: &bootstrap_ip 7.7.7.10 + # C: public node on the wan, reachable, the one that downloads from B + client_ip: &client_ip 7.7.7.20 + # router's public face + router_wan_ip: &router_wan_ip 7.7.7.2 + # router's private face = B's gateway + router_lan_ip: &router_lan_ip 10.99.0.2 + # B, behind the NAT + node_ip: &node_ip 10.99.0.10 + +networks: + wan: + # Keep the fake public range private, not exposed to the host + internal: true + ipam: + config: + - subnet: *wan_subnet + lan: + ipam: + config: + - subnet: *lan_subnet + +services: + router: + image: localhost/storage-nat + cap_add: [NET_ADMIN] + sysctls: + net.ipv4.ip_forward: 1 + networks: + wan: + ipv4_address: *router_wan_ip + lan: + ipv4_address: *router_lan_ip + environment: + ROUTER_WAN_IP: *router_wan_ip + LAN_SUBNET: *lan_subnet + # scripts mounted, so editing them needs no image rebuild + volumes: + - ../router-common.sh:/scripts/router-common.sh:ro,z + - ./router-entrypoint.sh:/scripts/router-entrypoint.sh:ro,z + entrypoint: ["bash", "/scripts/router-entrypoint.sh"] + + bootstrap: + image: localhost/storage-nat + networks: + wan: + ipv4_address: *bootstrap_ip + entrypoint: ["/app/build/storage"] + command: + - --listen-ip=0.0.0.0 + - --api-bindaddr=0.0.0.0 + - --listen-port=8070 + - --disc-port=8090 + - --api-port=8080 + # bootstrap_ip (anchors can't go inside a string) + - --nat=extip:7.7.7.10 + - --relay-server + - --autonat-server + - --no-bootstrap-node + - --data-dir=/data + - --log-level=DEBUG + + # C sits on the wan, directly reachable: no NAT, so it leaves ROUTER_LAN_IP + # unset and keeps its default route (see node-entrypoint.sh). + client: + image: localhost/storage-nat + depends_on: [bootstrap] + networks: + wan: + ipv4_address: *client_ip + # C's API, published so the test can drive the download from C and poll it + ports: + - "127.0.0.1:18087:8080" + environment: + # C fetches A's SPR from this API at startup to join the network (bootstrap_ip) + BOOTSTRAP_API: http://7.7.7.10:8080 + volumes: + - ../node-entrypoint.sh:/scripts/node-entrypoint.sh:ro,z + entrypoint: ["bash", "/scripts/node-entrypoint.sh"] + + node: + image: localhost/storage-nat + cap_add: [NET_ADMIN] + depends_on: [router, bootstrap] + networks: + lan: + ipv4_address: *node_ip + # B's API, published so the test can upload to it and poll it + ports: + - "127.0.0.1:18086:8080" + environment: + ROUTER_LAN_IP: *router_lan_ip + # B fetches A's SPR from this API at startup to join the network (bootstrap_ip) + BOOTSTRAP_API: http://7.7.7.10:8080 + volumes: + - ../node-entrypoint.sh:/scripts/node-entrypoint.sh:ro,z + entrypoint: ["bash", "/scripts/node-entrypoint.sh"] diff --git a/tests/integration/nat/relay-download/router-entrypoint.sh b/tests/integration/nat/relay-download/router-entrypoint.sh new file mode 100755 index 00000000..21a8ef3b --- /dev/null +++ b/tests/integration/nat/relay-download/router-entrypoint.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +source "$(dirname "$0")/router-common.sh" + +echo "router ready (wan iface $wanif)" + +hold_until_stopped diff --git a/tests/integration/nat/relay-download/testrelaydownload.nim b/tests/integration/nat/relay-download/testrelaydownload.nim new file mode 100644 index 00000000..3b9ff498 --- /dev/null +++ b/tests/integration/nat/relay-download/testrelaydownload.nim @@ -0,0 +1,72 @@ +## NAT relay-download scenario — a node behind a NAT can be downloaded from +## through the relay. +## +## Same shape as the not-reachable test: compose.yml brings up a real NAT +## topology with bootstrap A running the relay server. B stays NotReachable, +## falls back to the relay and announces its circuit address, so a reachable +## peer C can fetch its data through the relay. +## +## Requires podman-compose and the scenario image: +## podman build -t localhost/storage-nat \ +## -f tests/integration/nat/Dockerfile . + +import std/[json, os, sequtils, strutils, times] +import pkg/chronos +import pkg/questionable/results + +import ../../../asynctest +import ../../../checktest +import ../../storageclient +import ../composehelper + +proc announcesCircuitAddr(info: JsonNode): bool = + ## A node behind the relay announces its circuit (p2p-circuit) address. + info{"announceAddresses"}.getElems.anyIt("p2p-circuit" in it.getStr) + +asyncchecksuite "NAT relay download": + let + composeFile = currentSourcePath.parentDir / "compose.yml" + nodeApiUrl = "http://127.0.0.1:18086/api/storage/v1" + clientApiUrl = "http://127.0.0.1:18087/api/storage/v1" + suiteName = "NAT relay download" + testName = "a NAT'd node behind a relay can be downloaded from" + services = ["router", "bootstrap", "client", "node"] + startTime = now().format("yyyy-MM-dd'_'HH:mm:ss") + var + nodeClient: StorageClient + clientC: StorageClient + + setup: + compose(composeFile, "up -d") + nodeClient = StorageClient.new(nodeApiUrl) + clientC = StorageClient.new(clientApiUrl) + + teardown: + await nodeClient.close() + await clientC.close() + saveContainerLogs(composeFile, suiteName, testName, startTime, services) + compose(composeFile, "down -v") + + test testName: + # B is NotReachable and falls back to the relay, announcing its circuit address + check eventuallyInfo( + nodeClient, + info{"nat"}{"reachability"}.getStr == "NotReachable" and + info.announcesCircuitAddr(), + ) + + let info = (await nodeClient.info()).get + # Double check B announces only its circuit address + check info.announcesCircuitAddr() + + # C is reachable + check eventuallyInfo(clientC, info{"nat"}{"reachability"}.getStr == "Reachable") + + # B uploads a file + let content = "hello from behind the relay" + let cid = (await nodeClient.upload(content)).get + + # C downloads it through the relay and gets the same content back + let res = await clientC.download(cid) + check res.isOk + check res.get == content From caff5f3136ae818cb7c1ca3a83afd47e311920c4 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Mon, 15 Jun 2026 22:27:24 +0400 Subject: [PATCH 141/167] Add hole punching test --- tests/integration/nat/hole-punching/README.md | 43 +++++++ .../integration/nat/hole-punching/compose.yml | 112 ++++++++++++++++++ .../nat/hole-punching/router-entrypoint.sh | 7 ++ .../nat/hole-punching/testholepunching.nim | 72 +++++++++++ 4 files changed, 234 insertions(+) create mode 100644 tests/integration/nat/hole-punching/README.md create mode 100644 tests/integration/nat/hole-punching/compose.yml create mode 100755 tests/integration/nat/hole-punching/router-entrypoint.sh create mode 100644 tests/integration/nat/hole-punching/testholepunching.nim diff --git a/tests/integration/nat/hole-punching/README.md b/tests/integration/nat/hole-punching/README.md new file mode 100644 index 00000000..93fa7097 --- /dev/null +++ b/tests/integration/nat/hole-punching/README.md @@ -0,0 +1,43 @@ +# NAT hole-punching scenario + +## Scenario + +A node behind a NAT is reachable only through A's relay. When the reachable node +C dials it through the relay, the relayed node dials C back directly (C is +public) and the relayed connection is upgraded to a direct one. + +## Topology + +``` +node B ──── lan ──── router (NAT) ──── wan ──── bootstrap A (relay) + └────── node C (reachable) +``` + +- **bootstrap A** — public node on the wan, autonat + relay server. +- **router** — `lan -> wan` masquerade and *no* inbound forward. +- **node B** — `nat=auto`, on the lan. NotReachable, takes a relay reservation + on A. When C reaches it through the relay, its hole-punching handler dials C + back directly and closes the relayed connection. +- **node C** — `nat=auto`, directly on the wan, so it is `Reachable`. It dials B + through the relay. + +## Run + +```bash +make testNatIntegration \ + STORAGE_INTEGRATION_TEST_INCLUDES=tests/integration/nat/hole-punching/testholepunching.nim +``` + +Builds the shared image and brings the compose topology up and down. Rootless, but +needs the host netfilter modules — if the router fails on iptables: +`sudo modprobe iptable_nat nf_conntrack`. + +## Expected result + +B is `NotReachable` behind the relay, C is `Reachable`. C downloads from B +through the relay, which opens a relayed connection; B then dials C back +directly. Hole punching has no REST surface, so the test asserts on B's log line +`Direct connection created.`. + +Per-run container logs (router, bootstrap, client, node) are written before teardown to +`tests/integration/logs/__NAT_hole_punching//.log`. diff --git a/tests/integration/nat/hole-punching/compose.yml b/tests/integration/nat/hole-punching/compose.yml new file mode 100644 index 00000000..49b49cc5 --- /dev/null +++ b/tests/integration/nat/hole-punching/compose.yml @@ -0,0 +1,112 @@ +# A node behind a NAT is reachable only through A's relay. When the reachable +# node C dials it through the relay, the relayed node dials C back directly +# (C is public) and the relayed connection is upgraded to a direct one. Same +# topology as relay-download. Run via testholepunching.nim. +# +# node B ──── lan ──── router (NAT) ──── wan ──── bootstrap A (relay) +# └────── node C (reachable) +name: nat-hole-punching + +# Topology addresses, named for their role (defined once, referenced below). +x-addresses: + # fake public internet; a routable range so B looks public to A + wan_subnet: &wan_subnet 7.7.7.0/24 + # private network behind the NAT + lan_subnet: &lan_subnet 10.99.0.0/24 + # A: public bootstrap, autonat + relay server + bootstrap_ip: &bootstrap_ip 7.7.7.10 + # C: public node on the wan, reachable, the one that dials B through the relay + client_ip: &client_ip 7.7.7.20 + # router's public face + router_wan_ip: &router_wan_ip 7.7.7.2 + # router's private face = B's gateway + router_lan_ip: &router_lan_ip 10.99.0.2 + # B, behind the NAT + node_ip: &node_ip 10.99.0.10 + +networks: + wan: + # Keep the fake public range private, not exposed to the host + internal: true + ipam: + config: + - subnet: *wan_subnet + lan: + ipam: + config: + - subnet: *lan_subnet + +services: + router: + image: localhost/storage-nat + cap_add: [NET_ADMIN] + sysctls: + net.ipv4.ip_forward: 1 + networks: + wan: + ipv4_address: *router_wan_ip + lan: + ipv4_address: *router_lan_ip + environment: + ROUTER_WAN_IP: *router_wan_ip + LAN_SUBNET: *lan_subnet + # scripts mounted, so editing them needs no image rebuild + volumes: + - ../router-common.sh:/scripts/router-common.sh:ro,z + - ./router-entrypoint.sh:/scripts/router-entrypoint.sh:ro,z + entrypoint: ["bash", "/scripts/router-entrypoint.sh"] + + bootstrap: + image: localhost/storage-nat + networks: + wan: + ipv4_address: *bootstrap_ip + entrypoint: ["/app/build/storage"] + command: + - --listen-ip=0.0.0.0 + - --api-bindaddr=0.0.0.0 + - --listen-port=8070 + - --disc-port=8090 + - --api-port=8080 + # bootstrap_ip (anchors can't go inside a string) + - --nat=extip:7.7.7.10 + - --relay-server + - --autonat-server + - --no-bootstrap-node + - --data-dir=/data + - --log-level=DEBUG + + # C sits on the wan, directly reachable + client: + image: localhost/storage-nat + depends_on: [bootstrap] + networks: + wan: + ipv4_address: *client_ip + # C's API, published so the test can drive the download from C and poll it + ports: + - "127.0.0.1:18089:8080" + environment: + # C fetches A's SPR from this API at startup to join the network (bootstrap_ip) + BOOTSTRAP_API: http://7.7.7.10:8080 + volumes: + - ../node-entrypoint.sh:/scripts/node-entrypoint.sh:ro,z + entrypoint: ["bash", "/scripts/node-entrypoint.sh"] + + node: + image: localhost/storage-nat + cap_add: [NET_ADMIN] + depends_on: [router, bootstrap] + networks: + lan: + ipv4_address: *node_ip + # B's API, published so the test can upload to it and poll it + ports: + - "127.0.0.1:18088:8080" + environment: + ROUTER_LAN_IP: *router_lan_ip + # B fetches A's SPR from this API at startup to join the network (bootstrap_ip) + BOOTSTRAP_API: http://7.7.7.10:8080 + volumes: + - ../node-entrypoint.sh:/scripts/node-entrypoint.sh:ro,z + entrypoint: ["bash", "/scripts/node-entrypoint.sh"] diff --git a/tests/integration/nat/hole-punching/router-entrypoint.sh b/tests/integration/nat/hole-punching/router-entrypoint.sh new file mode 100755 index 00000000..21a8ef3b --- /dev/null +++ b/tests/integration/nat/hole-punching/router-entrypoint.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +source "$(dirname "$0")/router-common.sh" + +echo "router ready (wan iface $wanif)" + +hold_until_stopped diff --git a/tests/integration/nat/hole-punching/testholepunching.nim b/tests/integration/nat/hole-punching/testholepunching.nim new file mode 100644 index 00000000..47910b79 --- /dev/null +++ b/tests/integration/nat/hole-punching/testholepunching.nim @@ -0,0 +1,72 @@ +## NAT hole-punching scenario — a node reached through the relay is upgraded to +## a direct connection. +## +## B sits behind a NAT and is reachable only via A's relay. When the reachable +## node C dials B through the relay, B's hole-punching handler dials C back +## directly (C is public) and the relayed connection is replaced by a direct one. +## +## Hole punching has no REST surface, so success is asserted on B's container log +## line below (DEBUG). Brittle if that message ever changes. +## +## Requires podman-compose and the scenario image: +## podman build -t localhost/storage-nat \ +## -f tests/integration/nat/Dockerfile . + +import std/[json, os, sequtils, strutils, times] +import pkg/chronos +import pkg/questionable/results + +import ../../../asynctest +import ../../../checktest +import ../../storageclient +import ../composehelper + +const directConnLog = "Direct connection created." + +proc announcesCircuitAddr(info: JsonNode): bool = + ## A node behind the relay announces its circuit (p2p-circuit) address. + info{"announceAddresses"}.getElems.anyIt("p2p-circuit" in it.getStr) + +asyncchecksuite "NAT hole punching": + let + composeFile = currentSourcePath.parentDir / "compose.yml" + nodeApiUrl = "http://127.0.0.1:18088/api/storage/v1" + clientApiUrl = "http://127.0.0.1:18089/api/storage/v1" + suiteName = "NAT hole punching" + testName = "a relayed node is upgraded to a direct connection" + services = ["router", "bootstrap", "client", "node"] + startTime = now().format("yyyy-MM-dd'_'HH:mm:ss") + var + nodeClient: StorageClient + clientC: StorageClient + + setup: + compose(composeFile, "up -d") + nodeClient = StorageClient.new(nodeApiUrl) + clientC = StorageClient.new(clientApiUrl) + + teardown: + await nodeClient.close() + await clientC.close() + saveContainerLogs(composeFile, suiteName, testName, startTime, services) + compose(composeFile, "down -v") + + test testName: + # B is NotReachable behind the relay, C is reachable + check eventuallyInfo( + nodeClient, + info{"nat"}{"reachability"}.getStr == "NotReachable" and + info.announcesCircuitAddr(), + ) + check eventuallyInfo(clientC, info{"nat"}{"reachability"}.getStr == "Reachable") + + # C dials B through the relay; a download is enough to open the connection + let cid = (await nodeClient.upload("hole punch me")).get + check (await clientC.download(cid)).isOk + + # B sees the relayed peer C join and dials it back directly + check eventuallySafe( + directConnLog in serviceLogs(composeFile, "node"), + timeout = 60_000, + pollInterval = 2_000, + ) From c6bfc889260be3183c183dcb675fbc7692e8b480 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Mon, 15 Jun 2026 22:47:22 +0400 Subject: [PATCH 142/167] Add more checks for nat integration tests --- tests/integration/nat/composehelper.nim | 4 ++++ tests/integration/nat/not-reachable/testnotreachable.nim | 7 +++++++ tests/integration/nat/pcp/testpcp.nim | 6 ++++++ tests/integration/nat/reachable/testreachable.nim | 5 +++++ tests/integration/nat/upnp/testupnp.nim | 5 +++++ 5 files changed, 27 insertions(+) diff --git a/tests/integration/nat/composehelper.nim b/tests/integration/nat/composehelper.nim index 4996de7a..8386ffc3 100644 --- a/tests/integration/nat/composehelper.nim +++ b/tests/integration/nat/composehelper.nim @@ -5,6 +5,10 @@ import std/[os, osproc] import ../utils +const + routerWanIp* = "7.7.7.2" ## public IP AutoNAT observes for a NATed node (masquerade) + bootstrapIp* = "7.7.7.10" ## relay + bootstrap public IP + proc composeCmd(composeFile: string): string = ## Prefer podman (where the Makefile builds the image), fall back to docker. let base = diff --git a/tests/integration/nat/not-reachable/testnotreachable.nim b/tests/integration/nat/not-reachable/testnotreachable.nim index 33f4b510..9298b37e 100644 --- a/tests/integration/nat/not-reachable/testnotreachable.nim +++ b/tests/integration/nat/not-reachable/testnotreachable.nim @@ -45,3 +45,10 @@ asyncchecksuite "NAT not reachable": check nat{"relayRunning"}.getBool check nat{"portMapping"}.getStr == "none" check info.announcesCircuitAddr() + let announced = info{"announceAddresses"}.getElems.mapIt(it.getStr) + # the announced circuit address points at the bootstrap's relay + check announced.anyIt( + ("/ip4/" & bootstrapIp & "/tcp/8070" in it) and ("p2p-circuit" in it) + ) + # relay addresses go only into the provider record, never the DHT routing record + check info{"dhtAddresses"}.getElems.len == 0 diff --git a/tests/integration/nat/pcp/testpcp.nim b/tests/integration/nat/pcp/testpcp.nim index 6300bc3e..64c99c21 100644 --- a/tests/integration/nat/pcp/testpcp.nim +++ b/tests/integration/nat/pcp/testpcp.nim @@ -52,3 +52,9 @@ asyncchecksuite "NAT pcp": check nat{"relayRunning"}.getBool == false check nat{"portMapping"}.getStr == "pcp" check info.announcesDirectAddr() + let announced = info{"announceAddresses"}.getElems.mapIt(it.getStr) + # PCP may map a port different from the listen port, so check the IP only + check announced.anyIt(("/ip4/" & routerWanIp & "/tcp/") in it) + # the public mapped address + # a reachable node announces its UDP address to the DHT routing record + check info{"dhtAddresses"}.getElems.len > 0 diff --git a/tests/integration/nat/reachable/testreachable.nim b/tests/integration/nat/reachable/testreachable.nim index 7d5b1110..2601f0fe 100644 --- a/tests/integration/nat/reachable/testreachable.nim +++ b/tests/integration/nat/reachable/testreachable.nim @@ -52,3 +52,8 @@ asyncchecksuite "NAT reachable": check nat{"reachability"}.getStr == "Reachable" check nat{"relayRunning"}.getBool == false check info.announcesDirectAddr() + let announced = info{"announceAddresses"}.getElems.mapIt(it.getStr) + check announced.anyIt(("/ip4/" & routerWanIp & "/tcp/8070") in it) + # public forwarded address + # a reachable node announces its UDP address to the DHT routing record + check info{"dhtAddresses"}.getElems.len > 0 diff --git a/tests/integration/nat/upnp/testupnp.nim b/tests/integration/nat/upnp/testupnp.nim index 7f69facc..0fd623ab 100644 --- a/tests/integration/nat/upnp/testupnp.nim +++ b/tests/integration/nat/upnp/testupnp.nim @@ -52,3 +52,8 @@ asyncchecksuite "NAT upnp": check nat{"relayRunning"}.getBool == false check nat{"portMapping"}.getStr == "upnp" check info.announcesDirectAddr() + let announced = info{"announceAddresses"}.getElems.mapIt(it.getStr) + check announced.anyIt(("/ip4/" & routerWanIp & "/tcp/8070") in it) + # public mapped address + # a reachable node announces its UDP address to the DHT routing record + check info{"dhtAddresses"}.getElems.len > 0 From b055aefb52b965aba189a65abc6ab7d88824c7d3 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Tue, 16 Jun 2026 08:47:19 +0400 Subject: [PATCH 143/167] Improve testsi --- storage/nat.nim | 45 ++++++++---- tests/storage/testnatreaction.nim | 109 +++++++++++++++++++++++++++++- 2 files changed, 139 insertions(+), 15 deletions(-) diff --git a/storage/nat.nim b/storage/nat.nim index 2f87693f..b45cba0d 100644 --- a/storage/nat.nim +++ b/storage/nat.nim @@ -62,6 +62,31 @@ proc resetMappings(m: NatPortMapper) = m.activeTcpPort = none(Port) m.activeUdpPort = none(Port) +# libplum seams, extracted as methods so tests can override them without I/O. + +method initPlum*(m: NatPortMapper): Result[void, string] {.base, gcsafe.} = + let plumLogLevel = + if getEnv("DEBUG") == "1": PlumLogLevel.Verbose else: PlumLogLevel.None + init( + logLevel = plumLogLevel, + discoverTimeout = m.discoverTimeout.int32, + mappingTimeout = m.mappingTimeout.int32, + recheckPeriod = m.recheckPeriod.int32, + ) + +method createMappingFor*( + m: NatPortMapper, protocol: PlumProtocol, port: uint16 +): Future[Result[MappingResult, string]] {. + base, async: (raises: [CancelledError]), gcsafe +.} = + await createMapping(protocol, port, port) + +method destroyMappingFor*(m: NatPortMapper, id: cint) {.base, gcsafe.} = + destroyMapping(id) + +method hasLiveMapping*(m: NatPortMapper, id: cint): bool {.base, gcsafe.} = + hasMapping(id) + method mapNatPorts*( m: NatPortMapper ): Future[Option[(Port, Port, MappingProtocol)]] {. @@ -71,20 +96,12 @@ method mapNatPorts*( return none((Port, Port, MappingProtocol)) # If both mappings are still active, return the stored ports without recreating. - if m.tcpMappingId.isSome and hasMapping(m.tcpMappingId.get) and m.udpMappingId.isSome and - hasMapping(m.udpMappingId.get): + if m.tcpMappingId.isSome and m.hasLiveMapping(m.tcpMappingId.get) and + m.udpMappingId.isSome and m.hasLiveMapping(m.udpMappingId.get): return some((m.activeTcpPort.get, m.activeUdpPort.get, m.activeMappingProtocol.get)) if not m.plumInitialized: - # 5s matches the old NatPortMappingTimeout used with miniupnpc/libnatpmp. - let plumLogLevel = - if getEnv("DEBUG") == "1": PlumLogLevel.Verbose else: PlumLogLevel.None - let res = init( - logLevel = plumLogLevel, - discoverTimeout = m.discoverTimeout.int32, - mappingTimeout = m.mappingTimeout.int32, - recheckPeriod = m.recheckPeriod.int32, - ) + let res = m.initPlum() if res.isErr: warn "Failed to initialize plum", msg = res.error return none((Port, Port, MappingProtocol)) @@ -94,15 +111,15 @@ method mapNatPorts*( # so we delete the mappings to recreate them. m.resetMappings() - let tcpRes = await createMapping(TCP, m.tcpPort.uint16, m.tcpPort.uint16) + let tcpRes = await m.createMappingFor(TCP, m.tcpPort.uint16) if tcpRes.isErr: warn "TCP port mapping failed", msg = tcpRes.error return none((Port, Port, MappingProtocol)) - let udpRes = await createMapping(UDP, m.discoveryPort.uint16, m.discoveryPort.uint16) + let udpRes = await m.createMappingFor(UDP, m.discoveryPort.uint16) if udpRes.isErr: warn "UDP port mapping failed", msg = udpRes.error - destroyMapping(tcpRes.value.id) + m.destroyMappingFor(tcpRes.value.id) return none((Port, Port, MappingProtocol)) m.tcpMappingId = some(tcpRes.value.id) diff --git a/tests/storage/testnatreaction.nim b/tests/storage/testnatreaction.nim index fe265285..728bbcbb 100644 --- a/tests/storage/testnatreaction.nim +++ b/tests/storage/testnatreaction.nim @@ -1,4 +1,4 @@ -import std/[net] +import std/[importutils, net] import pkg/chronos import pkg/libp2p/[multiaddress, multihash, multicodec] import pkg/libp2p/protocols/connectivity/autonat/types @@ -29,6 +29,36 @@ method mapNatPorts*( method hasMappingIds*(m: MockNatPortMapper): bool = m.activeMapping +type MockMapNatPortMapper = ref object of NatPortMapper + tcpResult: Result[MappingResult, string] + udpResult: Result[MappingResult, string] + live: bool + created: seq[PlumProtocol] + destroyed: seq[cint] + +method initPlum(m: MockMapNatPortMapper): Result[void, string] {.gcsafe.} = + ok() + +method hasLiveMapping(m: MockMapNatPortMapper, id: cint): bool {.gcsafe.} = + m.live + +method createMappingFor( + m: MockMapNatPortMapper, protocol: PlumProtocol, port: uint16 +): Future[Result[MappingResult, string]] {.async: (raises: [CancelledError]), gcsafe.} = + m.created.add(protocol) + if protocol == TCP: m.tcpResult else: m.udpResult + +method destroyMappingFor(m: MockMapNatPortMapper, id: cint) {.gcsafe.} = + m.destroyed.add(id) + +proc mappingOk(id: cint, port: uint16): Result[MappingResult, string] = + Result[MappingResult, string].ok( + MappingResult( + id: id, + mapping: PlumMapping(mappingProtocol: MappingProtocol.UPnP, externalPort: port), + ) + ) + asyncchecksuite "NAT reaction - port mapping": var sw: Switch var key: PrivateKey @@ -230,3 +260,80 @@ asyncchecksuite "NAT reaction - address announcing": # Ensure that nothing is injected because there is no active mapping check sw.peerInfo.addrs == sw.peerInfo.listenAddrs + +proc mapperWith(protocol: MappingProtocol): Option[NatPortMapper] = + some(NatPortMapper(activeMappingProtocol: some(protocol))) + +asyncchecksuite "NAT - portMappingStr": + test "no mapper is none": + check portMappingStr(none(NatPortMapper)) == "none" + + test "mapper without an active protocol is none": + check portMappingStr(some(NatPortMapper())) == "none" + + test "UPnP maps to upnp": + check portMappingStr(mapperWith(MappingProtocol.UPnP)) == "upnp" + + test "NAT-PMP maps to pmp": + check portMappingStr(mapperWith(MappingProtocol.NatPmp)) == "pmp" + + test "PCP maps to pcp": + check portMappingStr(mapperWith(MappingProtocol.PCP)) == "pcp" + + test "Direct maps to direct": + check portMappingStr(mapperWith(MappingProtocol.Direct)) == "direct" + + test "Unknown maps to none": + check portMappingStr(mapperWith(MappingProtocol.Unknown)) == "none" + +asyncchecksuite "NAT - mapNatPorts": + test "returns the mapped ports when both mappings succeed": + let mapper = MockMapNatPortMapper( + tcpResult: mappingOk(cint(1), 9000), udpResult: mappingOk(cint(2), 9001) + ) + + check (await mapper.mapNatPorts()) == + some((Port(9000), Port(9001), MappingProtocol.UPnP)) + check mapper.destroyed.len == 0 + + test "destroys the TCP mapping when the UDP mapping fails": + let mapper = MockMapNatPortMapper( + tcpResult: mappingOk(cint(42), 9000), + udpResult: Result[MappingResult, string].err("udp mapping failed"), + ) + + check (await mapper.mapNatPorts()).isNone + check mapper.destroyed == @[cint(42)] + + test "gives up without touching UDP when the TCP mapping fails": + let mapper = MockMapNatPortMapper( + tcpResult: Result[MappingResult, string].err("tcp mapping failed"), + udpResult: mappingOk(cint(2), 9001), + ) + + check (await mapper.mapNatPorts()).isNone + check mapper.created == @[PlumProtocol.TCP] # UDP never attempted + check mapper.destroyed.len == 0 # nothing to clean up + + test "does not map when configured with an external IP": + let mapper = MockMapNatPortMapper( + natConfig: nat.NatConfig(hasExtIp: true, extIp: parseIpAddress("1.2.3.4")) + ) + + check (await mapper.mapNatPorts()).isNone + check mapper.created.len == 0 # short-circuits before any mapping + + test "reuses the existing mapping when both are still live": + privateAccess(NatPortMapper) + let mapper = MockMapNatPortMapper( + live: true, + activeTcpPort: some(Port(9000)), + activeUdpPort: some(Port(9001)), + activeMappingProtocol: some(MappingProtocol.UPnP), + ) + mapper.tcpMappingId = some(cint(1)) # private field, set via privateAccess + mapper.udpMappingId = some(cint(2)) + + check (await mapper.mapNatPorts()) == + some((Port(9000), Port(9001), MappingProtocol.UPnP)) + check mapper.created.len == 0 # reuse path, nothing recreated From 594d47fe23a03ae316c1954165c8168d72ddac95 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Tue, 16 Jun 2026 10:11:22 +0400 Subject: [PATCH 144/167] Add more tests --- storage/nat.nim | 16 +++++++++++ storage/storage.nim | 13 +-------- tests/storage/testnatreaction.nim | 46 +++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 12 deletions(-) diff --git a/storage/nat.nim b/storage/nat.nim index b45cba0d..2d984524 100644 --- a/storage/nat.nim +++ b/storage/nat.nim @@ -274,6 +274,22 @@ proc findReachableNodes*(bootstrapNodes: seq[SignedPeerRecord]): seq[SignedPeerR ## confirmed reachable by AutoNAT could be included. bootstrapNodes +proc announceRelayReservation*( + discovery: Discovery, addresses: seq[MultiAddress] +) {.gcsafe.} = + ## Announce the publicly dialable circuit addresses from a relay reservation. + ## A reservation response can also carry loopback/private addresses, which a + ## remote peer can never dial, so they are dropped. If none are public, the + ## previous announce is kept untouched. + let publicAddrs = addresses.filterIt(it.hasPublicRelayTransport()) + if publicAddrs.len == 0: + warn "Relay reservation has no publicly dialable address, keeping previous announce", + addresses + return + info "Relay reservation updated", addresses = publicAddrs + # relay addresses are for download traffic only, not DHT routing + discovery.announceRelayAddrs(publicAddrs) + # Hole punching logic below is adapted from libp2p's HPService # (libp2p/services/hpservice.nim). HPService cannot be used directly because it # depends on AutoNAT v1 and starts the relay immediately on NotReachable, diff --git a/storage/storage.nim b/storage/storage.nim index a57705ea..0dd6bd2f 100644 --- a/storage/storage.nim +++ b/storage/storage.nim @@ -453,18 +453,7 @@ proc new*( maxNumRelays = config.natMaxRelays, client = relayClient, onReservation = proc(addresses: seq[MultiAddress]) {.gcsafe, raises: [].} = - # A relay server is required to have a public extip, so its - # circuit addresses always include a public one. The relay's reservation - # response can also carry loopback/private addresses: - # they are never dialable by a remote peer, so drop them. - let publicAddrs = addresses.filterIt(it.hasPublicRelayTransport()) - if publicAddrs.len == 0: - warn "Relay reservation has no publicly dialable address, keeping previous announce", - addresses - return - info "Relay reservation updated", addresses = publicAddrs - # relay addresses are for download traffic only, not DHT routing - discovery.announceRelayAddrs(publicAddrs), + discovery.announceRelayReservation(addresses), rng = random.Rng.instance().libp2pRng, ) diff --git a/tests/storage/testnatreaction.nim b/tests/storage/testnatreaction.nim index 728bbcbb..1a6d1e31 100644 --- a/tests/storage/testnatreaction.nim +++ b/tests/storage/testnatreaction.nim @@ -59,6 +59,13 @@ proc mappingOk(id: cint, port: uint16): Result[MappingResult, string] = ) ) +const relayId = "16Uiu2HAmQu456Ae52JqPuqog6wCex47LLvNY8oHMBC4GRRtaStHs" + +proc circuitAddr(relayIp: string): MultiAddress = + MultiAddress + .init("/ip4/" & relayIp & "/tcp/8070/p2p/" & relayId & "/p2p-circuit") + .expect("valid") + asyncchecksuite "NAT reaction - port mapping": var sw: Switch var key: PrivateKey @@ -261,6 +268,45 @@ asyncchecksuite "NAT reaction - address announcing": # Ensure that nothing is injected because there is no active mapping check sw.peerInfo.addrs == sw.peerInfo.listenAddrs + test "handleNatStatus clears the DHT routing addresses when it becomes NotReachable": + let dialBack = MultiAddress.init("/ip4/1.2.3.4/tcp/9000").expect("valid") + let mapper = MockNatPortMapper(mappedPorts: none((Port, Port, MappingProtocol))) + + autorelayservice.setup(autoRelay, sw) + + # Reachable: the node announces direct addresses, including UDP for the DHT. + await mapper.handleNatStatus( + Reachable, Opt.some(dialBack), discoveryPort, disc, sw, autoRelay + ) + check disc.dhtAddrs.len > 0 + + # NotReachable: the DHT routing addresses are cleared + await mapper.handleNatStatus( + NotReachable, Opt.some(dialBack), discoveryPort, disc, sw, autoRelay + ) + check disc.dhtAddrs.len == 0 + + test "mapped-addr mapper does not inject a non-public mapped address": + # Active mapping, but no public observed address: the candidate stays private + # and must not be injected. + setupMappedAddrMapper(sw, NatPortMapper(activeTcpPort: some(Port(40000)))) + + await sw.peerInfo.update() + + check sw.peerInfo.addrs == sw.peerInfo.listenAddrs + + test "announceRelayReservation announces only the publicly dialable circuit address": + disc.announceRelayReservation( + @[circuitAddr("127.0.0.1"), circuitAddr("204.168.234.45")] + ) + + check disc.announceAddrs == @[circuitAddr("204.168.234.45")] + + test "announceRelayReservation does not announce a private circuit address": + disc.announceRelayReservation(@[circuitAddr("127.0.0.1")]) + + check disc.announceAddrs.len == 0 + proc mapperWith(protocol: MappingProtocol): Option[NatPortMapper] = some(NatPortMapper(activeMappingProtocol: some(protocol))) From e523c52aa4bcb25fb0cb0ac9216d4bb97adbcf92 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Tue, 16 Jun 2026 10:49:27 +0400 Subject: [PATCH 145/167] Use interal destroyMappingFor --- storage/nat.nim | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/storage/nat.nim b/storage/nat.nim index 2d984524..8dee4cd0 100644 --- a/storage/nat.nim +++ b/storage/nat.nim @@ -49,19 +49,6 @@ type NatPortMapper* = ref object of RootObj plumInitialized: bool closed: bool -proc resetMappings(m: NatPortMapper) = - if m.tcpMappingId.isSome: - destroyMapping(m.tcpMappingId.get) - m.tcpMappingId = none(cint) - - if m.udpMappingId.isSome: - destroyMapping(m.udpMappingId.get) - m.udpMappingId = none(cint) - - m.activeMappingProtocol = none(MappingProtocol) - m.activeTcpPort = none(Port) - m.activeUdpPort = none(Port) - # libplum seams, extracted as methods so tests can override them without I/O. method initPlum*(m: NatPortMapper): Result[void, string] {.base, gcsafe.} = @@ -87,6 +74,19 @@ method destroyMappingFor*(m: NatPortMapper, id: cint) {.base, gcsafe.} = method hasLiveMapping*(m: NatPortMapper, id: cint): bool {.base, gcsafe.} = hasMapping(id) +proc resetMappings(m: NatPortMapper) = + if m.tcpMappingId.isSome: + m.destroyMappingFor(m.tcpMappingId.get) + m.tcpMappingId = none(cint) + + if m.udpMappingId.isSome: + m.destroyMappingFor(m.udpMappingId.get) + m.udpMappingId = none(cint) + + m.activeMappingProtocol = none(MappingProtocol) + m.activeTcpPort = none(Port) + m.activeUdpPort = none(Port) + method mapNatPorts*( m: NatPortMapper ): Future[Option[(Port, Port, MappingProtocol)]] {. @@ -194,13 +194,13 @@ method handleNatStatus*( of Unknown: discard of Reachable: - if autoRelayService.isRunning: - await autoRelayService.stop(switch) - debug "AutoRelayService stopped" - - discovery.protocol.clientMode = false - if dialBackAddr.isSome: + if autoRelayService.isRunning: + await autoRelayService.stop(switch) + debug "AutoRelayService stopped" + + discovery.protocol.clientMode = false + discovery.announceDirectAddrs( @[dialBackAddr.get], udpPort = m.activeUdpPort.get(discoveryPort) ) From 54d9d8c2db3e8c29191de50eb3d7899f4eb7ff67 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Tue, 16 Jun 2026 10:49:34 +0400 Subject: [PATCH 146/167] Restaure nix file --- nix/nimble.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/nix/nimble.nix b/nix/nimble.nix index 86262b7c..7dde65c5 100644 --- a/nix/nimble.nix +++ b/nix/nimble.nix @@ -4,6 +4,7 @@ let tools = pkgs.callPackage ./tools.nix {}; nbsVersion = tools.findKeyValue "^[[:space:]]+NIMBLE_COMMIT='([a-f0-9]+)'.*$" ../vendor/nimbus-build-system/scripts/build_nim.sh; nimVersion = tools.findKeyValue "^ +NimbleStableCommit = \"([a-f0-9]+)\".+" ../vendor/nimbus-build-system/vendor/Nim/koch.nim; + sourceFile = ../vendor/nimbus-build-system/vendor/Nim/koch.nim; in pkgs.fetchFromGitHub { owner = "nim-lang"; repo = "nimble"; From dcdc9d045052aef5d391a4d004a56b7c7f983d7c Mon Sep 17 00:00:00 2001 From: Arnaud Date: Tue, 16 Jun 2026 10:53:01 +0400 Subject: [PATCH 147/167] Guard isSome --- storage/nat.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/storage/nat.nim b/storage/nat.nim index 8dee4cd0..61a07955 100644 --- a/storage/nat.nim +++ b/storage/nat.nim @@ -96,7 +96,8 @@ method mapNatPorts*( return none((Port, Port, MappingProtocol)) # If both mappings are still active, return the stored ports without recreating. - if m.tcpMappingId.isSome and m.hasLiveMapping(m.tcpMappingId.get) and + if m.activeTcpPort.isSome and m.activeUdpPort.isSome and m.activeMappingProtocol.isSome and + m.tcpMappingId.isSome and m.hasLiveMapping(m.tcpMappingId.get) and m.udpMappingId.isSome and m.hasLiveMapping(m.udpMappingId.get): return some((m.activeTcpPort.get, m.activeUdpPort.get, m.activeMappingProtocol.get)) From 540d8440dc2ac35d4f02b44a145a848b9071901d Mon Sep 17 00:00:00 2001 From: Arnaud Date: Tue, 16 Jun 2026 11:08:28 +0400 Subject: [PATCH 148/167] Cleanup --- storage/discovery.nim | 4 ++-- storage/storage.nim | 15 +++++---------- tests/storage/testnatreaction.nim | 3 ++- 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/storage/discovery.nim b/storage/discovery.nim index c7f17403..766a2fc0 100644 --- a/storage/discovery.nim +++ b/storage/discovery.nim @@ -203,8 +203,8 @@ proc announceDirectAddrs*( d.protocol.updateRecord(spr).expect("Should update SPR") proc announceRelayAddrs*(d: Discovery, addrs: openArray[MultiAddress]) = - ## Updates only announce addresses - ## When using relay, the DHT routing record is not updated to not pollute the DHT. + ## Updates the announce addresses and the SPR with the relay circuit addresses. + ## Unlike announceDirectAddrs, no UDP address is derived so dhtAddrs is left untouched. d.announceAddrs = @addrs info "Updating announce record", addrs = d.announceAddrs d.providerRecord = SignedPeerRecord diff --git a/storage/storage.nim b/storage/storage.nim index 0dd6bd2f..e27ea988 100644 --- a/storage/storage.nim +++ b/storage/storage.nim @@ -48,11 +48,6 @@ import ./utils/natutils logScope: topics = "storage node" -const StorageTransportFlags = {ServerFlags.ReuseAddr, ServerFlags.TcpNoDelay} - -proc tcpTransportBuilder(config: TransportConfig): Transport {.gcsafe, raises: [].} = - TcpTransport.new(StorageTransportFlags, config.upgr) - type StorageServer* = ref object config: StorageConf @@ -144,8 +139,8 @@ proc start*(s: StorageServer) {.async.} = await allFutures(findReachableNodes(s.bootstrapNodes).mapIt(connectBootstrapNode(it))) - # AutoNAT is not in switch.services: start it after the bootstrap dials - # so its first probe has peers to ask. + # AutoNAT is not in switch.services because we want to start it + # after the bootstrap connections to have connected peers for the first probe. if s.autonatService.isSome: await s.autonatService.get.start(s.storageNode.switch) @@ -238,10 +233,8 @@ proc new*( config: StorageConf, privateKey: StoragePrivateKey, logFile: Option[IoHandle] = IoHandle.none, - transportBuilder: TransportBuilder = tcpTransportBuilder, ): StorageServer = ## create StorageServer including setting up datastore, repostore, etc. - ## ``transportBuilder`` defaults to TCP; tests inject a simulated NAT transport. if err =? config.validateAutonatConfig().errorOption: raise newException(StorageError, err.msg) @@ -324,7 +317,9 @@ proc new*( # addresses. switchBuilder = switchBuilder.withAddressPolicy(dialableAddressPolicy) - let switch = switchBuilder.withTransport(transportBuilder).build() + let switch = switchBuilder + .withTcpTransport({ServerFlags.ReuseAddr, ServerFlags.TcpNoDelay}) + .build() var taskPool: Taskpool diff --git a/tests/storage/testnatreaction.nim b/tests/storage/testnatreaction.nim index 1a6d1e31..13e99ef8 100644 --- a/tests/storage/testnatreaction.nim +++ b/tests/storage/testnatreaction.nim @@ -155,12 +155,13 @@ asyncchecksuite "NAT reaction - port mapping": check disc.protocol.clientMode test "handleNatStatus stops relay and exits client mode when mapping is created and node is Reachable": + let dialBack = MultiAddress.init("/ip4/1.2.3.4/tcp/8080").expect("valid") let mapper = MockNatPortMapper(mappedPorts: none((Port, Port, MappingProtocol))) disc.protocol.clientMode = true autorelayservice.setup(autoRelay, sw) await mapper.handleNatStatus( - Reachable, Opt.none(MultiAddress), discoveryPort, disc, sw, autoRelay + Reachable, Opt.some(dialBack), discoveryPort, disc, sw, autoRelay ) check not autoRelay.isRunning From 5f96fed3e3896e94376421bb3f5b2e31929d8d1d Mon Sep 17 00:00:00 2001 From: Arnaud Date: Tue, 16 Jun 2026 11:21:26 +0400 Subject: [PATCH 149/167] Add more config checks --- storage/conf.nim | 9 +++++++++ storage/storage.nim | 8 ++------ tests/storage/testconf.nim | 26 ++++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 6 deletions(-) diff --git a/storage/conf.nim b/storage/conf.nim index b1ea573c..d3c14bce 100644 --- a/storage/conf.nim +++ b/storage/conf.nim @@ -396,12 +396,21 @@ func validateAutonatConfig*(config: StorageConf): ?!void = if config.isRelayServer and not config.nat.hasExtIp: return failure "--relay-server requires --nat=extip:" + if config.noBootstrapNode and not config.nat.hasExtIp: + return failure( + "--no-bootstrap-node requires --nat=extip:: without bootstrap peers " & + "AutoNAT has no one to probe and the node can never become reachable" + ) + if config.natMaxQueueSize < 1: return failure "--nat-max-queue-size must be at least 1" if config.natNumPeersToAsk < 1: return failure "--nat-num-peers-to-ask must be at least 1" + if config.natObservedAddrMinCount < 1: + return failure "--nat-observed-addr-min-count must be at least 1" + if config.natMinConfidence < 0.0 or config.natMinConfidence > 1.0: return failure "--nat-min-confidence must be between 0 and 1" diff --git a/storage/storage.nim b/storage/storage.nim index e27ea988..9c5690b0 100644 --- a/storage/storage.nim +++ b/storage/storage.nim @@ -302,12 +302,8 @@ proc new*( enableDialableCandidates = true, ) ) - # At the first AutoNAT probe, the only identify observations available come - # from the bootstrap nodes, so requiring more observations than there are - # bootstrap nodes would make the threshold unreachable. The floor of 1 - # covers the case where the bootstrap list is empty. - let observedAddrMinCount = - max(1, min(config.natObservedAddrMinCount, bootstrapNodes.len)) + + let observedAddrMinCount = min(config.natObservedAddrMinCount, bootstrapNodes.len) switchBuilder = switchBuilder.withObservedAddrManager( ObservedAddrManager.new(minCount = observedAddrMinCount) ) diff --git a/tests/storage/testconf.nim b/tests/storage/testconf.nim index ac096568..e82f29f8 100644 --- a/tests/storage/testconf.nim +++ b/tests/storage/testconf.nim @@ -10,6 +10,7 @@ proc validConfig(): StorageConf = natMaxQueueSize: 3, natNumPeersToAsk: 5, natMinConfidence: 0.7, + natObservedAddrMinCount: 1, ) suite "Conf - validateAutonatConfig": @@ -42,6 +43,19 @@ suite "Conf - validateAutonatConfig": check config.validateAutonatConfig().isOk + test "rejects no-bootstrap-node without extip": + var config = validConfig() + config.noBootstrapNode = true + + check config.validateAutonatConfig().isErr + + test "accepts no-bootstrap-node with extip": + var config = validConfig() + config.noBootstrapNode = true + config.nat = nat.NatConfig(hasExtIp: true, extIp: parseIpAddress("1.2.3.4")) + + check config.validateAutonatConfig().isOk + test "rejects nat-max-queue-size below 1": var config = validConfig() config.natMaxQueueSize = 0 @@ -66,6 +80,18 @@ suite "Conf - validateAutonatConfig": check config.validateAutonatConfig().isOk + test "rejects nat-observed-addr-min-count below 1": + var config = validConfig() + config.natObservedAddrMinCount = 0 + + check config.validateAutonatConfig().isErr + + test "accepts nat-observed-addr-min-count of 1": + var config = validConfig() + config.natObservedAddrMinCount = 1 + + check config.validateAutonatConfig().isOk + test "rejects negative nat-min-confidence": var config = validConfig() config.natMinConfidence = -0.1 From 725633d58741d7a83a69a0b6f4593a92db0b5bc5 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Tue, 16 Jun 2026 11:26:26 +0400 Subject: [PATCH 150/167] Cleanup --- tests/integration/nat/hole-punching/compose.yml | 8 +------- .../nat/hole-punching/testholepunching.nim | 15 +++------------ .../integration/nat/not-downloadable/compose.yml | 9 +-------- .../nat/not-downloadable/testnotdownloadable.nim | 12 +----------- tests/integration/nat/not-reachable/compose.yml | 7 +------ .../nat/not-reachable/testnotreachable.nim | 6 +----- tests/integration/nat/pcp/compose.yml | 7 +------ tests/integration/nat/pcp/testpcp.nim | 11 +---------- tests/integration/nat/reachable/compose.yml | 5 +---- tests/integration/nat/reachable/testreachable.nim | 12 +----------- tests/integration/nat/relay-download/compose.yml | 8 +------- .../nat/relay-download/testrelaydownload.nim | 12 +----------- tests/integration/nat/upnp/compose.yml | 6 +----- tests/integration/nat/upnp/testupnp.nim | 11 +---------- 14 files changed, 16 insertions(+), 113 deletions(-) diff --git a/tests/integration/nat/hole-punching/compose.yml b/tests/integration/nat/hole-punching/compose.yml index 49b49cc5..0d09ef50 100644 --- a/tests/integration/nat/hole-punching/compose.yml +++ b/tests/integration/nat/hole-punching/compose.yml @@ -1,10 +1,4 @@ -# A node behind a NAT is reachable only through A's relay. When the reachable -# node C dials it through the relay, the relayed node dials C back directly -# (C is public) and the relayed connection is upgraded to a direct one. Same -# topology as relay-download. Run via testholepunching.nim. -# -# node B ──── lan ──── router (NAT) ──── wan ──── bootstrap A (relay) -# └────── node C (reachable) +# NAT hole-punching scenario — see README.md. Run via testholepunching.nim. name: nat-hole-punching # Topology addresses, named for their role (defined once, referenced below). diff --git a/tests/integration/nat/hole-punching/testholepunching.nim b/tests/integration/nat/hole-punching/testholepunching.nim index 47910b79..c6b1550d 100644 --- a/tests/integration/nat/hole-punching/testholepunching.nim +++ b/tests/integration/nat/hole-punching/testholepunching.nim @@ -1,16 +1,7 @@ -## NAT hole-punching scenario — a node reached through the relay is upgraded to -## a direct connection. +## NAT hole-punching scenario. See README.md. ## -## B sits behind a NAT and is reachable only via A's relay. When the reachable -## node C dials B through the relay, B's hole-punching handler dials C back -## directly (C is public) and the relayed connection is replaced by a direct one. -## -## Hole punching has no REST surface, so success is asserted on B's container log -## line below (DEBUG). Brittle if that message ever changes. -## -## Requires podman-compose and the scenario image: -## podman build -t localhost/storage-nat \ -## -f tests/integration/nat/Dockerfile . +## Success is asserted on node B's container log line (no REST surface for the +## connection type); brittle if that message changes. import std/[json, os, sequtils, strutils, times] import pkg/chronos diff --git a/tests/integration/nat/not-downloadable/compose.yml b/tests/integration/nat/not-downloadable/compose.yml index eb29d5f2..414f127c 100644 --- a/tests/integration/nat/not-downloadable/compose.yml +++ b/tests/integration/nat/not-downloadable/compose.yml @@ -1,11 +1,4 @@ -# A node behind a NAT with no relay can't be reached from outside, so it can't -# be downloaded from: it announces no dialable address, the reachable node C -# finds it as a provider but never dials it. Same real iptables NAT as -# not-reachable, but bootstrap A runs *without* the relay server. Run via -# testnotdownloadable.nim. -# -# node B ──── lan ──── router (NAT) ──── wan ──── bootstrap A -# └────── node C (reachable) +# NAT not-downloadable scenario — see README.md. Run via testnotdownloadable.nim. name: nat-not-downloadable # Topology addresses, named for their role (defined once, referenced below). diff --git a/tests/integration/nat/not-downloadable/testnotdownloadable.nim b/tests/integration/nat/not-downloadable/testnotdownloadable.nim index d6a4fb30..79045b56 100644 --- a/tests/integration/nat/not-downloadable/testnotdownloadable.nim +++ b/tests/integration/nat/not-downloadable/testnotdownloadable.nim @@ -1,14 +1,4 @@ -## NAT not-downloadable scenario — a node behind a NAT with no relay cannot be -## downloaded from. -## -## Same shape as the not-reachable test: compose.yml brings up a real NAT -## topology, but bootstrap A runs without the relay server. B stays NotReachable -## and announces no dialable address, so a reachable peer C finds it as a -## provider but can never dial it — the manifest fetch fails. -## -## Requires podman-compose and the scenario image: -## podman build -t localhost/storage-nat \ -## -f tests/integration/nat/Dockerfile . +## NAT not-downloadable scenario. See README.md. import std/[json, os, times] import pkg/chronos diff --git a/tests/integration/nat/not-reachable/compose.yml b/tests/integration/nat/not-reachable/compose.yml index 400ef872..80294483 100644 --- a/tests/integration/nat/not-reachable/compose.yml +++ b/tests/integration/nat/not-reachable/compose.yml @@ -1,9 +1,4 @@ -# A node behind a NAT that can't be reached from outside must be detected -# NotReachable and fall back to the relay. This checks it on a real container -# network with real iptables NAT, not the in-process simulation the unit tests -# use. Run via testnotreachable.nim. -# -# node B ──── lan ──── router (NAT) ──── wan ──── bootstrap A +# NAT not-reachable scenario — see README.md. Run via testnotreachable.nim. name: nat-not-reachable # Topology addresses, named for their role (defined once, referenced below). diff --git a/tests/integration/nat/not-reachable/testnotreachable.nim b/tests/integration/nat/not-reachable/testnotreachable.nim index 9298b37e..f485fb90 100644 --- a/tests/integration/nat/not-reachable/testnotreachable.nim +++ b/tests/integration/nat/not-reachable/testnotreachable.nim @@ -1,8 +1,4 @@ -## NAT not-reachable scenario — node behind a real NAT falls back to relay. -## -## Requires podman-compose and the scenario image: -## podman build -t localhost/storage-nat:not-reachable \ -## -f tests/integration/nat/not-reachable/Dockerfile . +## NAT not-reachable scenario. See README.md. import std/[json, os, sequtils, strutils, times] import pkg/chronos diff --git a/tests/integration/nat/pcp/compose.yml b/tests/integration/nat/pcp/compose.yml index 1cf8d43e..19d27558 100644 --- a/tests/integration/nat/pcp/compose.yml +++ b/tests/integration/nat/pcp/compose.yml @@ -1,9 +1,4 @@ -# Same NAT topology as upnp, but miniupnpd has PCP enabled and the node maps its -# port over PCP (libplum's preferred protocol), which installs a real DNAT on the -# router, so AutoNAT's dial-back reaches it and it is detected Reachable — no -# relay. Run via testpcp.nim. -# -# node B ──── lan ──── router (NAT + miniupnpd/PCP) ──── wan ──── bootstrap A +# NAT pcp scenario — see README.md. Run via testpcp.nim. name: nat-pcp # Topology addresses, named for their role (defined once, referenced below). diff --git a/tests/integration/nat/pcp/testpcp.nim b/tests/integration/nat/pcp/testpcp.nim index 64c99c21..612aec67 100644 --- a/tests/integration/nat/pcp/testpcp.nim +++ b/tests/integration/nat/pcp/testpcp.nim @@ -1,13 +1,4 @@ -## NAT pcp scenario — node behind a real NAT becomes Reachable by mapping its -## port over PCP. -## -## Same shape as the upnp test, but miniupnpd has PCP enabled and the node maps -## its TCP/UDP ports via PCP (libplum's preferred protocol), which installs a real -## DNAT on the router. AutoNAT's dial-back then reaches the node, so it is -## detected Reachable with an active PCP mapping — no relay. -## -## Requires podman-compose and the scenario image: -## podman build -t localhost/storage-nat -f tests/integration/nat/Dockerfile . +## NAT pcp scenario. See README.md. import std/[json, os, sequtils, strutils, times] import pkg/chronos diff --git a/tests/integration/nat/reachable/compose.yml b/tests/integration/nat/reachable/compose.yml index 77da9992..f5936baa 100644 --- a/tests/integration/nat/reachable/compose.yml +++ b/tests/integration/nat/reachable/compose.yml @@ -1,7 +1,4 @@ -# Same setup as not-reachable, but the router forwards B's port (DNAT), so -# AutoNAT's dial-back reaches B and it is detected Reachable — no relay needed. -# -# node B ──── lan ──── router (NAT + port forward) ──── wan ──── bootstrap A +# NAT reachable scenario — see README.md. Run via testreachable.nim. name: nat-reachable # Topology addresses, named for their role (defined once, referenced below). diff --git a/tests/integration/nat/reachable/testreachable.nim b/tests/integration/nat/reachable/testreachable.nim index 2601f0fe..d1f87456 100644 --- a/tests/integration/nat/reachable/testreachable.nim +++ b/tests/integration/nat/reachable/testreachable.nim @@ -1,14 +1,4 @@ -## NAT reachable scenario — node behind a real NAT is Reachable because the -## router forwards its port. -## -## Same shape as the not-reachable test: compose.yml brings up a real NAT -## topology, but the router has a static inbound port-forward (DNAT) to the node. -## AutoNAT's dial-back reaches the node, so it is detected Reachable (no relay) — -## a manual port-forward / endpoint-independent NAT, no miniupnpd. -## -## Requires podman-compose and the scenario image: -## podman build -t localhost/storage-nat:reachable \ -## -f tests/integration/nat/reachable/Dockerfile . +## NAT reachable scenario. See README.md. import std/[json, os, sequtils, strutils, times] import pkg/chronos diff --git a/tests/integration/nat/relay-download/compose.yml b/tests/integration/nat/relay-download/compose.yml index d079d587..2e2652c7 100644 --- a/tests/integration/nat/relay-download/compose.yml +++ b/tests/integration/nat/relay-download/compose.yml @@ -1,10 +1,4 @@ -# A node behind a NAT falls back to bootstrap A's relay and announces its -# circuit address, so a reachable node C can download from it through the relay. -# Same real iptables NAT as not-reachable, with C added as the downloader. Run -# via testrelaydownload.nim. -# -# node B ──── lan ──── router (NAT) ──── wan ──── bootstrap A (relay) -# └────── node C (reachable) +# NAT relay-download scenario — see README.md. Run via testrelaydownload.nim. name: nat-relay-download # Topology addresses, named for their role (defined once, referenced below). diff --git a/tests/integration/nat/relay-download/testrelaydownload.nim b/tests/integration/nat/relay-download/testrelaydownload.nim index 3b9ff498..0f7349b5 100644 --- a/tests/integration/nat/relay-download/testrelaydownload.nim +++ b/tests/integration/nat/relay-download/testrelaydownload.nim @@ -1,14 +1,4 @@ -## NAT relay-download scenario — a node behind a NAT can be downloaded from -## through the relay. -## -## Same shape as the not-reachable test: compose.yml brings up a real NAT -## topology with bootstrap A running the relay server. B stays NotReachable, -## falls back to the relay and announces its circuit address, so a reachable -## peer C can fetch its data through the relay. -## -## Requires podman-compose and the scenario image: -## podman build -t localhost/storage-nat \ -## -f tests/integration/nat/Dockerfile . +## NAT relay-download scenario. See README.md. import std/[json, os, sequtils, strutils, times] import pkg/chronos diff --git a/tests/integration/nat/upnp/compose.yml b/tests/integration/nat/upnp/compose.yml index 50ca1df8..8de469f1 100644 --- a/tests/integration/nat/upnp/compose.yml +++ b/tests/integration/nat/upnp/compose.yml @@ -1,8 +1,4 @@ -# Same NAT topology as not-reachable, but the router runs miniupnpd. The node -# maps its port over UPnP, which installs a real DNAT on the router, so AutoNAT's -# dial-back reaches it and it is detected Reachable — no relay. Run via testupnp.nim. -# -# node B ──── lan ──── router (NAT + miniupnpd) ──── wan ──── bootstrap A +# NAT upnp scenario — see README.md. Run via testupnp.nim. name: nat-upnp # Topology addresses, named for their role (defined once, referenced below). diff --git a/tests/integration/nat/upnp/testupnp.nim b/tests/integration/nat/upnp/testupnp.nim index 0fd623ab..8e72e5de 100644 --- a/tests/integration/nat/upnp/testupnp.nim +++ b/tests/integration/nat/upnp/testupnp.nim @@ -1,13 +1,4 @@ -## NAT upnp scenario — node behind a real NAT becomes Reachable by mapping its -## port over UPnP. -## -## Same shape as the reachable test, but the router opens no port itself: it runs -## miniupnpd and the node maps its TCP/UDP ports via UPnP, which installs a real -## DNAT on the router. AutoNAT's dial-back then reaches the node, so it is -## detected Reachable with an active UPnP mapping — no relay. -## -## Requires podman-compose and the scenario image: -## podman build -t localhost/storage-nat -f tests/integration/nat/Dockerfile . +## NAT upnp scenario. See README.md. import std/[json, os, sequtils, strutils, times] import pkg/chronos From 71a479a558e4e49af33b2b772473e6071fab607e Mon Sep 17 00:00:00 2001 From: Arnaud Date: Tue, 16 Jun 2026 11:46:17 +0400 Subject: [PATCH 151/167] Cleanup --- .../nat/hole-punching/testholepunching.nim | 5 +- tests/integration/storageclient.nim | 46 ---------- tests/integration/storageconfig.nim | 90 ------------------- 3 files changed, 2 insertions(+), 139 deletions(-) diff --git a/tests/integration/nat/hole-punching/testholepunching.nim b/tests/integration/nat/hole-punching/testholepunching.nim index c6b1550d..3d5dc3a4 100644 --- a/tests/integration/nat/hole-punching/testholepunching.nim +++ b/tests/integration/nat/hole-punching/testholepunching.nim @@ -1,7 +1,4 @@ ## NAT hole-punching scenario. See README.md. -## -## Success is asserted on node B's container log line (no REST surface for the -## connection type); brittle if that message changes. import std/[json, os, sequtils, strutils, times] import pkg/chronos @@ -49,6 +46,8 @@ asyncchecksuite "NAT hole punching": info{"nat"}{"reachability"}.getStr == "NotReachable" and info.announcesCircuitAddr(), ) + + # C is Reachable check eventuallyInfo(clientC, info{"nat"}{"reachability"}.getStr == "Reachable") # C dials B through the relay; a download is enough to open the connection diff --git a/tests/integration/storageclient.nim b/tests/integration/storageclient.nim index 50b03376..17931f34 100644 --- a/tests/integration/storageclient.nim +++ b/tests/integration/storageclient.nim @@ -69,16 +69,6 @@ proc delete( .} = return self.request(MethodDelete, url, headers = headers) -proc patch*( - self: StorageClient, - url: string, - body: string = "", - headers: seq[HttpHeaderTuple] = @[], -): Future[HttpClientResponseRef] {. - async: (raw: true, raises: [CancelledError, HttpError]) -.} = - return self.request(MethodPatch, url, headers = headers, body = body) - proc body*( response: HttpClientResponseRef ): Future[string] {.async: (raises: [CancelledError, HttpError]).} = @@ -230,20 +220,6 @@ proc list*( RestContentList.fromJson(await response.body) -proc space*( - client: StorageClient -): Future[?!RestRepoStore] {.async: (raises: [CancelledError, HttpError]).} = - let url = client.baseurl & "/space" - let response = await client.get(url) - - if response.status != 200: - return failure($response.status) - - RestRepoStore.fromJson(await response.body) - -proc buildUrl*(client: StorageClient, path: string): string = - return client.baseurl & path - proc hasBlock*( client: StorageClient, cid: Cid ): Future[?!bool] {.async: (raises: [CancelledError, HttpError]).} = @@ -261,25 +237,3 @@ proc hasBlockRaw*( .} = let url = client.baseurl & "/data/" & cid & "/exists" return client.get(url) - -proc natRelayRunning*( - client: StorageClient -): Future[?!bool] {.async: (raises: [CancelledError, HttpError]).} = - let info = await client.info() - if info.isErr: - return failure "Failed to get node info" - try: - return info.get()["nat"]["relayRunning"].getBool().success - except KeyError as e: - return failure e.msg - -proc natPortMapping*( - client: StorageClient -): Future[?!string] {.async: (raises: [CancelledError, HttpError]).} = - let info = await client.info() - if info.isErr: - return failure "Failed to get node info" - try: - return info.get()["nat"]["portMapping"].getStr().success - except KeyError as e: - return failure e.msg diff --git a/tests/integration/storageconfig.nim b/tests/integration/storageconfig.nim index 2f64ef79..8c50fa0d 100644 --- a/tests/integration/storageconfig.nim +++ b/tests/integration/storageconfig.nim @@ -5,9 +5,7 @@ import std/strutils import std/sugar import std/tables from pkg/chronicles import LogLevel -import pkg/chronos import pkg/storage/conf -import pkg/storage/units import pkg/confutils import pkg/confutils/defs import libp2p except setup @@ -235,85 +233,6 @@ proc withBlockMaintenanceInterval*( config.addCliOption("--block-mi", $interval) return startConfig -proc logLevelWithTopics( - config: StorageConfig, topics: varargs[string] -): string {.raises: [StorageConfigError].} = - convertError: - var logLevel = LogLevel.INFO - let built = config.buildConfig("Invalid storage config cli params") - logLevel = parseEnum[LogLevel](built.logLevel.toUpperAscii) - let level = $logLevel & ";TRACE: " & topics.join(",") - return level - -proc withLogTopics*( - self: StorageConfigs, idx: int, topics: varargs[string] -): StorageConfigs {.raises: [StorageConfigError].} = - self.checkBounds idx - - convertError: - let config = self.configs[idx] - let level = config.logLevelWithTopics(topics) - var startConfig = self - return startConfig.withLogLevel(idx, level) - -proc withLogTopics*( - self: StorageConfigs, topics: varargs[string] -): StorageConfigs {.raises: [StorageConfigError].} = - var startConfig = self - for config in startConfig.configs.mitems: - let level = config.logLevelWithTopics(topics) - config = config.withLogLevel(level) - return startConfig - -proc withStorageQuota*( - self: StorageConfigs, idx: int, quota: NBytes -): StorageConfigs {.raises: [StorageConfigError].} = - self.checkBounds idx - - var startConfig = self - startConfig.configs[idx].addCliOption("--storage-quota", $quota) - return startConfig - -proc withStorageQuota*( - self: StorageConfigs, quota: NBytes -): StorageConfigs {.raises: [StorageConfigError].} = - var startConfig = self - for config in startConfig.configs.mitems: - config.addCliOption("--storage-quota", $quota) - return startConfig - -proc withNatNumPeersToAsk*( - self: StorageConfigs, numPeersToAsk: int -): StorageConfigs {.raises: [StorageConfigError].} = - var startConfig = self - for config in startConfig.configs.mitems: - config.addCliOption("--nat-num-peers-to-ask", $numPeersToAsk) - return startConfig - -proc withNatMaxQueueSize*( - self: StorageConfigs, maxQueueSize: int -): StorageConfigs {.raises: [StorageConfigError].} = - var startConfig = self - for config in startConfig.configs.mitems: - config.addCliOption("--nat-max-queue-size", $maxQueueSize) - return startConfig - -proc withNatMinConfidence*( - self: StorageConfigs, minConfidence: float -): StorageConfigs {.raises: [StorageConfigError].} = - var startConfig = self - for config in startConfig.configs.mitems: - config.addCliOption("--nat-min-confidence", $minConfidence) - return startConfig - -proc withNatScheduleInterval*( - self: StorageConfigs, scheduleInterval: Duration -): StorageConfigs {.raises: [StorageConfigError].} = - var startConfig = self - for config in startConfig.configs.mitems: - config.addCliOption("--nat-schedule-interval", $scheduleInterval) - return startConfig - proc withExtIp*( self: StorageConfigs, idx: int, ip = "127.0.0.1" ): StorageConfigs {.raises: [StorageConfigError].} = @@ -323,15 +242,6 @@ proc withExtIp*( startConfig.configs[idx].addCliOption("--nat", "extip:" & ip) return startConfig -proc withRelay*( - self: StorageConfigs, idx: int -): StorageConfigs {.raises: [StorageConfigError].} = - self.checkBounds idx - - var startConfig = self - startConfig.configs[idx].addCliOption("--relay-server") - return startConfig - # For testing, a node with extip (not behind nat) with autonat server # enabled is a bootstrap node proc isBootstrapNode*(config: StorageConfig): bool {.raises: [].} = From 02d90291bfdc55ae780fc099a11a85bf731fdd2c Mon Sep 17 00:00:00 2001 From: Arnaud Date: Tue, 16 Jun 2026 11:57:39 +0400 Subject: [PATCH 152/167] Cleanup --- tests/storage/testnatreaction.nim | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/tests/storage/testnatreaction.nim b/tests/storage/testnatreaction.nim index 13e99ef8..0a0781c1 100644 --- a/tests/storage/testnatreaction.nim +++ b/tests/storage/testnatreaction.nim @@ -17,7 +17,6 @@ import ../../storage/utils type MockNatPortMapper = ref object of NatPortMapper mappedPorts: Option[(Port, Port, MappingProtocol)] - activeMapping: bool method mapNatPorts*( m: MockNatPortMapper @@ -26,14 +25,14 @@ method mapNatPorts*( .} = m.mappedPorts -method hasMappingIds*(m: MockNatPortMapper): bool = - m.activeMapping +method destroyMappingFor(m: MockNatPortMapper, id: cint) {.gcsafe.} = + discard type MockMapNatPortMapper = ref object of NatPortMapper tcpResult: Result[MappingResult, string] udpResult: Result[MappingResult, string] live: bool - created: seq[PlumProtocol] + createAttempts: seq[PlumProtocol] destroyed: seq[cint] method initPlum(m: MockMapNatPortMapper): Result[void, string] {.gcsafe.} = @@ -45,7 +44,7 @@ method hasLiveMapping(m: MockMapNatPortMapper, id: cint): bool {.gcsafe.} = method createMappingFor( m: MockMapNatPortMapper, protocol: PlumProtocol, port: uint16 ): Future[Result[MappingResult, string]] {.async: (raises: [CancelledError]), gcsafe.} = - m.created.add(protocol) + m.createAttempts.add(protocol) if protocol == TCP: m.tcpResult else: m.udpResult method destroyMappingFor(m: MockMapNatPortMapper, id: cint) {.gcsafe.} = @@ -130,8 +129,11 @@ asyncchecksuite "NAT reaction - port mapping": check disc.protocol.clientMode test "handleNatStatus tears down an active mapping and starts relay when NotReachable with dialBackAddr": + privateAccess(NatPortMapper) let dialBack = MultiAddress.init("/ip4/1.2.3.4/tcp/8080").expect("valid") - let mapper = MockNatPortMapper(activeMapping: true) + let mapper = MockNatPortMapper() + mapper.tcpMappingId = some(cint(1)) + mapper.udpMappingId = some(cint(2)) autorelayservice.setup(autoRelay, sw) await mapper.handleNatStatus( @@ -141,9 +143,13 @@ asyncchecksuite "NAT reaction - port mapping": check autoRelay.isRunning check disc.announceAddrs == newSeq[MultiAddress]() check disc.protocol.clientMode + check not mapper.hasMappingIds() # the active mapping was torn down test "handleNatStatus tears down an active mapping and starts relay when NotReachable without dialBackAddr": - let mapper = MockNatPortMapper(activeMapping: true) + privateAccess(NatPortMapper) + let mapper = MockNatPortMapper() + mapper.tcpMappingId = some(cint(1)) + mapper.udpMappingId = some(cint(2)) autorelayservice.setup(autoRelay, sw) await mapper.handleNatStatus( @@ -153,6 +159,7 @@ asyncchecksuite "NAT reaction - port mapping": check autoRelay.isRunning check disc.announceAddrs == newSeq[MultiAddress]() check disc.protocol.clientMode + check not mapper.hasMappingIds() # the active mapping was torn down test "handleNatStatus stops relay and exits client mode when mapping is created and node is Reachable": let dialBack = MultiAddress.init("/ip4/1.2.3.4/tcp/8080").expect("valid") @@ -359,7 +366,7 @@ asyncchecksuite "NAT - mapNatPorts": ) check (await mapper.mapNatPorts()).isNone - check mapper.created == @[PlumProtocol.TCP] # UDP never attempted + check mapper.createAttempts == @[PlumProtocol.TCP] # UDP never attempted check mapper.destroyed.len == 0 # nothing to clean up test "does not map when configured with an external IP": @@ -368,7 +375,7 @@ asyncchecksuite "NAT - mapNatPorts": ) check (await mapper.mapNatPorts()).isNone - check mapper.created.len == 0 # short-circuits before any mapping + check mapper.createAttempts.len == 0 # short-circuits before any mapping test "reuses the existing mapping when both are still live": privateAccess(NatPortMapper) @@ -378,9 +385,9 @@ asyncchecksuite "NAT - mapNatPorts": activeUdpPort: some(Port(9001)), activeMappingProtocol: some(MappingProtocol.UPnP), ) - mapper.tcpMappingId = some(cint(1)) # private field, set via privateAccess + mapper.tcpMappingId = some(cint(1)) mapper.udpMappingId = some(cint(2)) check (await mapper.mapNatPorts()) == some((Port(9000), Port(9001), MappingProtocol.UPnP)) - check mapper.created.len == 0 # reuse path, nothing recreated + check mapper.createAttempts.len == 0 From 6395da590a50c45519bf879003f1af9f349e306d Mon Sep 17 00:00:00 2001 From: Arnaud Date: Wed, 17 Jun 2026 02:11:34 +0400 Subject: [PATCH 153/167] Rename test --- .../README.md | 13 ++++++++----- .../compose.yml | 4 ++-- .../router-entrypoint.sh | 0 .../testconnectionreversal.nim} | 6 +++--- 4 files changed, 13 insertions(+), 10 deletions(-) rename tests/integration/nat/{hole-punching => connection-reversal}/README.md (73%) rename tests/integration/nat/{hole-punching => connection-reversal}/compose.yml (96%) rename tests/integration/nat/{hole-punching => connection-reversal}/router-entrypoint.sh (100%) rename tests/integration/nat/{hole-punching/testholepunching.nim => connection-reversal/testconnectionreversal.nim} (93%) diff --git a/tests/integration/nat/hole-punching/README.md b/tests/integration/nat/connection-reversal/README.md similarity index 73% rename from tests/integration/nat/hole-punching/README.md rename to tests/integration/nat/connection-reversal/README.md index 93fa7097..459987d6 100644 --- a/tests/integration/nat/hole-punching/README.md +++ b/tests/integration/nat/connection-reversal/README.md @@ -1,4 +1,4 @@ -# NAT hole-punching scenario +# NAT connection-reversal scenario ## Scenario @@ -6,6 +6,9 @@ A node behind a NAT is reachable only through A's relay. When the reachable node C dials it through the relay, the relayed node dials C back directly (C is public) and the relayed connection is upgraded to a direct one. +This is a unilateral reversal — only the NATed node dials, because C is public. +It is not a coordinated hole punch; see `../hole-punch` for the both-NATed case. + ## Topology ``` @@ -25,7 +28,7 @@ node B ──── lan ──── router (NAT) ──── wan ──── ```bash make testNatIntegration \ - STORAGE_INTEGRATION_TEST_INCLUDES=tests/integration/nat/hole-punching/testholepunching.nim + STORAGE_INTEGRATION_TEST_INCLUDES=tests/integration/nat/connection-reversal/testconnectionreversal.nim ``` Builds the shared image and brings the compose topology up and down. Rootless, but @@ -36,8 +39,8 @@ needs the host netfilter modules — if the router fails on iptables: B is `NotReachable` behind the relay, C is `Reachable`. C downloads from B through the relay, which opens a relayed connection; B then dials C back -directly. Hole punching has no REST surface, so the test asserts on B's log line -`Direct connection created.`. +directly. The direct upgrade has no REST surface, so the test asserts on B's log +line `Direct connection created.`. Per-run container logs (router, bootstrap, client, node) are written before teardown to -`tests/integration/logs/__NAT_hole_punching//.log`. +`tests/integration/logs/__NAT_connection_reversal//.log`. diff --git a/tests/integration/nat/hole-punching/compose.yml b/tests/integration/nat/connection-reversal/compose.yml similarity index 96% rename from tests/integration/nat/hole-punching/compose.yml rename to tests/integration/nat/connection-reversal/compose.yml index 0d09ef50..4a75ba32 100644 --- a/tests/integration/nat/hole-punching/compose.yml +++ b/tests/integration/nat/connection-reversal/compose.yml @@ -1,5 +1,5 @@ -# NAT hole-punching scenario — see README.md. Run via testholepunching.nim. -name: nat-hole-punching +# NAT connection-reversal scenario — see README.md. Run via testconnectionreversal.nim. +name: nat-connection-reversal # Topology addresses, named for their role (defined once, referenced below). x-addresses: diff --git a/tests/integration/nat/hole-punching/router-entrypoint.sh b/tests/integration/nat/connection-reversal/router-entrypoint.sh similarity index 100% rename from tests/integration/nat/hole-punching/router-entrypoint.sh rename to tests/integration/nat/connection-reversal/router-entrypoint.sh diff --git a/tests/integration/nat/hole-punching/testholepunching.nim b/tests/integration/nat/connection-reversal/testconnectionreversal.nim similarity index 93% rename from tests/integration/nat/hole-punching/testholepunching.nim rename to tests/integration/nat/connection-reversal/testconnectionreversal.nim index 3d5dc3a4..0c4c1306 100644 --- a/tests/integration/nat/hole-punching/testholepunching.nim +++ b/tests/integration/nat/connection-reversal/testconnectionreversal.nim @@ -1,4 +1,4 @@ -## NAT hole-punching scenario. See README.md. +## NAT connection-reversal scenario. See README.md. import std/[json, os, sequtils, strutils, times] import pkg/chronos @@ -15,12 +15,12 @@ proc announcesCircuitAddr(info: JsonNode): bool = ## A node behind the relay announces its circuit (p2p-circuit) address. info{"announceAddresses"}.getElems.anyIt("p2p-circuit" in it.getStr) -asyncchecksuite "NAT hole punching": +asyncchecksuite "NAT connection reversal": let composeFile = currentSourcePath.parentDir / "compose.yml" nodeApiUrl = "http://127.0.0.1:18088/api/storage/v1" clientApiUrl = "http://127.0.0.1:18089/api/storage/v1" - suiteName = "NAT hole punching" + suiteName = "NAT connection reversal" testName = "a relayed node is upgraded to a direct connection" services = ["router", "bootstrap", "client", "node"] startTime = now().format("yyyy-MM-dd'_'HH:mm:ss") From 182cd8df259c56643755c45de6b3323a1751bcf6 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Wed, 17 Jun 2026 02:11:51 +0400 Subject: [PATCH 154/167] Add hole punch test --- tests/integration/nat/hole-punch/README.md | 43 ++++++ tests/integration/nat/hole-punch/compose.yml | 126 ++++++++++++++++++ .../nat/hole-punch/router-entrypoint.sh | 11 ++ .../nat/hole-punch/testholepunch.nim | 64 +++++++++ 4 files changed, 244 insertions(+) create mode 100644 tests/integration/nat/hole-punch/README.md create mode 100644 tests/integration/nat/hole-punch/compose.yml create mode 100644 tests/integration/nat/hole-punch/router-entrypoint.sh create mode 100644 tests/integration/nat/hole-punch/testholepunch.nim diff --git a/tests/integration/nat/hole-punch/README.md b/tests/integration/nat/hole-punch/README.md new file mode 100644 index 00000000..809b7aea --- /dev/null +++ b/tests/integration/nat/hole-punch/README.md @@ -0,0 +1,43 @@ +# NAT hole-punch scenario + +## Scenario + +Two nodes are each behind their **own** NAT, so neither can reach the other on a +shared lan. Both are `NotReachable` and take a relay reservation on A. When D +downloads from B through the relay, B drives a **coordinated** DCUtR +simultaneous-open and starts hole-punching. + +## Topology + +``` +node B ── lan1 ── router1 (NAT) ──┐ + ├── wan ── bootstrap A (relay + autonat) +node D ── lan2 ── router2 (NAT) ──┘ +``` + +- **bootstrap A** — public node on the wan, autonat + relay server. +- **router1 / router2** — `lan -> wan` masquerade, *no* inbound forward. +- **node B** — behind router1, NotReachable, uploaded to and downloaded from. + Holds the inbound relayed connection, so it is the DCUtR initiator. +- **node D** — behind router2, NotReachable, downloads from B through the relay. + +## Run + +```bash +make testNatIntegration \ + STORAGE_INTEGRATION_TEST_INCLUDES=tests/integration/nat/hole-punch/testholepunch.nim +``` + +Rootless, but needs the host netfilter modules — if a router fails on iptables: +`sudo modprobe iptable_nat nf_conntrack`. + +## Expected result + +Both nodes are `NotReachable`. D downloads from B through the relay, opening a +relayed connection; B then runs DCUtR and the connection is upgraded to a direct +one. The test asserts B's log line +`Dcutr initiator has directly connected to the remote peer.` — the line is +unique to the simultaneous-open path, so it cannot be produced by a reversal. + +Per-run container logs are written before teardown to +`tests/integration/logs/__NAT_hole_punching//.log`. diff --git a/tests/integration/nat/hole-punch/compose.yml b/tests/integration/nat/hole-punch/compose.yml new file mode 100644 index 00000000..9a3b641c --- /dev/null +++ b/tests/integration/nat/hole-punch/compose.yml @@ -0,0 +1,126 @@ +# NAT hole-punch scenario — see README.md. Run via testholepunch.nim. +name: nat-hole-punch + +# Two NATed nodes, one behind each router (separate lans), so neither can reach +# the other on a shared lan: the only direct path is a coordinated hole punch. +x-addresses: + wan_subnet: &wan_subnet 7.7.7.0/24 + lan1_subnet: &lan1_subnet 10.99.1.0/24 + lan2_subnet: &lan2_subnet 10.99.2.0/24 + # A: public bootstrap, autonat + relay server + bootstrap_ip: &bootstrap_ip 7.7.7.10 + # router1 (B's NAT) + router1_wan_ip: &router1_wan_ip 7.7.7.3 + router1_lan_ip: &router1_lan_ip 10.99.1.2 + # router2 (D's NAT) + router2_wan_ip: &router2_wan_ip 7.7.7.4 + router2_lan_ip: &router2_lan_ip 10.99.2.2 + # B behind router1, D behind router2 + node_ip: &node_ip 10.99.1.10 + peer_ip: &peer_ip 10.99.2.10 + +networks: + wan: + internal: true + ipam: + config: + - subnet: *wan_subnet + lan1: + ipam: + config: + - subnet: *lan1_subnet + lan2: + ipam: + config: + - subnet: *lan2_subnet + +services: + router1: + image: localhost/storage-nat + cap_add: [NET_ADMIN] + sysctls: + net.ipv4.ip_forward: 1 + networks: + wan: + ipv4_address: *router1_wan_ip + lan1: + ipv4_address: *router1_lan_ip + environment: + ROUTER_WAN_IP: *router1_wan_ip + LAN_SUBNET: *lan1_subnet + volumes: + - ../router-common.sh:/scripts/router-common.sh:ro,z + - ./router-entrypoint.sh:/scripts/router-entrypoint.sh:ro,z + entrypoint: ["bash", "/scripts/router-entrypoint.sh"] + + router2: + image: localhost/storage-nat + cap_add: [NET_ADMIN] + sysctls: + net.ipv4.ip_forward: 1 + networks: + wan: + ipv4_address: *router2_wan_ip + lan2: + ipv4_address: *router2_lan_ip + environment: + ROUTER_WAN_IP: *router2_wan_ip + LAN_SUBNET: *lan2_subnet + volumes: + - ../router-common.sh:/scripts/router-common.sh:ro,z + - ./router-entrypoint.sh:/scripts/router-entrypoint.sh:ro,z + entrypoint: ["bash", "/scripts/router-entrypoint.sh"] + + bootstrap: + image: localhost/storage-nat + networks: + wan: + ipv4_address: *bootstrap_ip + entrypoint: ["/app/build/storage"] + command: + - --listen-ip=0.0.0.0 + - --api-bindaddr=0.0.0.0 + - --listen-port=8070 + - --disc-port=8090 + - --api-port=8080 + # bootstrap_ip (anchors can't go inside a string) + - --nat=extip:7.7.7.10 + - --relay-server + - --autonat-server + - --no-bootstrap-node + - --data-dir=/data + - --log-level=DEBUG + + # B: behind router1, uploaded to and downloaded from + node: + image: localhost/storage-nat + cap_add: [NET_ADMIN] + depends_on: [router1, bootstrap] + networks: + lan1: + ipv4_address: *node_ip + ports: + - "127.0.0.1:18090:8080" + environment: + ROUTER_LAN_IP: *router1_lan_ip + BOOTSTRAP_API: http://7.7.7.10:8080 + volumes: + - ../node-entrypoint.sh:/scripts/node-entrypoint.sh:ro,z + entrypoint: ["bash", "/scripts/node-entrypoint.sh"] + + # D: behind router2, the one that downloads from B through the relay + peer: + image: localhost/storage-nat + cap_add: [NET_ADMIN] + depends_on: [router2, bootstrap] + networks: + lan2: + ipv4_address: *peer_ip + ports: + - "127.0.0.1:18091:8080" + environment: + ROUTER_LAN_IP: *router2_lan_ip + BOOTSTRAP_API: http://7.7.7.10:8080 + volumes: + - ../node-entrypoint.sh:/scripts/node-entrypoint.sh:ro,z + entrypoint: ["bash", "/scripts/node-entrypoint.sh"] diff --git a/tests/integration/nat/hole-punch/router-entrypoint.sh b/tests/integration/nat/hole-punch/router-entrypoint.sh new file mode 100644 index 00000000..11d6d17d --- /dev/null +++ b/tests/integration/nat/hole-punch/router-entrypoint.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +source "$(dirname "$0")/router-common.sh" + +# Drop early punch SYN so TCP retransmits until the +# pinhole is open and the SYN gets forwarded. +iptables -A INPUT -i "$wanif" -p tcp --dport 8070 -j DROP + +echo "router ready (wan iface $wanif)" + +hold_until_stopped diff --git a/tests/integration/nat/hole-punch/testholepunch.nim b/tests/integration/nat/hole-punch/testholepunch.nim new file mode 100644 index 00000000..e72464a2 --- /dev/null +++ b/tests/integration/nat/hole-punch/testholepunch.nim @@ -0,0 +1,64 @@ +## Coordinated DCUtR hole-punching scenario (both peers NATed). See README.md. + +import std/[json, os, sequtils, strutils, times] +import pkg/chronos +import pkg/questionable/results + +import ../../../asynctest +import ../../../checktest +import ../../storageclient +import ../composehelper + +const dcutrConnectedLog = "Dcutr initiator has directly connected to the remote peer." + +proc announcesCircuitAddr(info: JsonNode): bool = + info{"announceAddresses"}.getElems.anyIt("p2p-circuit" in it.getStr) + +asyncchecksuite "NAT hole punching": + let + composeFile = currentSourcePath.parentDir / "compose.yml" + nodeApiUrl = "http://127.0.0.1:18090/api/storage/v1" + peerApiUrl = "http://127.0.0.1:18091/api/storage/v1" + suiteName = "NAT hole punching" + testName = "two NATed nodes upgrade a relayed connection to a direct one" + services = ["router1", "router2", "bootstrap", "node", "peer"] + startTime = now().format("yyyy-MM-dd'_'HH:mm:ss") + var + nodeClient: StorageClient + peerClient: StorageClient + + setup: + compose(composeFile, "up -d") + nodeClient = StorageClient.new(nodeApiUrl) + peerClient = StorageClient.new(peerApiUrl) + + teardown: + await nodeClient.close() + await peerClient.close() + saveContainerLogs(composeFile, suiteName, testName, startTime, services) + compose(composeFile, "down -v") + + test testName: + # Both nodes are NotReachable behind their own NAT and take a relay reservation. + check eventuallyInfo( + nodeClient, + info{"nat"}{"reachability"}.getStr == "NotReachable" and + info.announcesCircuitAddr(), + ) + check eventuallyInfo( + peerClient, + info{"nat"}{"reachability"}.getStr == "NotReachable" and + info.announcesCircuitAddr(), + ) + + # D downloads from B through the relay; that opens the relayed connection. + let cid = (await nodeClient.upload("punch me for real")).get + check (await peerClient.download(cid)).isOk + + # B sees the relayed peer D join and, since D has no public address, drives + # the coordinated DCUtR simultaneous-open instead of a unilateral reversal. + check eventuallySafe( + dcutrConnectedLog in serviceLogs(composeFile, "node"), + timeout = 60_000, + pollInterval = 2_000, + ) From 6307ffc418a96a12a29774356896eada8f124585 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Wed, 17 Jun 2026 02:43:52 +0400 Subject: [PATCH 155/167] Activate SO_REUSEPORT for hole punching --- storage/storage.nim | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/storage/storage.nim b/storage/storage.nim index 9c5690b0..4088dcb8 100644 --- a/storage/storage.nim +++ b/storage/storage.nim @@ -88,6 +88,13 @@ proc start*(s: StorageServer) {.async.} = await s.storageNode.switch.start() + # Activate SO_REUSEPORT for hole punching in tcptransport.nim. + # Without that, hole punching would use an ephemeral port assigned by the OS. + # NotReachable has nothing to do with AutoNAT Reachability + if s.holePunchHandler.isSome: + for t in s.storageNode.switch.transports: + t.networkReachability = NetworkReachability.NotReachable + # When listenPort is 0 the OS assigns a random port. For UDP, the port # doesn't change so there is no need to update it. if s.natMapper.isSome and s.config.listenPort == Port(0): From 0c938e6b1a94c9551e60b4122c265824d4643e7f Mon Sep 17 00:00:00 2001 From: Arnaud Date: Wed, 17 Jun 2026 02:56:29 +0400 Subject: [PATCH 156/167] Add peer connections in debug --- openapi.yaml | 16 ++++++++++++++++ storage/nat.nim | 10 +++++++++- storage/rest/api.nim | 1 + tests/integration/nat/hole-punch/README.md | 5 ++--- .../integration/nat/hole-punch/testholepunch.nim | 16 ++++++++-------- 5 files changed, 36 insertions(+), 12 deletions(-) diff --git a/openapi.yaml b/openapi.yaml index 9bb3ab03..5743de87 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -135,6 +135,10 @@ components: $ref: "#/components/schemas/StorageVersion" nat: $ref: "#/components/schemas/NatInfo" + connections: + type: array + items: + $ref: "#/components/schemas/Connection" NatInfo: type: object @@ -159,6 +163,18 @@ components: enum: [none, upnp, pmp, pcp, direct] description: Active NAT port mapping type + Connection: + type: object + required: + - peerId + - direct + properties: + peerId: + $ref: "#/components/schemas/PeerId" + direct: + type: boolean + description: Whether at least one connection to this peer is direct (not relayed) + DataList: type: object required: diff --git a/storage/nat.nim b/storage/nat.nim index 61a07955..63ffff9a 100644 --- a/storage/nat.nim +++ b/storage/nat.nim @@ -8,7 +8,7 @@ {.push raises: [].} -import std/[options, net, os, sequtils] +import std/[options, net, os, sequtils, json] import results import pkg/chronos @@ -269,6 +269,14 @@ proc portMappingStr*(natMapper: Option[NatPortMapper]): string = of MappingProtocol.Direct: "direct" of MappingProtocol.Unknown: "none" +proc peerConnections*(switch: Switch): JsonNode = + result = newJArray() + for peerId, muxers in switch.connManager.getConnections(): + let entry = newJObject() + entry["peerId"] = newJString($peerId) + entry["direct"] = newJBool(muxers.anyIt(not isRelayed(it.connection))) + result.add(entry) + proc findReachableNodes*(bootstrapNodes: seq[SignedPeerRecord]): seq[SignedPeerRecord] = ## Returns the list of nodes known to be directly reachable. ## Currently returns bootstrap nodes. In the future, any network participant diff --git a/storage/rest/api.nim b/storage/rest/api.nim index 597ff321..ff9ac7f4 100644 --- a/storage/rest/api.nim +++ b/storage/rest/api.nim @@ -592,6 +592,7 @@ proc initDebugApi( "relayRunning": autoRelay.isSome and autoRelay.get.isRunning, "portMapping": portMappingStr(natMapper), }, + "connections": peerConnections(node.switch), } # return pretty json for human readability diff --git a/tests/integration/nat/hole-punch/README.md b/tests/integration/nat/hole-punch/README.md index 809b7aea..b0c25fd7 100644 --- a/tests/integration/nat/hole-punch/README.md +++ b/tests/integration/nat/hole-punch/README.md @@ -35,9 +35,8 @@ Rootless, but needs the host netfilter modules — if a router fails on iptables Both nodes are `NotReachable`. D downloads from B through the relay, opening a relayed connection; B then runs DCUtR and the connection is upgraded to a direct -one. The test asserts B's log line -`Dcutr initiator has directly connected to the remote peer.` — the line is -unique to the simultaneous-open path, so it cannot be produced by a reversal. +one. The test polls B's `/debug/info` and asserts its connection to D becomes +non-relayed (`connections[].direct == true` for D's peerId). Per-run container logs are written before teardown to `tests/integration/logs/__NAT_hole_punching//.log`. diff --git a/tests/integration/nat/hole-punch/testholepunch.nim b/tests/integration/nat/hole-punch/testholepunch.nim index e72464a2..5a4db7ad 100644 --- a/tests/integration/nat/hole-punch/testholepunch.nim +++ b/tests/integration/nat/hole-punch/testholepunch.nim @@ -9,8 +9,6 @@ import ../../../checktest import ../../storageclient import ../composehelper -const dcutrConnectedLog = "Dcutr initiator has directly connected to the remote peer." - proc announcesCircuitAddr(info: JsonNode): bool = info{"announceAddresses"}.getElems.anyIt("p2p-circuit" in it.getStr) @@ -55,10 +53,12 @@ asyncchecksuite "NAT hole punching": let cid = (await nodeClient.upload("punch me for real")).get check (await peerClient.download(cid)).isOk - # B sees the relayed peer D join and, since D has no public address, drives - # the coordinated DCUtR simultaneous-open instead of a unilateral reversal. - check eventuallySafe( - dcutrConnectedLog in serviceLogs(composeFile, "node"), - timeout = 60_000, - pollInterval = 2_000, + # B should upgrade the relayed connection to a direct one: its connection to D + # becomes non-relayed + let peerId = (await peerClient.info()).get{"id"}.getStr + check eventuallyInfo( + nodeClient, + info{"connections"}.getElems.anyIt( + it{"peerId"}.getStr == peerId and it{"direct"}.getBool + ), ) From 488f2b57b3450c54961f7cef4be81bd4b8486532 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Wed, 17 Jun 2026 09:25:35 +0400 Subject: [PATCH 157/167] Skip test temporary --- tests/storage/testnatdetection.nim | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/storage/testnatdetection.nim b/tests/storage/testnatdetection.nim index 0673be44..c4b75f6e 100644 --- a/tests/storage/testnatdetection.nim +++ b/tests/storage/testnatdetection.nim @@ -258,6 +258,9 @@ asyncchecksuite "NAT detection - dial request candidates": await sw2.stop() test "after a port mapping, the mapped address is AutoNAT's first dial candidate": + # Temporary skipped because it might not work if PCP creates a mapping on a different port + return + let mapper = MockMappingNatPortMapper() setupMappedAddrMapper(sw, mapper) From 164f2dac3abdc1f4882cdca1084045141d4e9a04 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Wed, 17 Jun 2026 09:26:17 +0400 Subject: [PATCH 158/167] Fallback on relay when port mapping is not reachable --- storage/nat.nim | 14 +++---------- tests/storage/testnatreaction.nim | 34 +++---------------------------- 2 files changed, 6 insertions(+), 42 deletions(-) diff --git a/storage/nat.nim b/storage/nat.nim index 63ffff9a..02b54a34 100644 --- a/storage/nat.nim +++ b/storage/nat.nim @@ -217,17 +217,9 @@ method handleNatStatus*( # If the relay is running, the addresses will be updated on reservation. discovery.announceDirectAddrs(@[], udpPort = discoveryPort) - if dialBackAddr.isNone: - warn "Got empty dialback address in AutoNat when node is NotReachable" - - if m.hasMappingIds(): - m.close() - elif m.hasMappingIds(): - warn "Not Reachable with active port mapping. The port mapping will be deleted and relay will start." - - # The mapping was created the the node is still not reachable. - # In that case, we delete the mapping and relay will start. - m.close() + if m.hasMappingIds(): + # The mapping was created but the node is still not reachable. + debug "Not Reachable with active port mapping, keeping it and starting relay if not started" else: debug "Node is not reachable trying port mapping now" diff --git a/tests/storage/testnatreaction.nim b/tests/storage/testnatreaction.nim index 0a0781c1..4e229f6e 100644 --- a/tests/storage/testnatreaction.nim +++ b/tests/storage/testnatreaction.nim @@ -104,7 +104,7 @@ asyncchecksuite "NAT reaction - port mapping": check not autoRelay.isRunning check disc.protocol.clientMode - test "handleNatStatus starts autoRelay when NotReachable and no dialBackAddr but no mapped ports": + test "handleNatStatus starts autoRelay when NotReachable with no mapped ports": let mapper = MockNatPortMapper(mappedPorts: none((Port, Port, MappingProtocol))) autorelayservice.setup(autoRelay, sw) @@ -112,23 +112,11 @@ asyncchecksuite "NAT reaction - port mapping": NotReachable, Opt.none(MultiAddress), discoveryPort, disc, sw, autoRelay ) - check autoRelay.isRunning - check disc.protocol.clientMode - - test "handleNatStatus starts autoRelay when NotReachable and dialBackAddr but no mapped ports": - let dialBack = MultiAddress.init("/ip4/1.2.3.4/tcp/8080").expect("valid") - let mapper = MockNatPortMapper(mappedPorts: none((Port, Port, MappingProtocol))) - - autorelayservice.setup(autoRelay, sw) - await mapper.handleNatStatus( - NotReachable, Opt.some(dialBack), discoveryPort, disc, sw, autoRelay - ) - check autoRelay.isRunning check disc.announceAddrs == newSeq[MultiAddress]() check disc.protocol.clientMode - test "handleNatStatus tears down an active mapping and starts relay when NotReachable with dialBackAddr": + test "handleNatStatus starts relay when NotReachable with an active mapping": privateAccess(NatPortMapper) let dialBack = MultiAddress.init("/ip4/1.2.3.4/tcp/8080").expect("valid") let mapper = MockNatPortMapper() @@ -143,23 +131,7 @@ asyncchecksuite "NAT reaction - port mapping": check autoRelay.isRunning check disc.announceAddrs == newSeq[MultiAddress]() check disc.protocol.clientMode - check not mapper.hasMappingIds() # the active mapping was torn down - - test "handleNatStatus tears down an active mapping and starts relay when NotReachable without dialBackAddr": - privateAccess(NatPortMapper) - let mapper = MockNatPortMapper() - mapper.tcpMappingId = some(cint(1)) - mapper.udpMappingId = some(cint(2)) - - autorelayservice.setup(autoRelay, sw) - await mapper.handleNatStatus( - NotReachable, Opt.none(MultiAddress), discoveryPort, disc, sw, autoRelay - ) - - check autoRelay.isRunning - check disc.announceAddrs == newSeq[MultiAddress]() - check disc.protocol.clientMode - check not mapper.hasMappingIds() # the active mapping was torn down + check mapper.hasMappingIds() # the active mapping is kept test "handleNatStatus stops relay and exits client mode when mapping is created and node is Reachable": let dialBack = MultiAddress.init("/ip4/1.2.3.4/tcp/8080").expect("valid") From c3002824372365a6a3189357cc57a56c267b31b0 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Wed, 17 Jun 2026 18:11:44 +0400 Subject: [PATCH 159/167] Remove port mapper: PCP on another port is a limitation --- storage/nat.nim | 28 -------------- storage/storage.nim | 2 - tests/storage/testnatdetection.nim | 59 ------------------------------ tests/storage/testnatreaction.nim | 42 --------------------- 4 files changed, 131 deletions(-) diff --git a/storage/nat.nim b/storage/nat.nim index 02b54a34..22f99ef9 100644 --- a/storage/nat.nim +++ b/storage/nat.nim @@ -151,34 +151,6 @@ method hasMappingIds*(m: NatPortMapper): bool {.base, gcsafe.} = # (use hasMapping() for liveness check). m.tcpMappingId.isSome and m.udpMappingId.isSome -proc setupMappedAddrMapper*(switch: Switch, natMapper: NatPortMapper) = - ## We define a custom mapper that adds the externally-mapped address to - ## peerInfo.addrs when a port mapping is active, so AutoNAT tests that port. - ## PCP/NAT-PMP may grant an external port different from the listen port. - let mapper: AddressMapper = proc( - addrs: seq[MultiAddress] - ): Future[seq[MultiAddress]] {.gcsafe, async: (raises: [CancelledError]).} = - result = addrs - - if natMapper.activeTcpPort.isNone: - return result - - let mappedPort = natMapper.activeTcpPort.get - for listenAddr in switch.peerInfo.listenAddrs: - # Dialable IP (observed public, or the listen IP if already public) - # used with the mapped port. - let mappedAddr = switch.peerStore.guessDialableAddr(listenAddr).remapAddr( - port = some(mappedPort) - ) - if mappedAddr.isPublicMA(): - # Insert first so AutoNAT dials it before the listen-port candidate (the - # server tests only the first dialable address). - result.insert(mappedAddr, 0) - - return result.deduplicate() - - switch.peerInfo.addressMappers.add(mapper) - method handleNatStatus*( m: NatPortMapper, networkReachability: NetworkReachability, diff --git a/storage/storage.nim b/storage/storage.nim index 4088dcb8..e3b87704 100644 --- a/storage/storage.nim +++ b/storage/storage.nim @@ -469,8 +469,6 @@ proc new*( ) ) - setupMappedAddrMapper(switch, natMapper.get) - autonatService.get.setStatusAndConfidenceHandler( proc( networkReachability: NetworkReachability, diff --git a/tests/storage/testnatdetection.nim b/tests/storage/testnatdetection.nim index c4b75f6e..c58b10d8 100644 --- a/tests/storage/testnatdetection.nim +++ b/tests/storage/testnatdetection.nim @@ -256,62 +256,3 @@ asyncchecksuite "NAT detection - dial request candidates": await autonat.stop(sw) await sw2.stop() - - test "after a port mapping, the mapped address is AutoNAT's first dial candidate": - # Temporary skipped because it might not work if PCP creates a mapping on a different port - return - - let mapper = MockMappingNatPortMapper() - - setupMappedAddrMapper(sw, mapper) - - # Reach the observation quorum so guessDialableAddr trusts 8.8.8.8 - let observed = MultiAddress.init("/ip4/8.8.8.8/tcp/4001").expect("valid") - let quorum = 3 - for _ in 0 ..< quorum: - discard sw.peerStore.identify.observedAddrManager.addObservation(observed) - - # Setup AutoRelayService - let relay = AutoRelayService.new( - 1, relayClientModule.RelayClient.new(), nil, Rng.instance().libp2pRng - ) - autorelayservice.setup(relay, sw) - - # Define our handleNatStatus callback - let disc = Discovery.new( - PrivateKey.random(Rng.instance().libp2pRng).get(), announceAddrs = @[] - ) - let dialBack = MultiAddress.init("/ip4/8.8.8.8/tcp/8080").expect("valid") - await mapper.handleNatStatus( - NotReachable, Opt.some(dialBack), discoveryPort, disc, sw, relay - ) - - # Define our AutonatV2Service - let mockClient = MockAutonatV2Client() - let autonat = AutonatV2Service.new( - Rng.instance().libp2pRng, - mockClient, - AutonatV2ServiceConfig.new( - enableDialableCandidates = true, maxQueueSize = 1, minConfidence = 0.5 - ), - ) - service.setup(autonat, sw) - await autonat.start(sw) - - # Connect to a second switch to test NAT detection - let sw2 = newStandardSwitch() - await sw2.start() - await sw.connect(sw2.peerInfo.peerId, sw2.peerInfo.addrs) - - # The expected mapped address should be the guessDialableAddr (8.8.8.8) - # using the mapping mocked port (40000) because a mapping was created. - let mapped = - MultiAddress.init("/ip4/8.8.8.8/tcp/" & $mockMappedTcpPort).expect("valid") - check eventually(mapped in mockClient.reqAddrs) - # Ensute that it comes first (because AutonatV2 test only the first candidate) - check mockClient.reqAddrs[0] == mapped - - await autonat.stop(sw) - await sw2.stop() - if relay.isRunning: - await relay.stop(sw) diff --git a/tests/storage/testnatreaction.nim b/tests/storage/testnatreaction.nim index 4e229f6e..76887506 100644 --- a/tests/storage/testnatreaction.nim +++ b/tests/storage/testnatreaction.nim @@ -215,39 +215,6 @@ asyncchecksuite "NAT reaction - address announcing": check disc.announceAddrs == newSeq[MultiAddress]() - test "mapped-addr mapper injects the mapped port as the first candidate": - const mockMappedTcpPort = 40000 - - setupMappedAddrMapper( - sw, NatPortMapper(activeTcpPort: some(Port(mockMappedTcpPort))) - ) - - # Reach the observation quorum so guessDialableAddr trusts 8.8.8.8 - let observed = MultiAddress.init("/ip4/8.8.8.8/tcp/4001").expect("valid") - let quorum = 3 - for _ in 0 ..< quorum: - discard sw.peerStore.identify.observedAddrManager.addObservation(observed) - - await sw.peerInfo.update() - - # Ensure that the address mapper injects the mapped port as the first candidate - # after peer info update - check sw.peerInfo.addrs[0] == - MultiAddress.init("/ip4/8.8.8.8/tcp/" & $mockMappedTcpPort).expect("valid") - - test "mapped-addr mapper is a no-op without an active mapping": - setupMappedAddrMapper(sw, NatPortMapper()) - - let observed = MultiAddress.init("/ip4/8.8.8.8/tcp/4001").expect("valid") - let quorum = 3 - for _ in 0 ..< quorum: - discard sw.peerStore.identify.observedAddrManager.addObservation(observed) - - await sw.peerInfo.update() - - # Ensure that nothing is injected because there is no active mapping - check sw.peerInfo.addrs == sw.peerInfo.listenAddrs - test "handleNatStatus clears the DHT routing addresses when it becomes NotReachable": let dialBack = MultiAddress.init("/ip4/1.2.3.4/tcp/9000").expect("valid") let mapper = MockNatPortMapper(mappedPorts: none((Port, Port, MappingProtocol))) @@ -266,15 +233,6 @@ asyncchecksuite "NAT reaction - address announcing": ) check disc.dhtAddrs.len == 0 - test "mapped-addr mapper does not inject a non-public mapped address": - # Active mapping, but no public observed address: the candidate stays private - # and must not be injected. - setupMappedAddrMapper(sw, NatPortMapper(activeTcpPort: some(Port(40000)))) - - await sw.peerInfo.update() - - check sw.peerInfo.addrs == sw.peerInfo.listenAddrs - test "announceRelayReservation announces only the publicly dialable circuit address": disc.announceRelayReservation( @[circuitAddr("127.0.0.1"), circuitAddr("204.168.234.45")] From 4bcabb5ec5d927eca8f3ada966da53a8af0ee329 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Wed, 17 Jun 2026 20:27:58 +0400 Subject: [PATCH 160/167] Refactor PortMapping object --- storage/nat.nim | 100 ++++++++++++++-------------- tests/storage/natsimulation.nim | 3 +- tests/storage/testnatdetection.nim | 10 ++- tests/storage/testnatreaction.nim | 86 +++++++++++++++--------- tests/storage/testnatsimulation.nim | 6 +- 5 files changed, 116 insertions(+), 89 deletions(-) diff --git a/storage/nat.nim b/storage/nat.nim index 22f99ef9..a3bafd5b 100644 --- a/storage/nat.nim +++ b/storage/nat.nim @@ -34,6 +34,13 @@ type NatConfig* = object of true: extIp*: IpAddress of false: nat*: NatStrategy +type PortMapping* = object + tcpMappingId: cint + udpMappingId: cint + activeMappingProtocol*: MappingProtocol + activeTcpPort*: Port + activeUdpPort*: Port + type NatPortMapper* = ref object of RootObj natConfig*: NatConfig tcpPort*: Port @@ -41,11 +48,7 @@ type NatPortMapper* = ref object of RootObj discoverTimeout*: int mappingTimeout*: int recheckPeriod*: int - tcpMappingId: Option[cint] - udpMappingId: Option[cint] - activeMappingProtocol*: Option[MappingProtocol] - activeTcpPort*: Option[Port] - activeUdpPort*: Option[Port] + portMapping*: Option[PortMapping] plumInitialized: bool closed: bool @@ -71,21 +74,21 @@ method createMappingFor*( method destroyMappingFor*(m: NatPortMapper, id: cint) {.base, gcsafe.} = destroyMapping(id) -method hasLiveMapping*(m: NatPortMapper, id: cint): bool {.base, gcsafe.} = - hasMapping(id) +method hasLivePortMapping*(m: NatPortMapper): bool {.base, gcsafe.} = + ## True only when a mapping was created AND both the TCP and UDP mappings are + ## still live in the router. + if m.portMapping.isNone: + return false + + let pm = m.portMapping.get + hasMapping(pm.tcpMappingId) and hasMapping(pm.udpMappingId) proc resetMappings(m: NatPortMapper) = - if m.tcpMappingId.isSome: - m.destroyMappingFor(m.tcpMappingId.get) - m.tcpMappingId = none(cint) - - if m.udpMappingId.isSome: - m.destroyMappingFor(m.udpMappingId.get) - m.udpMappingId = none(cint) - - m.activeMappingProtocol = none(MappingProtocol) - m.activeTcpPort = none(Port) - m.activeUdpPort = none(Port) + if m.portMapping.isSome: + let pm = m.portMapping.get + m.destroyMappingFor(pm.tcpMappingId) + m.destroyMappingFor(pm.udpMappingId) + m.portMapping = none(PortMapping) method mapNatPorts*( m: NatPortMapper @@ -95,11 +98,10 @@ method mapNatPorts*( if m.closed or m.natConfig.hasExtIp: return none((Port, Port, MappingProtocol)) - # If both mappings are still active, return the stored ports without recreating. - if m.activeTcpPort.isSome and m.activeUdpPort.isSome and m.activeMappingProtocol.isSome and - m.tcpMappingId.isSome and m.hasLiveMapping(m.tcpMappingId.get) and - m.udpMappingId.isSome and m.hasLiveMapping(m.udpMappingId.get): - return some((m.activeTcpPort.get, m.activeUdpPort.get, m.activeMappingProtocol.get)) + # If both mappings are still live, return the stored ports without recreating. + if m.hasLivePortMapping(): + let pm = m.portMapping.get + return some((pm.activeTcpPort, pm.activeUdpPort, pm.activeMappingProtocol)) if not m.plumInitialized: let res = m.initPlum() @@ -123,13 +125,18 @@ method mapNatPorts*( m.destroyMappingFor(tcpRes.value.id) return none((Port, Port, MappingProtocol)) - m.tcpMappingId = some(tcpRes.value.id) - m.udpMappingId = some(udpRes.value.id) - m.activeMappingProtocol = some(tcpRes.value.mapping.mappingProtocol) - m.activeTcpPort = some(Port(tcpRes.value.mapping.externalPort)) - m.activeUdpPort = some(Port(udpRes.value.mapping.externalPort)) + m.portMapping = some( + PortMapping( + tcpMappingId: tcpRes.value.id, + udpMappingId: udpRes.value.id, + activeMappingProtocol: tcpRes.value.mapping.mappingProtocol, + activeTcpPort: Port(tcpRes.value.mapping.externalPort), + activeUdpPort: Port(udpRes.value.mapping.externalPort), + ) + ) - some((m.activeTcpPort.get, m.activeUdpPort.get, m.activeMappingProtocol.get)) + let pm = m.portMapping.get + some((pm.activeTcpPort, pm.activeUdpPort, pm.activeMappingProtocol)) proc close*(m: NatPortMapper) = m.resetMappings() @@ -143,14 +150,6 @@ proc stop*(m: NatPortMapper) = m.closed = true m.close() -proc isPortMapped*(m: NatPortMapper, port: Port): bool = - m.activeTcpPort.isSome and m.activeTcpPort.get == port - -method hasMappingIds*(m: NatPortMapper): bool {.base, gcsafe.} = - # Only checks that mappings were created, not that they are still live - # (use hasMapping() for liveness check). - m.tcpMappingId.isSome and m.udpMappingId.isSome - method handleNatStatus*( m: NatPortMapper, networkReachability: NetworkReachability, @@ -174,13 +173,14 @@ method handleNatStatus*( discovery.protocol.clientMode = false - discovery.announceDirectAddrs( - @[dialBackAddr.get], udpPort = m.activeUdpPort.get(discoveryPort) - ) + # Here we don't rely on the port mapping because we consider + # that port mapped is the same as the discovery port. + # This can be wrong for PCP but it is an accepted limitation + discovery.announceDirectAddrs(@[dialBackAddr.get], udpPort = discoveryPort) else: warn "Empty dialback address in AutoNat when node is Reachable" of NotReachable: - var hasPortMapping = false + var mappingCreated = false discovery.protocol.clientMode = true @@ -189,9 +189,10 @@ method handleNatStatus*( # If the relay is running, the addresses will be updated on reservation. discovery.announceDirectAddrs(@[], udpPort = discoveryPort) - if m.hasMappingIds(): - # The mapping was created but the node is still not reachable. - debug "Not Reachable with active port mapping, keeping it and starting relay if not started" + if m.hasLivePortMapping(): + # The mapping is still live but the node is not reachable: keep it and let + # the relay take over. A dead mapping falls through to be recreated. + debug "Not Reachable with live port mapping, keeping it and starting relay if not started" else: debug "Node is not reachable trying port mapping now" @@ -202,16 +203,15 @@ method handleNatStatus*( info "Port mapping created successfully", tcpPort, udpPort, protocol - # The address mapper uses the mapped port to build the candidate address - # for AutoNAT; the announce happens once AutoNAT confirms Reachable. + # The announce happens once AutoNAT confirms Reachable. - hasPortMapping = true + mappingCreated = true else: # In case of failure, close the port mapping in order to rerun discover # on the next iteration m.close() - if not hasPortMapping and not autoRelayService.isRunning: + if not mappingCreated and not autoRelayService.isRunning: debug "No port mapping found let's start autorelay" await autoRelayService.start(switch) @@ -224,9 +224,9 @@ proc reachabilityStr*(autonat: Option[AutonatV2Service]): string = "Unknown" proc portMappingStr*(natMapper: Option[NatPortMapper]): string = - if natMapper.isNone or natMapper.get.activeMappingProtocol.isNone: + if natMapper.isNone or natMapper.get.portMapping.isNone: return "none" - case natMapper.get.activeMappingProtocol.get + case natMapper.get.portMapping.get.activeMappingProtocol of MappingProtocol.UPnP: "upnp" of MappingProtocol.NatPmp: "pmp" of MappingProtocol.PCP: "pcp" diff --git a/tests/storage/natsimulation.nim b/tests/storage/natsimulation.nim index 2ae0b77d..19ec0a33 100644 --- a/tests/storage/natsimulation.nim +++ b/tests/storage/natsimulation.nim @@ -70,7 +70,8 @@ proc allowInbound(r: NatRouter, remote: TransportAddress, localPort: Port): bool else: discard - if r.natMapper.isSome and r.natMapper.get.isPortMapped(localPort): + if r.natMapper.isSome and r.natMapper.get.portMapping.isSome and + r.natMapper.get.portMapping.get.activeTcpPort == localPort: return true case r.filtering diff --git a/tests/storage/testnatdetection.nim b/tests/storage/testnatdetection.nim index c58b10d8..ddd19fdf 100644 --- a/tests/storage/testnatdetection.nim +++ b/tests/storage/testnatdetection.nim @@ -50,9 +50,13 @@ method mapNatPorts*( ): Future[Option[(Port, Port, MappingProtocol)]] {. async: (raises: [CancelledError]), gcsafe .} = - m.activeTcpPort = some(mockMappedTcpPort) - m.activeUdpPort = some(mockMappedUdpPort) - m.activeMappingProtocol = some(MappingProtocol.PCP) + m.portMapping = some( + PortMapping( + activeMappingProtocol: MappingProtocol.PCP, + activeTcpPort: mockMappedTcpPort, + activeUdpPort: mockMappedUdpPort, + ) + ) some((mockMappedTcpPort, mockMappedUdpPort, MappingProtocol.PCP)) # Captures the candidate addresses the service sends and answers Reachable, so diff --git a/tests/storage/testnatreaction.nim b/tests/storage/testnatreaction.nim index 76887506..5e5de7bc 100644 --- a/tests/storage/testnatreaction.nim +++ b/tests/storage/testnatreaction.nim @@ -4,7 +4,6 @@ import pkg/libp2p/[multiaddress, multihash, multicodec] import pkg/libp2p/protocols/connectivity/autonat/types import pkg/libp2p/protocols/connectivity/relay/client as relayClientModule import pkg/libp2p/services/autorelayservice except setup -import pkg/libp2p/observedaddrmanager import pkg/results import ./helpers @@ -38,8 +37,8 @@ type MockMapNatPortMapper = ref object of NatPortMapper method initPlum(m: MockMapNatPortMapper): Result[void, string] {.gcsafe.} = ok() -method hasLiveMapping(m: MockMapNatPortMapper, id: cint): bool {.gcsafe.} = - m.live +method hasLivePortMapping(m: MockMapNatPortMapper): bool {.gcsafe.} = + m.portMapping.isSome and m.live method createMappingFor( m: MockMapNatPortMapper, protocol: PlumProtocol, port: uint16 @@ -116,12 +115,19 @@ asyncchecksuite "NAT reaction - port mapping": check disc.announceAddrs == newSeq[MultiAddress]() check disc.protocol.clientMode - test "handleNatStatus starts relay when NotReachable with an active mapping": - privateAccess(NatPortMapper) + test "handleNatStatus keeps a live mapping and starts relay when NotReachable": + privateAccess(PortMapping) let dialBack = MultiAddress.init("/ip4/1.2.3.4/tcp/8080").expect("valid") - let mapper = MockNatPortMapper() - mapper.tcpMappingId = some(cint(1)) - mapper.udpMappingId = some(cint(2)) + let mapper = MockMapNatPortMapper(live: true) + mapper.portMapping = some( + PortMapping( + tcpMappingId: cint(1), + udpMappingId: cint(2), + activeMappingProtocol: MappingProtocol.UPnP, + activeTcpPort: Port(9000), + activeUdpPort: Port(9001), + ) + ) autorelayservice.setup(autoRelay, sw) await mapper.handleNatStatus( @@ -131,7 +137,35 @@ asyncchecksuite "NAT reaction - port mapping": check autoRelay.isRunning check disc.announceAddrs == newSeq[MultiAddress]() check disc.protocol.clientMode - check mapper.hasMappingIds() # the active mapping is kept + check mapper.portMapping.isSome # the live mapping is kept + check mapper.destroyed.len == 0 # never torn down + + test "handleNatStatus recreates a dead mapping instead of pinning it": + privateAccess(PortMapping) + let dialBack = MultiAddress.init("/ip4/1.2.3.4/tcp/8080").expect("valid") + let mapper = MockMapNatPortMapper( + live: false, + tcpResult: mappingOk(cint(10), 9000), + udpResult: mappingOk(cint(20), 9001), + ) + mapper.portMapping = some( + PortMapping( + tcpMappingId: cint(1), + udpMappingId: cint(2), + activeMappingProtocol: MappingProtocol.UPnP, + activeTcpPort: Port(9000), + activeUdpPort: Port(9001), + ) + ) + + autorelayservice.setup(autoRelay, sw) + await mapper.handleNatStatus( + NotReachable, Opt.some(dialBack), discoveryPort, disc, sw, autoRelay + ) + + check mapper.destroyed == @[cint(1), cint(2)] # the dead mapping is torn down + check mapper.portMapping.isSome # replaced by a fresh one + check not autoRelay.isRunning # direct path kept, no relay test "handleNatStatus stops relay and exits client mode when mapping is created and node is Reachable": let dialBack = MultiAddress.init("/ip4/1.2.3.4/tcp/8080").expect("valid") @@ -193,20 +227,6 @@ asyncchecksuite "NAT reaction - address announcing": check disc.announceAddrs == @[dialBack] - test "handleNatStatus announces the mapped external UDP port when a mapping is active": - let dialBack = MultiAddress.init("/ip4/1.2.3.4/tcp/9000").expect("valid") - - let mapper = - NatPortMapper(discoveryPort: discoveryPort, activeUdpPort: some(Port(40001))) - await mapper.handleNatStatus( - Reachable, Opt.some(dialBack), discoveryPort, disc, sw, autoRelay - ) - - let sprAddrs = disc.getSpr().data.addresses.mapIt(it.address) - check MultiAddress.init("/ip4/1.2.3.4/udp/40001").expect("valid") in sprAddrs - check MultiAddress.init("/ip4/1.2.3.4/udp/" & $discoveryPort).expect("valid") notin - sprAddrs - test "handleNatStatus does not announce when Reachable without a dial-back address": let mapper = NatPortMapper(discoveryPort: discoveryPort) await mapper.handleNatStatus( @@ -246,7 +266,7 @@ asyncchecksuite "NAT reaction - address announcing": check disc.announceAddrs.len == 0 proc mapperWith(protocol: MappingProtocol): Option[NatPortMapper] = - some(NatPortMapper(activeMappingProtocol: some(protocol))) + some(NatPortMapper(portMapping: some(PortMapping(activeMappingProtocol: protocol)))) asyncchecksuite "NAT - portMappingStr": test "no mapper is none": @@ -308,15 +328,17 @@ asyncchecksuite "NAT - mapNatPorts": check mapper.createAttempts.len == 0 # short-circuits before any mapping test "reuses the existing mapping when both are still live": - privateAccess(NatPortMapper) - let mapper = MockMapNatPortMapper( - live: true, - activeTcpPort: some(Port(9000)), - activeUdpPort: some(Port(9001)), - activeMappingProtocol: some(MappingProtocol.UPnP), + privateAccess(PortMapping) + let mapper = MockMapNatPortMapper(live: true) + mapper.portMapping = some( + PortMapping( + tcpMappingId: cint(1), + udpMappingId: cint(2), + activeMappingProtocol: MappingProtocol.UPnP, + activeTcpPort: Port(9000), + activeUdpPort: Port(9001), + ) ) - mapper.tcpMappingId = some(cint(1)) - mapper.udpMappingId = some(cint(2)) check (await mapper.mapNatPorts()) == some((Port(9000), Port(9001), MappingProtocol.UPnP)) diff --git a/tests/storage/testnatsimulation.nim b/tests/storage/testnatsimulation.nim index 6e046526..077f8d62 100644 --- a/tests/storage/testnatsimulation.nim +++ b/tests/storage/testnatsimulation.nim @@ -143,7 +143,7 @@ asyncchecksuite "Nat transport - Double NAT": test "bootstrap cannot connect to nat node regardless of port mapping": let actualPort = initTAddress(natNode.peerInfo.addrs[0]).get().port let natMapper = NatPortMapper() - natMapper.activeTcpPort = some(actualPort) + natMapper.portMapping = some(PortMapping(activeTcpPort: actualPort)) router.natMapper = some(natMapper) check await cannotConnect(bootstrap, natNode) @@ -166,7 +166,7 @@ asyncchecksuite "Nat transport - Port Mapping": test "bootstrap can connect to nat node when port mapping matches listen port": let actualPort = initTAddress(natNode.peerInfo.addrs[0]).get().port let natMapper = NatPortMapper() - natMapper.activeTcpPort = some(actualPort) + natMapper.portMapping = some(PortMapping(activeTcpPort: actualPort)) router.natMapper = some(natMapper) await bootstrap.connect(natNode.peerInfo.peerId, natNode.peerInfo.addrs) @@ -174,7 +174,7 @@ asyncchecksuite "Nat transport - Port Mapping": test "bootstrap cannot connect to nat node when port mapping does not match": let natMapper = NatPortMapper() - natMapper.activeTcpPort = some(Port(1)) + natMapper.portMapping = some(PortMapping(activeTcpPort: Port(1))) router.natMapper = some(natMapper) check await cannotConnect(bootstrap, natNode) From 3d4400ce012a381bbe00b356cb483db9fc7e3063 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Wed, 17 Jun 2026 20:34:36 +0400 Subject: [PATCH 161/167] Add more guard --- storage/nat.nim | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/storage/nat.nim b/storage/nat.nim index a3bafd5b..53735cab 100644 --- a/storage/nat.nim +++ b/storage/nat.nim @@ -115,11 +115,21 @@ method mapNatPorts*( m.resetMappings() let tcpRes = await m.createMappingFor(TCP, m.tcpPort.uint16) + + if m.closed: + # Double check in case the node is stopping + return none((Port, Port, MappingProtocol)) + if tcpRes.isErr: warn "TCP port mapping failed", msg = tcpRes.error return none((Port, Port, MappingProtocol)) let udpRes = await m.createMappingFor(UDP, m.discoveryPort.uint16) + + if m.closed: + # Double check in case the node is stopping + return none((Port, Port, MappingProtocol)) + if udpRes.isErr: warn "UDP port mapping failed", msg = udpRes.error m.destroyMappingFor(tcpRes.value.id) @@ -180,8 +190,6 @@ method handleNatStatus*( else: warn "Empty dialback address in AutoNat when node is Reachable" of NotReachable: - var mappingCreated = false - discovery.protocol.clientMode = true if not autoRelayService.isRunning and discovery.announceAddrs.len > 0: @@ -198,6 +206,10 @@ method handleNatStatus*( let maybePorts = await m.mapNatPorts() + if m.closed: + # Double check in case the node is stopping + return + if maybePorts.isSome: let (tcpPort, udpPort, protocol) = maybePorts.get() @@ -205,13 +217,13 @@ method handleNatStatus*( # The announce happens once AutoNAT confirms Reachable. - mappingCreated = true + return else: # In case of failure, close the port mapping in order to rerun discover # on the next iteration m.close() - if not mappingCreated and not autoRelayService.isRunning: + if not autoRelayService.isRunning: debug "No port mapping found let's start autorelay" await autoRelayService.start(switch) From d9c2b3a5d6091d1cf427db10c13cd5f4b9b6b770 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Wed, 17 Jun 2026 21:03:21 +0400 Subject: [PATCH 162/167] Update default config --- storage/conf.nim | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/storage/conf.nim b/storage/conf.nim index d3c14bce..850f0f62 100644 --- a/storage/conf.nim +++ b/storage/conf.nim @@ -330,8 +330,11 @@ type .}: int natMinConfidence* {. + # With maxQueueSize=3, 0.6 confirms reachability on a 2/3 majority + # (2/3=0.667) instead of a 3/3 unanimous round, tolerating one inconsistent + # peer. desc: "Minimum confidence threshold to confirm reachability", - defaultValue: 0.7, + defaultValue: 0.6, name: "nat-min-confidence" .}: float From d71b221a4c3c0176b7c829d0f7ffaf7fd576cd1f Mon Sep 17 00:00:00 2001 From: Arnaud Date: Wed, 17 Jun 2026 21:13:24 +0400 Subject: [PATCH 163/167] Align libstorage debug with api --- .../storage_thread_requests/requests/node_debug_request.nim | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/library/storage_thread_requests/requests/node_debug_request.nim b/library/storage_thread_requests/requests/node_debug_request.nim index 8d5fe2f4..7ccb092f 100644 --- a/library/storage_thread_requests/requests/node_debug_request.nim +++ b/library/storage_thread_requests/requests/node_debug_request.nim @@ -15,7 +15,7 @@ import ../../../storage/conf import ../../../storage/rest/json import ../../../storage/node -from ../../../storage/storage import StorageServer, node +from ../../../storage/storage import StorageServer, node, config import ../../../storage/nat import ../../../storage/discovery @@ -59,10 +59,12 @@ proc getDebug( let json = %*{ "id": $node.switch.peerInfo.peerId, "addrs": node.switch.peerInfo.addrs.mapIt($it), + "repo": $storage[].config.dataDir, "spr": nodeSpr.toURI, "announceAddresses": node.discovery.announceAddrs, "dhtAddresses": node.discovery.dhtAddrs, "table": table, + "storage": {"version": $storageVersion, "revision": $storageRevision}, "nat": { "reachability": reachabilityStr(storage[].autonatService), "clientMode": node.discovery.protocol.clientMode, @@ -70,6 +72,7 @@ proc getDebug( storage[].autoRelayService.isSome and storage[].autoRelayService.get.isRunning, "portMapping": portMappingStr(storage[].natMapper), }, + "connections": peerConnections(node.switch), } return ok($json) From 402be0370d84dd47c8c4a1e3a107c5ee596a2fad Mon Sep 17 00:00:00 2001 From: Arnaud Date: Wed, 17 Jun 2026 21:13:41 +0400 Subject: [PATCH 164/167] Add nocancel for boostrap nodes connect --- storage/storage.nim | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/storage/storage.nim b/storage/storage.nim index e3b87704..2a96d28a 100644 --- a/storage/storage.nim +++ b/storage/storage.nim @@ -144,7 +144,11 @@ proc start*(s: StorageServer) {.async.} = except CatchableError as e: warn "Cannot connect to bootstrap node", error = e.msg - await allFutures(findReachableNodes(s.bootstrapNodes).mapIt(connectBootstrapNode(it))) + # noCancel: cancelling allFutures does not cancel the + # connectBootstrapNode futures. + await noCancel allFutures( + findReachableNodes(s.bootstrapNodes).mapIt(connectBootstrapNode(it)) + ) # AutoNAT is not in switch.services because we want to start it # after the bootstrap connections to have connected peers for the first probe. From ed3da20a95208bbd673c992c85068a61b42f8fc7 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Wed, 17 Jun 2026 21:13:49 +0400 Subject: [PATCH 165/167] Update libp2p --- vendor/nim-libp2p | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/nim-libp2p b/vendor/nim-libp2p index 1bd3b986..c470b114 160000 --- a/vendor/nim-libp2p +++ b/vendor/nim-libp2p @@ -1 +1 @@ -Subproject commit 1bd3b986c82ab37a509fc84ca0ad7ea67a705a1b +Subproject commit c470b1146fa2ef23ab88c5a0940923cf7645e9c5 From fc6469172036353efa3d6d491e3aae2de5fb4ddc Mon Sep 17 00:00:00 2001 From: Arnaud Date: Wed, 17 Jun 2026 22:22:11 +0400 Subject: [PATCH 166/167] Update conf tests --- storage/conf.nim | 15 +++++++++++++++ tests/storage/testconf.nim | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/storage/conf.nim b/storage/conf.nim index 850f0f62..b965ea39 100644 --- a/storage/conf.nim +++ b/storage/conf.nim @@ -417,6 +417,21 @@ func validateAutonatConfig*(config: StorageConf): ?!void = if config.natMinConfidence < 0.0 or config.natMinConfidence > 1.0: return failure "--nat-min-confidence must be between 0 and 1" + if config.natScheduleInterval <= 0.seconds: + return failure "--nat-schedule-interval must be greater than 0" + + if config.natMaxRelays < 1: + return failure "--nat-max-relays must be at least 1" + + if config.natPortMappingDiscoverTimeout < 1: + return failure "--nat-port-mapping-discover-timeout must be greater than 0" + + if config.natPortMappingTimeout < 1: + return failure "--nat-port-mapping-timeout must be greater than 0" + + if config.natPortMappingRecheckPeriod < 1: + return failure "--nat-port-mapping-recheck-period must be greater than 0" + success() proc getStorageVersion(): string = diff --git a/tests/storage/testconf.nim b/tests/storage/testconf.nim index e82f29f8..836135d5 100644 --- a/tests/storage/testconf.nim +++ b/tests/storage/testconf.nim @@ -11,6 +11,11 @@ proc validConfig(): StorageConf = natNumPeersToAsk: 5, natMinConfidence: 0.7, natObservedAddrMinCount: 1, + natScheduleInterval: DefaultNatScheduleInterval, + natMaxRelays: 2, + natPortMappingDiscoverTimeout: 500, + natPortMappingTimeout: 500, + natPortMappingRecheckPeriod: 300000, ) suite "Conf - validateAutonatConfig": @@ -112,3 +117,33 @@ suite "Conf - validateAutonatConfig": config.natMinConfidence = 1.0 check config.validateAutonatConfig().isOk + + test "rejects nat-schedule-interval of zero": + var config = validConfig() + config.natScheduleInterval = 0.seconds + + check config.validateAutonatConfig().isErr + + test "rejects nat-max-relays below 1": + var config = validConfig() + config.natMaxRelays = 0 + + check config.validateAutonatConfig().isErr + + test "rejects nat-port-mapping-discover-timeout of zero": + var config = validConfig() + config.natPortMappingDiscoverTimeout = 0 + + check config.validateAutonatConfig().isErr + + test "rejects nat-port-mapping-timeout of zero": + var config = validConfig() + config.natPortMappingTimeout = 0 + + check config.validateAutonatConfig().isErr + + test "rejects nat-port-mapping-recheck-period of zero": + var config = validConfig() + config.natPortMappingRecheckPeriod = 0 + + check config.validateAutonatConfig().isErr From f1bb83c0365e1ecb3dbbf9391a49f9a36eff1b63 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Wed, 17 Jun 2026 22:53:56 +0400 Subject: [PATCH 167/167] Fix compilation --- library/storage_thread_requests/requests/node_debug_request.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library/storage_thread_requests/requests/node_debug_request.nim b/library/storage_thread_requests/requests/node_debug_request.nim index 7ccb092f..2c2808a2 100644 --- a/library/storage_thread_requests/requests/node_debug_request.nim +++ b/library/storage_thread_requests/requests/node_debug_request.nim @@ -59,7 +59,7 @@ proc getDebug( let json = %*{ "id": $node.switch.peerInfo.peerId, "addrs": node.switch.peerInfo.addrs.mapIt($it), - "repo": $storage[].config.dataDir, + "repo": storage[].config.dataDir.string, "spr": nodeSpr.toURI, "announceAddresses": node.discovery.announceAddrs, "dhtAddresses": node.discovery.dhtAddrs,