From 56e8ebe36b74db149bad70c0e90c3c89650df0e1 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Fri, 12 Jun 2026 19:27:16 +0400 Subject: [PATCH] Add NAT integration tests --- .github/workflows/ci-reusable.yml | 17 ++-- Makefile | 18 ++-- build.nims | 9 +- tests/imports.nim | 24 +++-- tests/integration/nat/Dockerfile | 36 ++++---- tests/integration/nat/composehelper.nim | 28 ++++++ tests/integration/nat/docker-entrypoint.sh | 70 -------------- tests/integration/nat/miniupnpd_stub_rdr.c | 2 +- tests/integration/nat/node-entrypoint.sh | 26 ++++++ tests/integration/nat/not-reachable/README.md | 55 +++++++++++ .../integration/nat/not-reachable/compose.yml | 91 ++++++++++++++++++ .../nat/not-reachable/router-entrypoint.sh | 7 ++ .../nat/not-reachable/testnotreachable.nim | 68 ++++++++++++++ tests/integration/nat/reachable/README.md | 54 +++++++++++ tests/integration/nat/reachable/compose.yml | 92 +++++++++++++++++++ .../nat/reachable/router-entrypoint.sh | 12 +++ .../nat/reachable/testreachable.nim | 77 ++++++++++++++++ tests/integration/nat/router-common.sh | 25 +++++ tests/testIntegration.nim | 5 +- tests/testNatIntegration.nim | 16 ++++ tests/testStorage.nim | 2 +- tools/scripts/ci-job-matrix.sh | 14 ++- 22 files changed, 617 insertions(+), 131 deletions(-) create mode 100644 tests/integration/nat/composehelper.nim delete mode 100644 tests/integration/nat/docker-entrypoint.sh create mode 100644 tests/integration/nat/node-entrypoint.sh create mode 100644 tests/integration/nat/not-reachable/README.md create mode 100644 tests/integration/nat/not-reachable/compose.yml create mode 100755 tests/integration/nat/not-reachable/router-entrypoint.sh create mode 100644 tests/integration/nat/not-reachable/testnotreachable.nim create mode 100644 tests/integration/nat/reachable/README.md create mode 100644 tests/integration/nat/reachable/compose.yml create mode 100755 tests/integration/nat/reachable/router-entrypoint.sh create mode 100644 tests/integration/nat/reachable/testreachable.nim create mode 100644 tests/integration/nat/router-common.sh create mode 100644 tests/testNatIntegration.nim diff --git a/.github/workflows/ci-reusable.yml b/.github/workflows/ci-reusable.yml index 89d42b00..dd4227ca 100644 --- a/.github/workflows/ci-reusable.yml +++ b/.github/workflows/ci-reusable.yml @@ -57,7 +57,7 @@ jobs: - name: Upload integration tests log files uses: actions/upload-artifact@v7 - if: (matrix.tests == 'integration' || matrix.tests == 'all') && always() + if: (matrix.tests == 'integration' || matrix.tests == 'nat-integration' || matrix.tests == 'all') && always() with: name: ${{ matrix.os }}-${{ matrix.cpu }}-${{ matrix.nim_version }}-${{ matrix.job_number }}-integration-tests-logs path: tests/integration/logs/ @@ -69,13 +69,14 @@ jobs: run: make -j${ncpu} testLibstorage ## Part 4 Tests ## - - name: NAT UPnP integration tests - if: matrix.tests == 'nat-upnp-integration' - run: make testNatUpnpIntegration - - - name: NAT PCP integration tests - if: matrix.tests == 'nat-pcp-integration' - run: make testNatPcpIntegration + - name: NAT integration tests + if: matrix.tests == 'nat-integration' + env: + STORAGE_INTEGRATION_TEST_INCLUDES: ${{ matrix.includes }} + run: | + sudo modprobe iptable_nat nf_conntrack + pipx install podman-compose + make testNatIntegration status: if: always() diff --git a/Makefile b/Makefile index 6828081d..10333663 100644 --- a/Makefile +++ b/Makefile @@ -87,8 +87,8 @@ endif testAll \ testIntegration \ testLibstorage \ - testNatUpnpIntegration \ - testNatPcpIntegration \ + buildNatImage \ + testNatIntegration \ update ifeq ($(NIM_PARAMS),) @@ -152,13 +152,15 @@ testIntegration: | build deps DOCKER := $(or $(shell which podman 2>/dev/null), $(shell which docker 2>/dev/null)) -testNatUpnpIntegration: - $(DOCKER) build -t miniupnpd-test -f tests/integration/nat/Dockerfile . - $(DOCKER) run --rm --cap-add NET_ADMIN -e DEBUG=$(DEBUG) miniupnpd-test +# NAT real-topology scenarios (podman-compose), all sharing one image built +# here. Runs every scenario; run one with +# `make testNatIntegration STORAGE_INTEGRATION_TEST_INCLUDES=` (the +# scenario's folder name, e.g. reachable). +buildNatImage: + $(DOCKER) build -t localhost/storage-nat -f tests/integration/nat/Dockerfile . -testNatPcpIntegration: - $(DOCKER) build -t miniupnpd-test -f tests/integration/nat/Dockerfile . - $(DOCKER) run --rm --cap-add NET_ADMIN -e DEBUG=$(DEBUG) -e TEST_PCP=1 miniupnpd-test +testNatIntegration: | deps buildNatImage + $(ENV_SCRIPT) nim testNatIntegration $(NIM_PARAMS) build.nims # Builds a C example that uses the libstorage C library and runs it testLibstorage: | build deps diff --git a/build.nims b/build.nims index f1831020..25a31538 100644 --- a/build.nims +++ b/build.nims @@ -78,12 +78,9 @@ task testIntegration, "Run integration tests": # test "testIntegration", params = "-d:chronicles_sinks=textlines[notimestamps,stdout],textlines[dynamic] " & # "-d:chronicles_enabled_topics:integration:TRACE" -task testNatNotReachable, - "Run NAT not-reachable scenario (needs the image + podman-compose)": - test "integration/nat/not-reachable/testnotreachable", outName = "testNatNotReachable" - -task testNatReachable, "Run NAT reachable scenario (needs the image + podman-compose)": - test "integration/nat/reachable/testreachable", outName = "testNatReachable" +task testNatIntegration, + "Run NAT real-topology scenarios (needs the storage-nat image + podman-compose)": + test "testNatIntegration" task build, "build Logos Storage binary": storageTask() diff --git a/tests/imports.nim b/tests/imports.nim index fbe642fc..da22f3c1 100644 --- a/tests/imports.nim +++ b/tests/imports.nim @@ -2,17 +2,25 @@ import std/macros import std/os import std/strutils -macro importTests*(dir: static string): untyped = - ## imports all files in the specified directory whose filename - ## starts with "test" and ends in ".nim" +macro importTests*( + dir: static string, exclude: static string, only: static string +): untyped = + ## imports every test*.nim file under `dir` (recursively). + ## `exclude` (when non-empty) skips files whose path contains it. + ## `only` (when non-empty) keeps only files whose path contains it. let imports = newStmtList() for file in walkDirRec(dir): let (_, name, ext) = splitFile(file) - if name.startsWith("test") and ext == ".nim": - imports.add( - quote do: - import `file` - ) + if not (name.startsWith("test") and ext == ".nim"): + continue + if exclude.len > 0 and exclude in file: + continue + if only.len > 0 and only notin file: + continue + imports.add( + quote do: + import `file` + ) imports macro importAll*(paths: static seq[string]): untyped = diff --git a/tests/integration/nat/Dockerfile b/tests/integration/nat/Dockerfile index 1795e1e6..82c35d3c 100644 --- a/tests/integration/nat/Dockerfile +++ b/tests/integration/nat/Dockerfile @@ -1,3 +1,7 @@ +# One image for every podman NAT scenario, built as localhost/storage-nat. +# Carries the storage binary + miniupnpd (for the upnp/pmp routers); scenarios +# differ only in their entrypoint scripts, which compose mounts. +# Build context = project root. FROM ubuntu:24.04 ARG NIM_VERSION=2.2.10 @@ -5,15 +9,12 @@ ARG NIM_VERSION=2.2.10 RUN apt-get update && apt-get install -y --no-install-recommends \ gcc g++ make cmake git curl ca-certificates xz-utils \ libc-dev ccache \ - iproute2 \ + iproute2 iptables jq \ && rm -rf /var/lib/apt/lists/* -# Build miniupnpd with a stub redirector. miniupnpd normally calls iptables/nftables -# to install the actual port forwarding rules when it receives a mapping request. -# In Docker, those calls fail because the container lacks the required kernel -# capabilities, causing every mapping request to return an error to the client. -# The stub replaces the firewall backend with no-ops that always return success, -# so mapping requests complete normally without touching the kernel. +# miniupnpd with a stub firewall backend: the real backend needs kernel caps a +# container lacks, so the stub makes mapping requests succeed without touching +# the kernel. Only the upnp/pmp routers use it. COPY tests/integration/nat/miniupnpd_stub_rdr.c /tmp/stub_rdr.c RUN git clone --depth=1 --branch miniupnpd_2_3_9 \ https://github.com/miniupnp/miniupnp.git /tmp/miniupnp \ @@ -24,32 +25,29 @@ RUN git clone --depth=1 --branch miniupnpd_2_3_9 \ && install -m 755 miniupnpd /usr/local/sbin/miniupnpd \ && rm -rf /tmp/miniupnp /tmp/stub_rdr.c -# Install Nim RUN curl -fsSL "https://nim-lang.org/download/nim-${NIM_VERSION}-linux_x64.tar.xz" \ - | tar -xJ -C /opt && \ - ln -s "/opt/nim-${NIM_VERSION}/bin/nim" /usr/local/bin/nim + | tar -xJ -C /opt +RUN ln -s "/opt/nim-${NIM_VERSION}/bin/nim" /usr/local/bin/nim WORKDIR /app -# Copy project source (build context must be the project root) +# vendor/ already has the checked-out submodules, so no `make update` here. COPY vendor/ vendor/ COPY storage/ storage/ -COPY library/ library/ -COPY tests/ tests/ COPY build.nims config.nims storage.nim ./ -# Build libplum C library. Nim binaries are compiled at test runtime. -# ccache caches C compilation across builds. +# libplum static lib, linked by nim-libplum. RUN --mount=type=cache,target=/root/.ccache \ export PATH="/usr/lib/ccache:$PATH" && \ rm -rf vendor/nim-libplum/vendor/libplum/build && \ cmake -B vendor/nim-libplum/vendor/libplum/build \ -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF \ vendor/nim-libplum/vendor/libplum && \ - make -j$(nproc) -C vendor/nim-libplum/vendor/libplum/build && \ + make -j"$(nproc)" -C vendor/nim-libplum/vendor/libplum/build && \ cp vendor/nim-libplum/vendor/libplum/build/libplum.a \ vendor/nim-libplum/vendor/libplum/libplum.a -COPY tests/integration/nat/docker-entrypoint.sh /entrypoint.sh -RUN chmod +x /entrypoint.sh -ENTRYPOINT ["/entrypoint.sh"] +RUN --mount=type=cache,target=/root/.ccache \ + export PATH="/usr/lib/ccache:$PATH" && \ + USE_SYSTEM_NIM=1 vendor/nimbus-build-system/scripts/env.sh \ + nim storage -d:disable_libbacktrace build.nims diff --git a/tests/integration/nat/composehelper.nim b/tests/integration/nat/composehelper.nim new file mode 100644 index 00000000..bf8e69a0 --- /dev/null +++ b/tests/integration/nat/composehelper.nim @@ -0,0 +1,28 @@ +## Helpers shared by the compose-driven NAT scenario tests (real topology, not +## the in-process simulation). Each scenario provides its own compose.yml and +## the list of services whose logs should be collected. + +import std/[os, osproc] +import ../utils + +proc compose*(composeFile, action: string) = + let cmd = "podman-compose -f \"" & composeFile & "\" " & action + doAssert execShellCmd(cmd) == 0, "command failed: " & cmd + +proc saveContainerLogs*( + composeFile, suiteName, testName, startTime: string, services: openArray[string] +) = + ## Writes each container's log via getLogFile, the same helper and layout as + ## the multinodes suite: tests/integration/logs/__/ + ## /.log. Must run before `down` destroys the containers. + for service in services: + try: + let + logFile = getLogFile("", startTime, suiteName, testName, service) + cmd = "podman-compose -f \"" & composeFile & "\" logs " & service + (output, code) = execCmdEx(cmd) + if code != 0: + echo "warning: '", cmd, "' exited ", code + writeFile(logFile, output) + except CatchableError as e: + echo "could not save logs for ", service, ": ", e.msg diff --git a/tests/integration/nat/docker-entrypoint.sh b/tests/integration/nat/docker-entrypoint.sh deleted file mode 100644 index 921a740e..00000000 --- a/tests/integration/nat/docker-entrypoint.sh +++ /dev/null @@ -1,70 +0,0 @@ -#!/bin/bash -set -euo pipefail - -RUNDIR=/tmp/miniupnpd -mkdir -p "$RUNDIR" - -# miniupnpd must listen on the same interface as the test node. -# We get the default route interface (e.g. eth0) and its IP. -LAN_IF=$(ip route show default | awk '/default/{print $5; exit}') -LAN_IP=$(ip -4 addr show "$LAN_IF" | awk '/inet /{print $2; exit}' | cut -d/ -f1) - -if [[ -z "$LAN_IF" ]]; then - echo "ERROR: could not determine LAN interface" >&2 - exit 1 -fi - -if [[ -z "$LAN_IP" ]]; then - echo "ERROR: could not determine LAN IP on $LAN_IF" >&2 - exit 1 -fi - -# We use a public WAN IP (1.2.3.4) on a dummy interface because miniupnpd -# disables port forwarding when the external interface has a private/RFC1918 -# address (treats it as double-NAT). -ip link add plum-wan type dummy -ip addr add 1.2.3.4/24 dev plum-wan -ip link set plum-wan up - -start_miniupnpd() { - local enable_pcp_pmp=$1 - cat > "$RUNDIR/miniupnpd.conf" << EOF -ext_ifname=plum-wan -listening_ip=$LAN_IF -enable_pcp_pmp=$enable_pcp_pmp -# port=0: pick a random HTTP port to avoid conflicts with host services. -port=0 -# Without an allow rule miniupnpd denies all mapping requests by default. -allow 1024-65535 0.0.0.0/0 1024-65535 -EOF - miniupnpd -d -f "$RUNDIR/miniupnpd.conf" > "$RUNDIR/miniupnpd.log" 2>&1 & - MINIUPNPD_PID=$! - sleep 1 - kill -0 "$MINIUPNPD_PID" 2>/dev/null \ - || { echo "ERROR: miniupnpd failed to start" >&2; cat "$RUNDIR/miniupnpd.log" >&2; exit 1; } - echo "miniupnpd started (pid $MINIUPNPD_PID)" -} - -export DEBUG=${DEBUG:-0} - -if [[ "${TEST_PCP:-0}" == "1" ]]; then - # PCP requires the UDP source IP to match the client_address in the MAP request. - # Point the default route at LAN_IP so libplum uses it as both gateway and PCP target. - ip route replace default via "$LAN_IP" dev "$LAN_IF" - start_miniupnpd yes - failed=0 - USE_SYSTEM_NIM=1 vendor/nimbus-build-system/scripts/env.sh \ - nim testNatPcpMapping -d:debug -d:disable_libbacktrace build.nims || failed=1 -else - start_miniupnpd no - failed=0 - USE_SYSTEM_NIM=1 vendor/nimbus-build-system/scripts/env.sh \ - nim testNatPortMapping -d:debug -d:disable_libbacktrace build.nims || failed=1 -fi - -if [[ "${DEBUG:-0}" == "1" ]]; then - echo "--- miniupnpd log ---" - cat "$RUNDIR/miniupnpd.log" 2>/dev/null || true -fi - -[ $failed -eq 0 ] || exit 1 diff --git a/tests/integration/nat/miniupnpd_stub_rdr.c b/tests/integration/nat/miniupnpd_stub_rdr.c index 9caf44b8..1e9be2b0 100644 --- a/tests/integration/nat/miniupnpd_stub_rdr.c +++ b/tests/integration/nat/miniupnpd_stub_rdr.c @@ -1,7 +1,7 @@ /* Stub firewall backend for miniupnpd used in Docker-based tests. * * miniupnpd normally calls iptables/nftables to install port forwarding rules - * when it processes a UPnP/PCP/NAT-PMP mapping request. In a Docker container + * when it processes a UPnP/PCP/NAT-PMP mapping request. In a container * those calls fail because the container lacks the required kernel capabilities, * causing every mapping request to return an error to the client. * diff --git a/tests/integration/nat/node-entrypoint.sh b/tests/integration/nat/node-entrypoint.sh new file mode 100644 index 00000000..b8b08ce4 --- /dev/null +++ b/tests/integration/nat/node-entrypoint.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# Redirect the traffic to our router instead +# of podman's own gateway to put B behind the NAT. +ip route replace default via "$ROUTER_LAN_IP" + +# Fetch the bootstrap SPR (retry: the bootstrap may still be starting). +echo "fetching bootstrap SPR from $BOOTSTRAP_API ..." +spr="" +for _ in $(seq 1 60); do + spr=$(curl -fsS -H 'Accept: text/plain' "$BOOTSTRAP_API/api/storage/v1/spr" || true) + [[ -n "$spr" ]] && break + sleep 1 +done +[[ -n "$spr" ]] || { echo "ERROR: could not fetch bootstrap SPR" >&2; exit 1; } + +# api-bindaddr=0.0.0.0 so the published host port reaches the REST API. +exec /app/build/storage \ + --listen-ip=0.0.0.0 --api-bindaddr=0.0.0.0 \ + --listen-port=8070 --disc-port=8090 --api-port=8080 \ + --bootstrap-node="$spr" \ + --nat-num-peers-to-ask=1 --nat-max-queue-size=1 \ + --nat-min-confidence=1.0 --nat-schedule-interval=30s \ + --data-dir=/data --log-level=DEBUG diff --git a/tests/integration/nat/not-reachable/README.md b/tests/integration/nat/not-reachable/README.md new file mode 100644 index 00000000..17762bb0 --- /dev/null +++ b/tests/integration/nat/not-reachable/README.md @@ -0,0 +1,55 @@ +# NAT not-reachable scenario + +## Scenario + +A node behind a NAT that cannot be reached from outside must be detected +`NotReachable` and fall back to bootstrap A's relay. + +## Topology + +``` +node B ──── lan ──── router (NAT) ──── wan ──── bootstrap A +``` + +- **bootstrap A** — public node on the wan, runs the relay + autonat server, + started with `--nat=extip` so it advertises its own public address. +- **router** — two interfaces (lan + wan). Does `lan -> wan` masquerade and *no* + inbound forward, so B can dial out but nothing can dial back in. +- **node B** — `nat=auto`, on the lan. Its default route points at the router, + so all wan-bound traffic is NATed. It fetches A's SPR over A's API to join, + then AutoNAT probes A and finds itself unreachable. + +The wan uses a real public range because our address policy keeps only public +dialable addresses: a private observed address would be filtered out and AutoNAT +would stay `Unknown` instead of `NotReachable`. The wan is `internal` so that +range never leaks to host routes. + +## Run + +```bash +make testNatIntegration STORAGE_INTEGRATION_TEST_INCLUDES=not-reachable +``` + +Runs this scenario (omit the var to run every NAT scenario). Builds the shared +image and runs `testnotreachable.nim`, which brings the compose topology up and +down. Rootless, but needs the host netfilter modules — if the router fails on +iptables: `sudo modprobe iptable_nat nf_conntrack`. + +## Expected result + +B ends up `NotReachable` with the relay running, announcing only its circuit +(relay) address — never a direct one. Its `debug/info`: + +```json +{ + "nat": { + "reachability": "NotReachable", + "clientMode": true, + "relayRunning": true, + "portMapping": "none" + } +} +``` + +Per-run container logs (router, bootstrap, node) are written before teardown to +`tests/integration/logs/__NAT_not_reachable//.log`. diff --git a/tests/integration/nat/not-reachable/compose.yml b/tests/integration/nat/not-reachable/compose.yml new file mode 100644 index 00000000..400ef872 --- /dev/null +++ b/tests/integration/nat/not-reachable/compose.yml @@ -0,0 +1,91 @@ +# A node behind a NAT that can't be reached from outside must be detected +# NotReachable and fall back to the relay. This checks it on a real container +# network with real iptables NAT, not the in-process simulation the unit tests +# use. Run via testnotreachable.nim. +# +# node B ──── lan ──── router (NAT) ──── wan ──── bootstrap A +name: nat-not-reachable + +# Topology addresses, named for their role (defined once, referenced below). +x-addresses: + # fake public internet; a routable range so B looks public to A + wan_subnet: &wan_subnet 7.7.7.0/24 + # private network behind the NAT + lan_subnet: &lan_subnet 10.99.0.0/24 + # A: public bootstrap, relay + autonat server + bootstrap_ip: &bootstrap_ip 7.7.7.10 + # router's public face + router_wan_ip: &router_wan_ip 7.7.7.2 + # router's private face = B's gateway + router_lan_ip: &router_lan_ip 10.99.0.2 + # B, behind the NAT + node_ip: &node_ip 10.99.0.10 + +networks: + wan: + # Keep the fake public range private, not exposed to the host + internal: true + ipam: + config: + - subnet: *wan_subnet + lan: + ipam: + config: + - subnet: *lan_subnet + +services: + router: + image: localhost/storage-nat + cap_add: [NET_ADMIN] + sysctls: + net.ipv4.ip_forward: 1 + networks: + wan: + ipv4_address: *router_wan_ip + lan: + ipv4_address: *router_lan_ip + environment: + ROUTER_WAN_IP: *router_wan_ip + LAN_SUBNET: *lan_subnet + # scripts mounted, so editing them needs no image rebuild + volumes: + - ../router-common.sh:/scripts/router-common.sh:ro,z + - ./router-entrypoint.sh:/scripts/router-entrypoint.sh:ro,z + entrypoint: ["bash", "/scripts/router-entrypoint.sh"] + + bootstrap: + image: localhost/storage-nat + networks: + wan: + ipv4_address: *bootstrap_ip + entrypoint: ["/app/build/storage"] + command: + - --listen-ip=0.0.0.0 + - --api-bindaddr=0.0.0.0 + - --listen-port=8070 + - --disc-port=8090 + - --api-port=8080 + # bootstrap_ip (anchors can't go inside a string) + - --nat=extip:7.7.7.10 + - --relay-server + - --autonat-server + - --no-bootstrap-node + - --data-dir=/data + - --log-level=DEBUG + + node: + image: localhost/storage-nat + cap_add: [NET_ADMIN] + depends_on: [router, bootstrap] + networks: + lan: + ipv4_address: *node_ip + ports: + - "127.0.0.1:18080:8080" + environment: + ROUTER_LAN_IP: *router_lan_ip + # B fetches A's SPR from this API at startup to join the network (bootstrap_ip) + BOOTSTRAP_API: http://7.7.7.10:8080 + volumes: + - ../node-entrypoint.sh:/scripts/node-entrypoint.sh:ro,z + entrypoint: ["bash", "/scripts/node-entrypoint.sh"] diff --git a/tests/integration/nat/not-reachable/router-entrypoint.sh b/tests/integration/nat/not-reachable/router-entrypoint.sh new file mode 100755 index 00000000..21a8ef3b --- /dev/null +++ b/tests/integration/nat/not-reachable/router-entrypoint.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +source "$(dirname "$0")/router-common.sh" + +echo "router ready (wan iface $wanif)" + +hold_until_stopped diff --git a/tests/integration/nat/not-reachable/testnotreachable.nim b/tests/integration/nat/not-reachable/testnotreachable.nim new file mode 100644 index 00000000..74b83f2c --- /dev/null +++ b/tests/integration/nat/not-reachable/testnotreachable.nim @@ -0,0 +1,68 @@ +## NAT not-reachable scenario — node behind a real NAT falls back to relay. +## +## Requires podman-compose and the scenario image: +## podman build -t localhost/storage-nat:not-reachable \ +## -f tests/integration/nat/not-reachable/Dockerfile . + +import std/[json, os, sequtils, strutils, times] +import pkg/chronos +import pkg/questionable/results + +import ../../../asynctest +import ../../../checktest +import ../../storageclient +import ../composehelper + +const + composeFile = currentSourcePath.parentDir / "compose.yml" + nodeApiUrl = "http://127.0.0.1:18080/api/storage/v1" + suiteName = "NAT not reachable" + testName = "node behind NAT is NotReachable and falls back to relay" + services = ["router", "bootstrap", "node"] + detectTimeout = 300_000 # ms + pollInterval = 5_000 # ms + +proc announcesCircuitAddr(info: JsonNode): bool = + info{"announceAddresses"}.getElems.anyIt("p2p-circuit" in it.getStr) + +asyncchecksuite suiteName: + let + composeFile = composeFile + nodeApiUrl = nodeApiUrl + suiteName = suiteName + testName = testName + services = services + startTime = now().format("yyyy-MM-dd'_'HH:mm:ss") + var client: StorageClient + + setup: + compose(composeFile, "up -d") + client = StorageClient.new(nodeApiUrl) + + teardown: + await client.close() + saveContainerLogs(composeFile, suiteName, testName, startTime, services) + compose(composeFile, "down -v") + + test testName: + # Wait for the announcements, after the relay reservation is created. + check eventuallySafe( + block: + var settled = false + try: + let info = await client.info() + settled = info.isOk and info.get.announcesCircuitAddr() + except HttpError: + # B's API is not up yet, keep polling + discard + settled, + timeout = detectTimeout, + pollInterval = pollInterval, + ) + + let info = (await client.info()).get + let nat = info{"nat"} + check nat{"reachability"}.getStr == "NotReachable" + check nat{"relayRunning"}.getBool + check nat{"portMapping"}.getStr == "none" + check info.announcesCircuitAddr() diff --git a/tests/integration/nat/reachable/README.md b/tests/integration/nat/reachable/README.md new file mode 100644 index 00000000..5289eed0 --- /dev/null +++ b/tests/integration/nat/reachable/README.md @@ -0,0 +1,54 @@ +# NAT reachable scenario + +## Scenario + +A node behind a NAT whose port is forwarded must be detected `Reachable` and +keep its direct address — no relay fallback. + +## Topology + +``` +node B ──── lan ──── router (NAT + port forward) ──── wan ──── bootstrap A +``` + +- **bootstrap A** — public node on the wan, runs the relay + autonat server. +- **router** — `lan -> wan` masquerade *plus* a static DNAT forwarding B's TCP + listen port (8070) and UDP disc port (8090) inbound. No miniupnpd: the router + opens the port itself, so B maps nothing. +- **node B** — `nat=auto`, on the lan, default route through the router. It dials + out from its listen port (8070) and the masquerade keeps that port, so A + observes it at `7.7.7.2:8070` — exactly what the DNAT forwards back, so the + dial-back reaches it. + +The wan public range and `internal` flag work as in +[not-reachable](../not-reachable/README.md). + +## Run + +```bash +make testNatIntegration STORAGE_INTEGRATION_TEST_INCLUDES=reachable +``` + +Runs this scenario (omit the var to run every NAT scenario). Builds the shared +image and runs `testreachable.nim`, which brings the compose topology up and +down. Rootless, but needs the host netfilter modules — if the router fails on +iptables: `sudo modprobe iptable_nat nf_conntrack`. + +## Expected result + +B ends up `Reachable`, the relay not running, announcing its direct address — +not a circuit one. Its `debug/info`: + +```json +{ + "nat": { + "reachability": "Reachable", + "clientMode": false, + "relayRunning": false, + "portMapping": "none" + } +} +``` + +Per-run container logs (router, bootstrap, node) are written before teardown to +`tests/integration/logs/__NAT_reachable//.log`. diff --git a/tests/integration/nat/reachable/compose.yml b/tests/integration/nat/reachable/compose.yml new file mode 100644 index 00000000..77da9992 --- /dev/null +++ b/tests/integration/nat/reachable/compose.yml @@ -0,0 +1,92 @@ +# Same setup as not-reachable, but the router forwards B's port (DNAT), so +# AutoNAT's dial-back reaches B and it is detected Reachable — no relay needed. +# +# node B ──── lan ──── router (NAT + port forward) ──── wan ──── bootstrap A +name: nat-reachable + +# Topology addresses, named for their role (defined once, referenced below). +x-addresses: + # fake public internet; a routable range so B looks public to A + wan_subnet: &wan_subnet 7.7.7.0/24 + # private network behind the NAT + lan_subnet: &lan_subnet 10.99.0.0/24 + # A: public bootstrap, relay + autonat server + bootstrap_ip: &bootstrap_ip 7.7.7.10 + # router's public face + router_wan_ip: &router_wan_ip 7.7.7.2 + # router's private face = B's gateway + router_lan_ip: &router_lan_ip 10.99.0.2 + # B, behind the NAT (also the DNAT target) + node_ip: &node_ip 10.99.0.10 + +networks: + wan: + # Keep the fake public range private, not exposed to the host + internal: true + ipam: + config: + - subnet: *wan_subnet + lan: + ipam: + config: + - subnet: *lan_subnet + +services: + router: + image: localhost/storage-nat + cap_add: [NET_ADMIN] + sysctls: + net.ipv4.ip_forward: 1 + networks: + wan: + ipv4_address: *router_wan_ip + lan: + ipv4_address: *router_lan_ip + environment: + ROUTER_WAN_IP: *router_wan_ip + LAN_SUBNET: *lan_subnet + # where the router forwards the port + NODE_IP: *node_ip + # scripts mounted, not baked, so editing them needs no image rebuild + volumes: + - ../router-common.sh:/scripts/router-common.sh:ro,z + - ./router-entrypoint.sh:/scripts/router-entrypoint.sh:ro,z + entrypoint: ["bash", "/scripts/router-entrypoint.sh"] + + bootstrap: + image: localhost/storage-nat + networks: + wan: + ipv4_address: *bootstrap_ip + entrypoint: ["/app/build/storage"] + command: + - --listen-ip=0.0.0.0 + - --api-bindaddr=0.0.0.0 + - --listen-port=8070 + - --disc-port=8090 + - --api-port=8080 + # bootstrap_ip (anchors can't go inside a string) + - --nat=extip:7.7.7.10 + - --relay-server + - --autonat-server + - --no-bootstrap-node + - --data-dir=/data + - --log-level=DEBUG + + node: + image: localhost/storage-nat + cap_add: [NET_ADMIN] + depends_on: [router, bootstrap] + networks: + lan: + ipv4_address: *node_ip + # B's API, published so the test can poll it + ports: + - "127.0.0.1:18081:8080" + environment: + ROUTER_LAN_IP: *router_lan_ip + # B fetches A's SPR from this API at startup to join the network (bootstrap_ip) + BOOTSTRAP_API: http://7.7.7.10:8080 + volumes: + - ../node-entrypoint.sh:/scripts/node-entrypoint.sh:ro,z + entrypoint: ["bash", "/scripts/node-entrypoint.sh"] diff --git a/tests/integration/nat/reachable/router-entrypoint.sh b/tests/integration/nat/reachable/router-entrypoint.sh new file mode 100755 index 00000000..1aa07e06 --- /dev/null +++ b/tests/integration/nat/reachable/router-entrypoint.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +source "$(dirname "$0")/router-common.sh" + +# Forward the node's TCP listen port (what AutoNAT dials back) and UDP disc port +# in order to simulate the port mapping. +iptables -t nat -A PREROUTING -i "$wanif" -p tcp --dport 8070 -j DNAT --to-destination "$NODE_IP:8070" +iptables -t nat -A PREROUTING -i "$wanif" -p udp --dport 8090 -j DNAT --to-destination "$NODE_IP:8090" + +echo "router ready (forwarding tcp/8070 + udp/8090 to $NODE_IP, wan iface $wanif)" + +hold_until_stopped diff --git a/tests/integration/nat/reachable/testreachable.nim b/tests/integration/nat/reachable/testreachable.nim new file mode 100644 index 00000000..93e26bdb --- /dev/null +++ b/tests/integration/nat/reachable/testreachable.nim @@ -0,0 +1,77 @@ +## NAT reachable scenario — node behind a real NAT is Reachable because the +## router forwards its port. +## +## Same shape as the not-reachable test: compose.yml brings up a real NAT +## topology, but the router has a static inbound port-forward (DNAT) to the node. +## AutoNAT's dial-back reaches the node, so it is detected Reachable (no relay) — +## a manual port-forward / endpoint-independent NAT, no miniupnpd. +## +## Requires podman-compose and the scenario image: +## podman build -t localhost/storage-nat:reachable \ +## -f tests/integration/nat/reachable/Dockerfile . + +import std/[json, os, sequtils, strutils, times] +import pkg/chronos +import pkg/questionable/results + +import ../../../asynctest +import ../../../checktest +import ../../storageclient +import ../composehelper + +const + composeFile = currentSourcePath.parentDir / "compose.yml" + nodeApiUrl = "http://127.0.0.1:18081/api/storage/v1" + suiteName = "NAT reachable" + testName = "node behind NAT with a forwarded port is Reachable" + services = ["router", "bootstrap", "node"] + detectTimeout = 120_000 # ms + pollInterval = 5_000 # ms + +proc announcesDirectAddr(info: JsonNode): bool = + ## A reachable node announces at least one direct (non-circuit) address. + info{"announceAddresses"}.getElems.anyIt("p2p-circuit" notin it.getStr) + +asyncchecksuite suiteName: + # chronos' async setup/teardown cannot reference module-level GC'ed consts + # (strings/seqs), so rebind the ones they use to suite locals. + let + composeFile = composeFile + nodeApiUrl = nodeApiUrl + suiteName = suiteName + testName = testName + services = services + startTime = now().format("yyyy-MM-dd'_'HH:mm:ss") + var client: StorageClient + + setup: + compose(composeFile, "up -d") + client = StorageClient.new(nodeApiUrl) + + teardown: + await client.close() + saveContainerLogs(composeFile, suiteName, testName, startTime, services) + compose(composeFile, "down -v") + + test testName: + # Reachable is the settling signal: wait for it, then assert each expected + # property separately so a failure points at the exact condition. + check eventuallySafe( + block: + var reachable = false + try: + let info = await client.info() + reachable = + info.isOk and info.get{"nat"}{"reachability"}.getStr == "Reachable" + except HttpError: + discard # B's API is not up yet, keep polling + reachable, + timeout = detectTimeout, + pollInterval = pollInterval, + ) + + let info = (await client.info()).get + let nat = info{"nat"} + check nat{"reachability"}.getStr == "Reachable" + check nat{"relayRunning"}.getBool == false + check info.announcesDirectAddr() diff --git a/tests/integration/nat/router-common.sh b/tests/integration/nat/router-common.sh new file mode 100644 index 00000000..66461f77 --- /dev/null +++ b/tests/integration/nat/router-common.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +# Shared router base, sourced by each scenario's router-entrypoint.sh + +set -euo pipefail + +# iptables needs the wan interface's name (eth0/eth1), but podman assigns those +# names arbitrarily — so look for the name using the wan IP, +# defined in the compose file. +wanif=$(ip -o -4 addr show | awk -v ip="$ROUTER_WAN_IP" '$0 ~ ip {print $2; exit}') + +if ! iptables -t nat -A POSTROUTING -s "$LAN_SUBNET" -o "$wanif" -j MASQUERADE; then + echo "ERROR: iptables NAT failed. Load netfilter modules on the host:" >&2 + echo " sudo modprobe iptable_nat nf_conntrack" >&2 + exit 1 +fi +iptables -P FORWARD ACCEPT + +# Block until `compose down`. sleep runs in the background so the SIGTERM trap +# fires immediately instead of waiting for sleep to return. +hold_until_stopped() { + trap 'exit 0' TERM INT + sleep infinity & + wait +} diff --git a/tests/testIntegration.nim b/tests/testIntegration.nim index 5e289ff5..488e4136 100644 --- a/tests/testIntegration.nim +++ b/tests/testIntegration.nim @@ -11,7 +11,8 @@ when includes != "": # import only the specified tests importAll(includes.split(",")) else: - # import all tests in the integration/ directory - importTests(currentSourcePath().parentDir() / "integration") + # all tests in integration/, except the nat/ real-topology scenarios, which + # need podman + the storage-nat image and run via testNatIntegration instead + importTests(currentSourcePath().parentDir() / "integration", "/nat/", "") {.warning[UnusedImport]: off.} diff --git a/tests/testNatIntegration.nim b/tests/testNatIntegration.nim new file mode 100644 index 00000000..d5d116c0 --- /dev/null +++ b/tests/testNatIntegration.nim @@ -0,0 +1,16 @@ +import std/os +import ./imports + +## Real-topology NAT scenarios (need podman + the storage-nat image). +## Run a single scenario by setting its folder name during compilation, e.g. +## STORAGE_INTEGRATION_TEST_INCLUDES=reachable +const scenario = getEnv("STORAGE_INTEGRATION_TEST_INCLUDES") +const only = + if scenario.len > 0: + "/" & scenario & "/" + else: + "" + +importTests(currentSourcePath().parentDir() / "integration" / "nat", "", only) + +{.warning[UnusedImport]: off.} diff --git a/tests/testStorage.nim b/tests/testStorage.nim index 609a50ac..55b22d4b 100644 --- a/tests/testStorage.nim +++ b/tests/testStorage.nim @@ -1,6 +1,6 @@ import std/os import ./imports -importTests(currentSourcePath().parentDir() / "storage") +importTests(currentSourcePath().parentDir() / "storage", "", "") {.warning[UnusedImport]: off.} diff --git a/tools/scripts/ci-job-matrix.sh b/tools/scripts/ci-job-matrix.sh index 15b5db96..9047df42 100755 --- a/tools/scripts/ci-job-matrix.sh +++ b/tools/scripts/ci-job-matrix.sh @@ -95,9 +95,10 @@ integration_test () { integration_test_job $tests done - # fail when there are integration tests with an unknown duration + # fail when there are integration tests with an unknown duration. nat/ is + # excluded: those real-topology scenarios run via the nat-integration job. local filter='1_minute\|5_minutes\|30_minutes' - local unknown=$(find_tests tests/integration | grep -v "$filter") + local unknown=$(find_tests tests/integration | grep -v "$filter" | grep -v '/nat/') if [ "$unknown" != "" ]; then echo "Error: Integration tests need to be in either the 1_minute," >&2 echo " 5_minutes, or 30_minutes directory, based on the maximum" >&2 @@ -117,13 +118,10 @@ libstorage_test () { job } -# outputs NAT integration test jobs -# Linux-only: miniupnpd is a Linux daemon, network namespace manipulation requires Linux +# outputs the NAT real-topology integration job (all scenarios). +# Linux-only: needs network-namespace + iptables manipulation. nat_integration_tests () { - job_tests="nat-upnp-integration" - job_includes="" - job - job_tests="nat-pcp-integration" + job_tests="nat-integration" job_includes="" job }