Merge branch 'nat'

.appveyor.yml

@@ -1,11 +1,11 @@
 version: '{build}'
 
+image: Visual Studio 2015
+
 cache:
 - sqlite-dll-win32-x86-3240000.zip -> .appveyor.yml
 - sqlite-dll-win64-x64-3240000.zip -> .appveyor.yml
 - nimbus-deps.zip -> .appveyor.yml
-- x86_64-4.9.2-release-win32-seh-rt_v4-rev4.7z -> .appveyor.yml
-- i686-4.9.2-release-win32-dwarf-rt_v4-rev4.7z -> .appveyor.yml
 - Nim -> .appveyor.yml
 
 matrix:
@@ -21,33 +21,24 @@ install:
   - setlocal EnableExtensions EnableDelayedExpansion
 
   - IF "%PLATFORM%" == "x86" (
-      SET "MINGW_ARCHIVE=i686-4.9.2-release-win32-dwarf-rt_v4-rev4.7z" &
-      SET "MINGW_URL=https://sourceforge.net/projects/mingw-w64/files/Toolchains%%20targetting%%20Win32/Personal%%20Builds/mingw-builds/4.9.2/threads-win32/dwarf/i686-4.9.2-release-win32-dwarf-rt_v4-rev4.7z" &
-      SET "MINGW_DIR=mingw32" &
       SET "SQLITE_URL=https://www.sqlite.org/2018/sqlite-dll-win32-x86-3240000.zip" &
       SET "SQLITE_ARCHIVE=sqlite-dll-win32-x86-3240000.zip" &
       SET "ROCKSDB_URL=https://github.com/status-im/nimbus-deps/releases/download/nimbus-deps/nimbus-deps.zip" &
       SET "ROCKSDB_ARCHIVE=nimbus-deps.zip"
-    ) ELSE (
+    )
-      IF "%PLATFORM%" == "x64" (
+  - IF "%PLATFORM%" == "x64" (
-        SET "MINGW_ARCHIVE=x86_64-4.9.2-release-win32-seh-rt_v4-rev4.7z" &
-        SET "MINGW_URL=https://sourceforge.net/projects/mingw-w64/files/Toolchains%%20targetting%%20Win64/Personal%%20Builds/mingw-builds/4.9.2/threads-win32/seh/x86_64-4.9.2-release-win32-seh-rt_v4-rev4.7z" &
-        SET "MINGW_DIR=mingw64" &
       SET "SQLITE_URL=https://www.sqlite.org/2018/sqlite-dll-win64-x64-3240000.zip" &
       SET "SQLITE_ARCHIVE=sqlite-dll-win64-x64-3240000.zip" &
       SET "ROCKSDB_URL=https://github.com/status-im/nimbus-deps/releases/download/nimbus-deps/nimbus-deps.zip" &
       SET "ROCKSDB_ARCHIVE=nimbus-deps.zip"
-      ) else (
-        echo "Unknown platform"
-      )
     )
 
+  # use the newest versions documented here: https://www.appveyor.com/docs/windows-images-software/#mingw-msys-cygwin
+  - IF "%PLATFORM%" == "x86" SET PATH=C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin;%PATH%
+  - IF "%PLATFORM%" == "x64" SET PATH=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%PATH%
+
   - MKDIR %CD%\bin
-  - SET PATH=%CD%\%MINGW_DIR%\bin;%CD%\bin;%CD%\Nim\bin;%PATH%
+  - SET PATH=%CD%\bin;%CD%\Nim\bin;%PATH%
-
-  # Unpack mingw
-  - IF NOT EXIST "%MINGW_ARCHIVE%" appveyor DownloadFile "%MINGW_URL%" -FileName "%MINGW_ARCHIVE%"
-  - 7z x -y "%MINGW_ARCHIVE%" > nul
 
   # Unpack sqlite
   - IF not exist "%SQLITE_ARCHIVE%" appveyor DownloadFile "%SQLITE_URL%" -FileName "%SQLITE_ARCHIVE%"
@@ -102,7 +93,8 @@ install:
 
 build_script:
   - cd C:\projects\%APPVEYOR_PROJECT_SLUG%
-  - nimble install -y
+  - bash -c "nimble install -y"
+
 test_script:
   - nimble test
 

Makefile

@@ -14,7 +14,7 @@ TOOLS := beacon_node validator_keygen bench_bls_sig_agggregation state_sim
 TOOLS_DIRS := beacon_chain benchmarks research
 TOOLS_CSV := $(subst $(SPACE),$(COMMA),$(TOOLS))
 
-.PHONY: all sanity-checks deps test $(TOOLS) clean_eth2_network_simulation_files eth2_network_simulation clean-testnet0 testnet0-nocleaning testnet0 clean-testnet1 testnet1-nocleaning testnet1 clean
+.PHONY: all sanity-checks deps nat-libs test $(TOOLS) clean_eth2_network_simulation_files eth2_network_simulation clean-testnet0 testnet0-nocleaning testnet0 clean-testnet1 testnet1-nocleaning testnet1 clean
 
 all: | $(TOOLS)
 
@@ -30,12 +30,15 @@ deps: | sanity-checks
 build:
 	mkdir $@
 
+nat-libs: | deps
+	+ $(MAKE) --silent -C ../../ nat-libs
+
 # Windows 10 with WSL enabled, but no distro installed, fails if "../../nimble.sh" is executed directly
 # in a Makefile recipe but works when prefixing it with `bash`. No idea how the PATH is overridden.
-test: | build deps
+test: | build deps nat-libs
 	bash ../../nimble.sh test $(NIM_PARAMS)
 
-$(TOOLS): | build deps
+$(TOOLS): | build deps nat-libs
 	for D in $(TOOLS_DIRS); do [ -e "$${D}/$@.nim" ] && TOOL_DIR="$${D}" && break; done && \
 		echo -e $(BUILD_MSG) "build/$@" && \
 		$(ENV_SCRIPT) nim c $(NIM_PARAMS) -o:build/$@ "$${TOOL_DIR}/$@.nim"
@@ -46,18 +49,15 @@ clean_eth2_network_simulation_files:
 eth2_network_simulation: | beacon_node validator_keygen clean_eth2_network_simulation_files
 	SKIP_BUILDS=1 GIT_ROOT="$$PWD" BUILD_OUTPUTS_DIR="./build" tests/simulation/start.sh
 
+testnet0 testnet1: | build deps nat-libs
+	../../env.sh scripts/build_testnet_node.sh $@
+
 clean-testnet0:
 	rm -rf ~/.cache/nimbus/BeaconNode/testnet0
 
-testnet0: | build deps
-	../../env.sh scripts/build_testnet_node.sh testnet0
-
 clean-testnet1:
 	rm -rf ~/.cache/nimbus/BeaconNode/testnet1
 
-testnet1: | build deps
-	../../env.sh scripts/build_testnet_node.sh testnet1
-
 clean:
 	rm -rf build/{$(TOOLS_CSV),all_tests,*_node,*.exe} nimcache
 

beacon_chain/conf.nim

@@ -59,7 +59,7 @@ type
         defaultValue: defaultPort(config) .}: int
 
       nat* {.
-        desc: "Specify method to use for determining public address. Must be one of: any, extip:<IP>"
+        desc: "Specify method to use for determining public address. Must be one of: any, none, upnp, pmp, extip:<IP>"
         defaultValue: "any" .}: string
 
       validators* {.

beacon_chain/eth2_network.nim

@@ -1,5 +1,6 @@
 import
   options, chronos, json_serialization, strutils,
+  chronicles,
   spec/digest, version, conf
 
 const
@@ -8,7 +9,7 @@ const
 when useRLPx:
   import
     os,
-    eth/[rlp, p2p, keys], gossipsub_protocol,
+    eth/[rlp, p2p, keys, net/nat], gossipsub_protocol,
     eth/p2p/peer_pool # for log on connected peers
 
   export
@@ -23,13 +24,40 @@ when useRLPx:
 
   template libp2pProtocol*(name, version: string) {.pragma.}
 
-  func parseNat(nat: string): IpAddress =
+  proc setupNat(conf: BeaconNodeConf): tuple[ip: IpAddress, tcpPort: Port, udpPort: Port] =
-    # TODO we should try to discover the actual external IP, in case we're
+    # defaults
-    #      behind a nat / upnp / etc..
+    result.ip = parseIpAddress("127.0.0.1")
-    if nat.startsWith("extip:"):
+    result.tcpPort = Port(conf.tcpPort)
-      parseIpAddress(nat[6..^1])
+    result.udpPort = Port(conf.udpPort)
+
+    var nat: NatStrategy
+    case conf.nat.toLowerAscii:
+      of "any":
+        nat = NatAny
+      of "none":
+        nat = NatNone
+      of "upnp":
+        nat = NatUpnp
+      of "pmp":
+        nat = NatPmp
       else:
-      parseIpAddress("127.0.0.1")
+        if conf.nat.startsWith("extip:") and isIpAddress(conf.nat[6..^1]):
+          # any required port redirection is assumed to be done by hand
+          result.ip = parseIpAddress(conf.nat[6..^1])
+          nat = NatNone
+        else:
+          error "not a valid NAT mechanism, nor a valid IP address", value = conf.nat
+          quit(QuitFailure)
+
+    if nat != NatNone:
+      let extIP = getExternalIP(nat)
+      if extIP.isSome:
+        result.ip = extIP.get()
+        let extPorts = redirectPorts(tcpPort = result.tcpPort,
+                                      udpPort = result.udpPort,
+                                      description = clientId)
+        if extPorts.isSome:
+          (result.tcpPort, result.udpPort) = extPorts.get()
 
   proc ensureNetworkKeys*(conf: BeaconNodeConf): KeyPair =
     let privateKeyFile = conf.dataDir / "network.privkey"
@@ -60,9 +88,10 @@ when useRLPx:
   proc createEth2Node*(conf: BeaconNodeConf): Future[EthereumNode] {.async.} =
     let
       keys = ensureNetworkKeys(conf)
-      address = Address(ip: parseNat(conf.nat),
+      (ip, tcpPort, udpPort) = setupNat(conf)
-                        tcpPort: Port conf.tcpPort,
+      address = Address(ip: ip,
-                        udpPort: Port conf.udpPort)
+                        tcpPort: tcpPort,
+                        udpPort: udpPort)
 
     # TODO there are more networking options to add here: local bind ip, ipv6
     #      etc.
@@ -80,7 +109,7 @@ when useRLPx:
 
 else:
   import
-    libp2p/daemon/daemonapi, chronicles,
+    libp2p/daemon/daemonapi,
     libp2p_backend
 
   export

tests/simulation/run_node.sh

@@ -10,9 +10,9 @@ DATA_DIR="${SIMULATION_DIR}/node-${1}"
 V_PREFIX="${VALIDATORS_DIR}/v$(printf '%06d' ${1})"
 PORT=$(printf '5%04d' ${1})
 
-NAT_FLAG=""
+NAT_FLAG="--nat:none"
 if [ "${NAT:-}" == "1" ]; then
-  NAT_FLAG="--nat:extip:$(curl -s ifconfig.me)"
+  NAT_FLAG="--nat:any"
 fi
 
 FIRST_VALIDATOR_IDX=$(printf '%07d' $(( (NUM_VALIDATORS / ($NUM_NODES + $NUM_MISSING_NODES)) * $1 )))