From aef7a25174da4442cbc925af9aef3aed3904c1d1 Mon Sep 17 00:00:00 2001 From: Kim De Mey Date: Wed, 16 Jun 2021 21:18:45 +0200 Subject: [PATCH] Add nlpn (#714) * Add start of nlpn client * Fix some error handling in nlpn * Clean-up GOARCH from nlpn.yml and add vendor to include paths --- .github/workflows/ci.yml | 3 +- .github/workflows/nlpn.yml | 192 +++++++++++++++++++++++++++++++++++++ Makefile | 7 +- nimbus.nimble | 3 + nlpn/conf.nim | 130 +++++++++++++++++++++++++ nlpn/nlpn.nim | 66 +++++++++++++ 6 files changed, 399 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/nlpn.yml create mode 100644 nlpn/conf.nim create mode 100644 nlpn/nlpn.nim diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c855eb64d..74be8464b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,7 +1,8 @@ name: CI on: push: - paths-ignore: ['doc/**', 'docs/**', '**/*.md', 'hive_integration/**'] + paths-ignore: ['doc/**', 'docs/**', '**/*.md', 'hive_integration/**', + 'nlpn/**', '.github/workflows/nlpn.yml'] # Disable `pull_request`. Experimenting with using only `push` for PRs. #pull_request: # paths-ignore: ['doc/**', 'docs/**', '**/*.md', 'hive_integration/**'] diff --git a/.github/workflows/nlpn.yml b/.github/workflows/nlpn.yml new file mode 100644 index 000000000..60e9cca16 --- /dev/null +++ b/.github/workflows/nlpn.yml @@ -0,0 +1,192 @@ +name: nlpn CI +on: + push: + paths: ['nlpn/**', '.github/workflows/nlpn.yml', 'vendor/**', 'Makefile', + 'nimbus.nimble'] + pull_request: + paths: ['nlpn/**', '.github/workflows/nlpn.yml', 'vendor/**', 'Makefile', + 'nimbus.nimble'] + +jobs: + build: + strategy: + fail-fast: false + max-parallel: 20 + matrix: + target: + - os: linux + cpu: amd64 + - os: linux + cpu: i386 + - os: macos + cpu: amd64 + - os: windows + cpu: amd64 + - os: windows + cpu: i386 + include: + - target: + os: linux + builder: ubuntu-18.04 + shell: bash + - target: + os: macos + builder: macos-10.15 + shell: bash + - target: + os: windows + builder: windows-latest + shell: msys2 {0} + + defaults: + run: + shell: ${{ matrix.shell }} + + name: '${{ matrix.target.os }}-${{ matrix.target.cpu }}' + runs-on: ${{ matrix.builder }} + steps: + - name: Checkout nimbus-eth1 + uses: actions/checkout@v2 + + - name: Derive environment variables + shell: bash + run: | + if [[ '${{ matrix.target.cpu }}' == 'amd64' ]]; then + PLATFORM=x64 + else + PLATFORM=x86 + fi + echo "PLATFORM=${PLATFORM}" >> $GITHUB_ENV + + # libminiupnp / natpmp + if [[ '${{ runner.os }}' == 'Linux' && '${{ matrix.target.cpu }}' == 'i386' ]]; then + export CFLAGS="${CFLAGS} -m32 -mno-adx" + echo "CFLAGS=${CFLAGS}" >> $GITHUB_ENV + fi + + ncpu='' + case '${{ runner.os }}' in + 'Linux') + ncpu=$(nproc) + ;; + 'macOS') + ncpu=$(sysctl -n hw.ncpu) + ;; + 'Windows') + ncpu=${NUMBER_OF_PROCESSORS} + ;; + esac + [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1 + echo "ncpu=${ncpu}" >> $GITHUB_ENV + + - name: Install build dependencies (Linux i386) + if: runner.os == 'Linux' && matrix.target.cpu == 'i386' + run: | + sudo dpkg --add-architecture i386 + sudo apt-fast update -qq + sudo DEBIAN_FRONTEND='noninteractive' apt-fast install \ + --no-install-recommends -yq gcc-multilib g++-multilib + mkdir -p external/bin + cat << EOF > external/bin/gcc + #!/bin/bash + exec $(which gcc) -m32 -mno-adx "\$@" + EOF + cat << EOF > external/bin/g++ + #!/bin/bash + exec $(which g++) -m32 -mno-adx "\$@" + EOF + chmod 755 external/bin/gcc external/bin/g++ + echo "${{ github.workspace }}/external/bin" >> $GITHUB_PATH + + - name: MSYS2 (Windows i386) + if: runner.os == 'Windows' && matrix.target.cpu == 'i386' + uses: msys2/setup-msys2@v2 + with: + path-type: inherit + msystem: MINGW32 + install: >- + base-devel + git + mingw-w64-i686-toolchain + + - name: MSYS2 (Windows amd64) + if: runner.os == 'Windows' && matrix.target.cpu == 'amd64' + uses: msys2/setup-msys2@v2 + with: + path-type: inherit + install: >- + base-devel + git + mingw-w64-x86_64-toolchain + + - name: Restore Nim DLLs dependencies (Windows) from cache + if: runner.os == 'Windows' + id: windows-dlls-cache + uses: actions/cache@v2 + with: + path: external/dlls-${{ matrix.target.cpu }} + key: 'dlls-${{ matrix.target.cpu }}' + + - name: Install DLLs dependencies (Windows) + if: > + steps.windows-dlls-cache.outputs.cache-hit != 'true' && + runner.os == 'Windows' + run: | + DLLPATH=external/dlls-${{ matrix.target.cpu }} + mkdir -p external + curl -L "https://nim-lang.org/download/windeps.zip" -o external/windeps.zip + 7z x -y external/windeps.zip -o"$DLLPATH" + + - name: Path to cached dependencies (Windows) + if: > + runner.os == 'Windows' + run: | + echo '${{ github.workspace }}'"/external/dlls-${{ matrix.target.cpu }}" >> $GITHUB_PATH + + - name: Get latest nimbus-build-system commit hash + id: versions + run: | + getHash() { + git ls-remote "https://github.com/$1" "${2:-HEAD}" | cut -f 1 + } + nbsHash=$(getHash status-im/nimbus-build-system) + echo "::set-output name=nimbus_build_system::$nbsHash" + + - name: Restore prebuilt Nim binaries from cache + id: nim-cache + uses: actions/cache@v2 + with: + path: NimBinaries + key: 'nim-${{ matrix.target.os }}-${{ matrix.target.cpu }}-${{ steps.versions.outputs.nimbus_build_system }}' + + - name: Build Nim and Nimbus-eth1 dependencies + run: | + make -j${ncpu} ARCH_OVERRIDE=${PLATFORM} CI_CACHE=NimBinaries update + + - name: Run nlpn tests (Windows) + if: runner.os == 'Windows' + run: | + gcc --version + DEFAULT_MAKE_FLAGS="-j${ncpu}" + mingw32-make ${DEFAULT_MAKE_FLAGS} nlpn + build/nlpn.exe --help + # mingw32-make ${DEFAULT_MAKE_FLAGS} test + + - name: Run nlpn tests (Linux) + if: runner.os == 'Linux' + run: | + export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/lib" + DEFAULT_MAKE_FLAGS="-j${ncpu}" + env CC=gcc make ${DEFAULT_MAKE_FLAGS} nlpn + build/nlpn --help + # CC is needed to select correct compiler 32/64 bit + # env CC=gcc CXX=g++ make ${DEFAULT_MAKE_FLAGS} test test-reproducibility + + - name: Run nlpn tests (Macos) + if: runner.os == 'Macos' + run: | + DEFAULT_MAKE_FLAGS="-j${ncpu}" + make ${DEFAULT_MAKE_FLAGS} nlpn + build/nlpn --help + # "-static" option will not work for osx unless static system libraries are provided + # make ${DEFAULT_MAKE_FLAGS} test test-reproducibility diff --git a/Makefile b/Makefile index 43afa7f65..b90e9005e 100644 --- a/Makefile +++ b/Makefile @@ -29,6 +29,7 @@ TOOLS_CSV := $(subst $(SPACE),$(COMMA),$(TOOLS)) deps \ update \ nimbus \ + nlpn \ test \ test-reproducibility \ clean \ @@ -109,6 +110,10 @@ nimbus: | build deps echo -e $(BUILD_MSG) "build/$@" && \ $(ENV_SCRIPT) nim nimbus $(NIM_PARAMS) nimbus.nims +nlpn: | build deps + echo -e $(BUILD_MSG) "build/$@" && \ + $(ENV_SCRIPT) nim nlpn $(NIM_PARAMS) nimbus.nims + # symlink nimbus.nims: ln -s nimbus.nimble $@ @@ -133,7 +138,7 @@ test-reproducibility: # usual cleaning clean: | clean-common - rm -rf build/{nimbus,$(TOOLS_CSV),all_tests,test_rpc} + rm -rf build/{nimbus,nlpn,$(TOOLS_CSV),all_tests,test_rpc} ifneq ($(USE_LIBBACKTRACE), 0) + $(MAKE) -C vendor/nim-libbacktrace clean $(HANDLE_OUTPUT) endif diff --git a/nimbus.nimble b/nimbus.nimble index 61002a6cf..13b0890b3 100644 --- a/nimbus.nimble +++ b/nimbus.nimble @@ -51,3 +51,6 @@ task test, "Run tests": task nimbus, "Build Nimbus": buildBinary "nimbus", "nimbus/", "-d:chronicles_log_level=TRACE" + +task nlpn, "Build nlpn": + buildBinary "nlpn", "nlpn/", "-d:chronicles_log_level=TRACE" diff --git a/nlpn/conf.nim b/nlpn/conf.nim new file mode 100644 index 000000000..688741448 --- /dev/null +++ b/nlpn/conf.nim @@ -0,0 +1,130 @@ +# Nimbus +# Copyright (c) 2021 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + +{.push raises: [Defect].} + +import + confutils, confutils/std/net, chronicles, + eth/keys, eth/net/nat, eth/p2p/discoveryv5/[enr, node] + +const + DefaultListenAddress* = (static ValidIpAddress.init("0.0.0.0")) + DefaultAdminListenAddress* = (static ValidIpAddress.init("127.0.0.1")) + +type + PortalCmd* = enum + noCommand + + PortalConf* = object + logLevel* {. + defaultValue: LogLevel.DEBUG + desc: "Sets the log level" + name: "log-level" .}: LogLevel + + udpPort* {. + defaultValue: 9009 + desc: "UDP listening port" + name: "udp-port" .}: uint16 + + listenAddress* {. + defaultValue: DefaultListenAddress + desc: "Listening address for the Discovery v5 traffic" + name: "listen-address" }: ValidIpAddress + + bootnodes* {. + desc: "ENR URI of node to bootstrap discovery with. Argument may be repeated" + name: "bootnode" .}: seq[Record] + + nat* {. + desc: "Specify method to use for determining public address. " & + "Must be one of: any, none, upnp, pmp, extip:" + defaultValue: NatConfig(hasExtIp: false, nat: NatAny) + name: "nat" .}: NatConfig + + enrAutoUpdate* {. + defaultValue: false + desc: "Discovery can automatically update its ENR with the IP address " & + "and UDP port as seen by other nodes it communicates with. " & + "This option allows to enable/disable this functionality" + name: "enr-auto-update" .}: bool + + nodeKey* {. + desc: "P2P node private key as hex", + defaultValue: PrivateKey.random(keys.newRng()[]) + name: "nodekey" .}: PrivateKey + + metricsEnabled* {. + defaultValue: false + desc: "Enable the metrics server" + name: "metrics" .}: bool + + metricsAddress* {. + defaultValue: DefaultAdminListenAddress + desc: "Listening address of the metrics server" + name: "metrics-address" .}: ValidIpAddress + + metricsPort* {. + defaultValue: 8008 + desc: "Listening HTTP port of the metrics server" + name: "metrics-port" .}: Port + + rpcEnabled* {. + desc: "Enable the JSON-RPC server" + defaultValue: false + name: "rpc" }: bool + + rpcPort* {. + desc: "HTTP port for the JSON-RPC service" + defaultValue: 8545 + name: "rpc-port" }: Port + + rpcAddress* {. + desc: "Listening address of the RPC server" + defaultValue: DefaultAdminListenAddress + name: "rpc-address" }: ValidIpAddress + + case cmd* {. + command + defaultValue: noCommand .}: PortalCmd + of noCommand: + discard + +proc parseCmdArg*(T: type enr.Record, p: TaintedString): T + {.raises: [Defect, ConfigurationError].} = + if not fromURI(result, p): + raise newException(ConfigurationError, "Invalid ENR") + +proc completeCmdArg*(T: type enr.Record, val: TaintedString): seq[string] = + return @[] + +proc parseCmdArg*(T: type Node, p: TaintedString): T + {.raises: [Defect, ConfigurationError].} = + var record: enr.Record + if not fromURI(record, p): + raise newException(ConfigurationError, "Invalid ENR") + + let n = newNode(record) + if n.isErr: + raise newException(ConfigurationError, $n.error) + + if n[].address.isNone(): + raise newException(ConfigurationError, "ENR without address") + + n[] + +proc completeCmdArg*(T: type Node, val: TaintedString): seq[string] = + return @[] + +proc parseCmdArg*(T: type PrivateKey, p: TaintedString): T + {.raises: [Defect, ConfigurationError].} = + try: + result = PrivateKey.fromHex(string(p)).tryGet() + except CatchableError: + raise newException(ConfigurationError, "Invalid private key") + +proc completeCmdArg*(T: type PrivateKey, val: TaintedString): seq[string] = + return @[] diff --git a/nlpn/nlpn.nim b/nlpn/nlpn.nim new file mode 100644 index 000000000..473344599 --- /dev/null +++ b/nlpn/nlpn.nim @@ -0,0 +1,66 @@ +# Nimbus +# Copyright (c) 2021 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + +{.push raises: [Defect].} + +import + confutils, confutils/std/net, chronicles, chronicles/topics_registry, + chronos, metrics, metrics/chronos_httpserver, + eth/keys, eth/net/nat, + eth/p2p/discoveryv5/protocol as discv5_protocol, + eth/p2p/portal/protocol as portal_protocol, + ./conf + +proc run(config: PortalConf) {.raises: [CatchableError, Defect].} = + let + rng = newRng() + bindIp = config.listenAddress + udpPort = Port(config.udpPort) + # TODO: allow for no TCP port mapping! + (extIp, _, extUdpPort) = + try: setupAddress(config.nat, + config.listenAddress, udpPort, udpPort, "dcli") + except CatchableError as exc: raise exc + # TODO: Ideally we don't have the Exception here + except Exception as exc: raiseAssert exc.msg + + let d = newProtocol(config.nodeKey, + extIp, none(Port), extUdpPort, + bootstrapRecords = config.bootnodes, + bindIp = bindIp, bindPort = udpPort, + enrAutoUpdate = config.enrAutoUpdate, + rng = rng) + + d.open() + + let portal = PortalProtocol.new(d) + + if config.metricsEnabled: + let + address = config.metricsAddress + port = config.metricsPort + notice "Starting metrics HTTP server", + url = "http://" & $address & ":" & $port & "/metrics" + try: + chronos_httpserver.startMetricsHttpServer($address, port) + except CatchableError as exc: raise exc + # TODO: Ideally we don't have the Exception here + except Exception as exc: raiseAssert exc.msg + + d.start() + + runForever() + +when isMainModule: + {.pop.} + let config = PortalConf.load() + {.push raises: [Defect].} + + setLogLevel(config.logLevel) + + case config.cmd + of noCommand: run(config)