diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 18436e9..3edb805 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -156,12 +156,4 @@ jobs: # https://github.com/status-im/nimbus-eth2/issues/3121 export NIMFLAGS="-d:nimRawSetjmp" fi - nim --version - nimble --version - nimble install -y --depsOnly - nimble test - if [[ "${{ matrix.branch }}" == "version-1-6" || "${{ matrix.branch }}" == "devel" ]]; then - echo -e "\nTesting with '--gc:orc':\n" - export NIMFLAGS="${NIMFLAGS} --gc:orc" - nimble test - fi; + make test diff --git a/.gitmodules b/.gitmodules index 4810e25..b78cd85 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,125 @@ [submodule "vendor/nimbus-build-system"] path = vendor/nimbus-build-system url = https://github.com/status-im/nimbus-build-system.git + ignore = untracked + branch = master +[submodule "vendor/asynctest"] + path = vendor/asynctest + url = https://github.com/status-im/asynctest.git + ignore = untracked + branch = master +[submodule "vendor/libp2p"] + ignore = untracked + branch = master +[submodule "vendor/nim-libp2p"] + path = vendor/nim-libp2p + url = https://github.com/status-im/nim-libp2p.git + ignore = untracked + branch = master +[submodule "vendor/nimcrypto"] + path = vendor/nimcrypto + url = https://github.com/status-im/nimcrypto.git + ignore = untracked + branch = master +[submodule "vendor/nim-stint"] + path = vendor/nim-stint + url = https://github.com/status-im/nim-stint.git + ignore = untracked + branch = master +[submodule "vendor/nim-secp256k1"] + path = vendor/nim-secp256k1 + url = https://github.com/status-im/nim-secp256k1.git + ignore = untracked + branch = master +[submodule "vendor/nim-stew"] + path = vendor/nim-stew + url = https://github.com/status-im/nim-stew.git + ignore = untracked + branch = master +[submodule "vendor/nim-datastore"] + path = vendor/nim-datastore + url = https://github.com/codex-storage/nim-datastore.git + ignore = untracked + branch = master +[submodule "vendor/nim-nat-traversal"] + path = vendor/nim-nat-traversal + url = https://github.com/status-im/nim-nat-traversal.git + ignore = untracked + branch = master +[submodule "vendor/nim-metrics"] + path = vendor/nim-metrics + url = https://github.com/status-im/nim-metrics.git + ignore = untracked + branch = master +[submodule "vendor/nim-bearssl"] + path = vendor/nim-bearssl + url = https://github.com/status-im/nim-bearssl.git + ignore = untracked + branch = master +[submodule "vendor/nim-chronicles"] + path = vendor/nim-chronicles + url = https://github.com/status-im/nim-chronicles.git + ignore = untracked + branch = master +[submodule "vendor/nim-chronos"] + path = vendor/nim-chronos + url = https://github.com/status-im/nim-chronos.git + ignore = untracked + branch = master +[submodule "vendor/nim-faststream"] + ignore = untracked + branch = master +[submodule "vendor/nim-faststreams"] + path = vendor/nim-faststreams + url = https://github.com/status-im/nim-faststreams.git + ignore = untracked + branch = master +[submodule "vendor/nim-json-serialization"] + path = vendor/nim-json-serialization + url = https://github.com/status-im/nim-json-serialization.git + ignore = untracked + branch = master +[submodule "vendor/nim-serialization"] + path = vendor/nim-serialization + url = https://github.com/status-im/nim-serialization.git + ignore = untracked + branch = master +[submodule "vendor/nim-websock"] + path = vendor/nim-websock + url = https://github.com/status-im/nim-websock.git + ignore = untracked + branch = master +[submodule "vendor/nim-httputils"] + ignore = untracked + branch = master +[submodule "vendor/nimhttputils"] + ignore = untracked + branch = master +[submodule "vendor/nim-http-utils"] + path = vendor/nim-http-utils + url = https://github.com/status-im/nim-http-utils.git + ignore = untracked + branch = master +[submodule "vendor/questionable"] + path = vendor/questionable + url = https://github.com/status-im/questionable.git + ignore = untracked + branch = master +[submodule "vendor/upraises"] + ignore = untracked + branch = master + path = vendor/upraises + url = https://github.com/markspanbroek/upraises.git +[submodule "vendor/nim-sqlite3-abi"] + path = vendor/nim-sqlite3-abi + url = https://github.com/arnetheduck/nim-sqlite3-abi.git + ignore = untracked + branch = master +[submodule "vendor/unittest2"] + ignore = untracked + branch = master +[submodule "vendor/nim-unittest2"] + path = vendor/nim-unittest2 + url = https://github.com/status-im/nim-unittest2.git + ignore = untracked + branch = master diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..4cb31ba --- /dev/null +++ b/Makefile @@ -0,0 +1,75 @@ +# Copyright (c) 2020 Status Research & Development GmbH. Licensed under +# either of: +# - Apache License, version 2.0 +# - MIT license +# at your option. This file may not be copied, modified, or distributed except +# according to those terms. + +SHELL := bash # the shell used internally by Make + +# used inside the included makefiles +BUILD_SYSTEM_DIR := vendor/nimbus-build-system + +# -d:insecure - Necessary to enable Prometheus HTTP endpoint for metrics +# -d:chronicles_colors:none - Necessary to disable colors in logs for Docker +DOCKER_IMAGE_NIM_PARAMS ?= -d:chronicles_colors:none -d:insecure + +LINK_PCRE := 0 + +# we don't want an error here, so we can handle things later, in the ".DEFAULT" target +-include $(BUILD_SYSTEM_DIR)/makefiles/variables.mk + +.PHONY: \ + all \ + clean \ + coverage \ + deps \ + libbacktrace \ + test \ + update + +ifeq ($(NIM_PARAMS),) +# "variables.mk" was not included, so we update the submodules. +GIT_SUBMODULE_UPDATE := git submodule update --init --recursive +.DEFAULT: + +@ echo -e "Git submodules not found. Running '$(GIT_SUBMODULE_UPDATE)'.\n"; \ + $(GIT_SUBMODULE_UPDATE); \ + echo +# Now that the included *.mk files appeared, and are newer than this file, Make will restart itself: +# https://www.gnu.org/software/make/manual/make.html#Remaking-Makefiles +# +# After restarting, it will execute its original goal, so we don't have to start a child Make here +# with "$(MAKE) $(MAKECMDGOALS)". Isn't hidden control flow great? + +else # "variables.mk" was included. Business as usual until the end of this file. + +# default target, because it's the first one that doesn't start with '.' + +# Builds the codex binary +all: | build deps + echo -e $(BUILD_MSG) "$@" && \ + $(ENV_SCRIPT) nim codex $(NIM_PARAMS) codexdht.nims + +# must be included after the default target +-include $(BUILD_SYSTEM_DIR)/makefiles/targets.mk + +deps: | deps-common nat-libs codexdht.nims + +#- deletes and recreates "codexdht.nims" which on Windows is a copy instead of a proper symlink +update: | update-common codexdht.nims + rm -rf codexdht.nims && \ + $(MAKE) codexdht.nims $(HANDLE_OUTPUT) + +# Builds and run a part of the test suite +test: | build deps + echo -e $(BUILD_MSG) "$@" && \ + $(ENV_SCRIPT) nim testAll $(NIM_PARAMS) codexdht.nims + +# symlink +codexdht.nims: + ln -s codexdht.nimble $@ + +# usual cleaning +clean: | clean-common + +endif # "variables.mk" was not included diff --git a/codexdht.nimble b/codexdht.nimble index 147c78d..378f632 100644 --- a/codexdht.nimble +++ b/codexdht.nimble @@ -23,40 +23,43 @@ requires "nim >= 1.2.0", "https://github.com/status-im/nim-datastore#head", "questionable" -task coverage, "generates code coverage report": - var (output, exitCode) = gorgeEx("which lcov") - if exitCode != 0: - echo "" - echo " ************************** ⛔️ ERROR ⛔️ **************************" - echo " ** **" - echo " ** ERROR: lcov not found, it must be installed to run code **" - echo " ** coverage locally **" - echo " ** **" - echo " *****************************************************************" - echo "" - quit 1 +task testAll, "Run DHT tests": + exec "nim c -r tests/testAll.nim" - (output, exitCode) = gorgeEx("gcov --version") - if output.contains("Apple LLVM"): - echo "" - echo " ************************* ⚠️ WARNING ⚠️ *************************" - echo " ** **" - echo " ** WARNING: Using Apple's llvm-cov in place of gcov, which **" - echo " ** emulates an old version of gcov (4.2.0) and therefore **" - echo " ** coverage results will differ than those on CI (which **" - echo " ** uses a much newer version of gcov). **" - echo " ** **" - echo " *****************************************************************" - echo "" +# task coverage, "generates code coverage report": +# var (output, exitCode) = gorgeEx("which lcov") +# if exitCode != 0: +# echo "" +# echo " ************************** ⛔️ ERROR ⛔️ **************************" +# echo " ** **" +# echo " ** ERROR: lcov not found, it must be installed to run code **" +# echo " ** coverage locally **" +# echo " ** **" +# echo " *****************************************************************" +# echo "" +# quit 1 - exec("nimble --verbose test --opt:speed -d:debug --verbosity:0 --hints:off --lineDir:on -d:chronicles_log_level=INFO --nimcache:nimcache --passC:-fprofile-arcs --passC:-ftest-coverage --passL:-fprofile-arcs --passL:-ftest-coverage") - exec("cd nimcache; rm *.c; cd ..") - mkDir("coverage") - exec("lcov --capture --directory nimcache --output-file coverage/coverage.info") - exec("$(which bash) -c 'shopt -s globstar; ls $(pwd)/libp2pdht/{*,**/*}.nim'") - exec("$(which bash) -c 'shopt -s globstar; lcov --extract coverage/coverage.info $(pwd)/libp2pdht/{*,**/*}.nim --output-file coverage/coverage.f.info'") - echo "Generating HTML coverage report" - exec("genhtml coverage/coverage.f.info --output-directory coverage/report") - echo "Opening HTML coverage report in browser..." - exec("open coverage/report/index.html") +# (output, exitCode) = gorgeEx("gcov --version") +# if output.contains("Apple LLVM"): +# echo "" +# echo " ************************* ⚠️ WARNING ⚠️ *************************" +# echo " ** **" +# echo " ** WARNING: Using Apple's llvm-cov in place of gcov, which **" +# echo " ** emulates an old version of gcov (4.2.0) and therefore **" +# echo " ** coverage results will differ than those on CI (which **" +# echo " ** uses a much newer version of gcov). **" +# echo " ** **" +# echo " *****************************************************************" +# echo "" + +# exec("nimble --verbose test --opt:speed -d:debug --verbosity:0 --hints:off --lineDir:on -d:chronicles_log_level=INFO --nimcache:nimcache --passC:-fprofile-arcs --passC:-ftest-coverage --passL:-fprofile-arcs --passL:-ftest-coverage") +# exec("cd nimcache; rm *.c; cd ..") +# mkDir("coverage") +# exec("lcov --capture --directory nimcache --output-file coverage/coverage.info") +# exec("$(which bash) -c 'shopt -s globstar; ls $(pwd)/libp2pdht/{*,**/*}.nim'") +# exec("$(which bash) -c 'shopt -s globstar; lcov --extract coverage/coverage.info $(pwd)/libp2pdht/{*,**/*}.nim --output-file coverage/coverage.f.info'") +# echo "Generating HTML coverage report" +# exec("genhtml coverage/coverage.f.info --output-directory coverage/report") +# echo "Opening HTML coverage report in browser..." +# exec("open coverage/report/index.html") diff --git a/env.sh b/env.sh new file mode 100755 index 0000000..697a426 --- /dev/null +++ b/env.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +# We use ${BASH_SOURCE[0]} instead of $0 to allow sourcing this file +# and we fall back to a Zsh-specific special var to also support Zsh. +REL_PATH="$(dirname ${BASH_SOURCE[0]:-${(%):-%x}})" +ABS_PATH="$(cd ${REL_PATH}; pwd)" +source ${ABS_PATH}/vendor/nimbus-build-system/scripts/env.sh diff --git a/libp2pdht/private/eth/p2p/discoveryv5/protocol.nim b/libp2pdht/private/eth/p2p/discoveryv5/protocol.nim index ef207a9..a801f27 100644 --- a/libp2pdht/private/eth/p2p/discoveryv5/protocol.nim +++ b/libp2pdht/private/eth/p2p/discoveryv5/protocol.nim @@ -1032,6 +1032,76 @@ func init*( bitsPerHop: bitsPerHop ) +proc newProtocol*( + privKey: PrivateKey, + enrIp: Option[ValidIpAddress], + enrTcpPort, enrUdpPort: Option[Port], + localEnrFields: openArray[(string, seq[byte])] = [], + bootstrapRecords: openArray[SignedPeerRecord] = [], + previousRecord = none[SignedPeerRecord](), + bindPort: Port, + bindIp = IPv4_any(), + enrAutoUpdate = false, + config = defaultDiscoveryConfig, + rng = newRng(), + providers = ProvidersManager.new( + SQLiteDatastore.new(Memory) + .expect("Should not fail!"))): + Protocol = + # TODO: Tried adding bindPort = udpPort as parameter but that gave + # "Error: internal error: environment misses: udpPort" in nim-beacon-chain. + # Anyhow, nim-beacon-chain would also require some changes to support port + # remapping through NAT and this API is also subject to change once we + # introduce support for ipv4 + ipv6 binding/listening. + + # TODO: Implement SignedPeerRecord custom fields? + # let extraFields = mapIt(localEnrFields, toFieldPair(it[0], it[1])) + + # TODO: + # - Defect as is now or return a result for spr errors? + # - In case incorrect key, allow for new spr based on new key (new node id)? + var record: SignedPeerRecord + if previousRecord.isSome(): + record = previousRecord.get() + record.update(privKey, enrIp, enrTcpPort, enrUdpPort) + .expect("SignedPeerRecord within size limits and correct key") + else: + record = SignedPeerRecord.init(1, privKey, enrIp, enrTcpPort, enrUdpPort) + .expect("SignedPeerRecord within size limits") + + info "SPR initialized", ip = enrIp, tcp = enrTcpPort, udp = enrUdpPort, + seqNum = record.seqNum, uri = toURI(record) + if enrIp.isNone(): + if enrAutoUpdate: + notice "No external IP provided for the SPR, this node will not be " & + "discoverable until the SPR is updated with the discovered external IP address" + else: + warn "No external IP provided for the SPR, this node will not be discoverable" + + let node = newNode(record).expect("Properly initialized record") + + # TODO Consider whether this should be a Defect + doAssert rng != nil, "RNG initialization failed" + + let + routingTable = RoutingTable.init( + node, + config.bitsPerHop, + config.tableIpLimits, + rng) + + result = Protocol( + privateKey: privKey, + localNode: node, + bootstrapRecords: @bootstrapRecords, + ipVote: IpVote.init(), + enrAutoUpdate: enrAutoUpdate, + routingTable: routingTable, + rng: rng, + providers: providers) + + result.transport = newTransport(result, privKey, node, bindPort, bindIp, rng) + proc newProtocol*( privKey: PrivateKey, bindPort: Port, diff --git a/vendor/asynctest b/vendor/asynctest new file mode 160000 index 0000000..a236a5f --- /dev/null +++ b/vendor/asynctest @@ -0,0 +1 @@ +Subproject commit a236a5f0f3031573ac2cb082b63dbf6e170e06e7 diff --git a/vendor/nim-bearssl b/vendor/nim-bearssl new file mode 160000 index 0000000..f4c4233 --- /dev/null +++ b/vendor/nim-bearssl @@ -0,0 +1 @@ +Subproject commit f4c4233de453cb7eac0ce3f3ffad6496295f83ab diff --git a/vendor/nim-chronicles b/vendor/nim-chronicles new file mode 160000 index 0000000..7631f7b --- /dev/null +++ b/vendor/nim-chronicles @@ -0,0 +1 @@ +Subproject commit 7631f7b2ee03398cb1512a79923264e8f9410af6 diff --git a/vendor/nim-chronos b/vendor/nim-chronos new file mode 160000 index 0000000..6525f4c --- /dev/null +++ b/vendor/nim-chronos @@ -0,0 +1 @@ +Subproject commit 6525f4ce1d1a7eba146e5f1a53f6f105077ae686 diff --git a/vendor/nim-datastore b/vendor/nim-datastore new file mode 160000 index 0000000..0cde8ae --- /dev/null +++ b/vendor/nim-datastore @@ -0,0 +1 @@ +Subproject commit 0cde8aeb67c59fd0ac95496dc6b5e1168d6632aa diff --git a/vendor/nim-faststreams b/vendor/nim-faststreams new file mode 160000 index 0000000..1b561a9 --- /dev/null +++ b/vendor/nim-faststreams @@ -0,0 +1 @@ +Subproject commit 1b561a9e71b6bdad1c1cdff753418906037e9d09 diff --git a/vendor/nim-http-utils b/vendor/nim-http-utils new file mode 160000 index 0000000..3b491a4 --- /dev/null +++ b/vendor/nim-http-utils @@ -0,0 +1 @@ +Subproject commit 3b491a40c60aad9e8d3407443f46f62511e63b18 diff --git a/vendor/nim-json-serialization b/vendor/nim-json-serialization new file mode 160000 index 0000000..e5b18fb --- /dev/null +++ b/vendor/nim-json-serialization @@ -0,0 +1 @@ +Subproject commit e5b18fb710c3d0167ec79f3b892f5a7a1bc6d1a4 diff --git a/vendor/nim-libp2p b/vendor/nim-libp2p new file mode 160000 index 0000000..a3e9d1e --- /dev/null +++ b/vendor/nim-libp2p @@ -0,0 +1 @@ +Subproject commit a3e9d1ed80c048cd5abc839cbe0863cefcedc702 diff --git a/vendor/nim-metrics b/vendor/nim-metrics new file mode 160000 index 0000000..743f81d --- /dev/null +++ b/vendor/nim-metrics @@ -0,0 +1 @@ +Subproject commit 743f81d4f6c6ebf0ac02389f2392ff8b4235bee5 diff --git a/vendor/nim-nat-traversal b/vendor/nim-nat-traversal new file mode 160000 index 0000000..27d314d --- /dev/null +++ b/vendor/nim-nat-traversal @@ -0,0 +1 @@ +Subproject commit 27d314d65c9078924b3239fe4e2f5af0c512b28c diff --git a/vendor/nim-secp256k1 b/vendor/nim-secp256k1 new file mode 160000 index 0000000..5340cf1 --- /dev/null +++ b/vendor/nim-secp256k1 @@ -0,0 +1 @@ +Subproject commit 5340cf188168d6afcafc8023770d880f067c0b2f diff --git a/vendor/nim-serialization b/vendor/nim-serialization new file mode 160000 index 0000000..493d18b --- /dev/null +++ b/vendor/nim-serialization @@ -0,0 +1 @@ +Subproject commit 493d18b8292fc03aa4f835fd825dea1183f97466 diff --git a/vendor/nim-sqlite3-abi b/vendor/nim-sqlite3-abi new file mode 160000 index 0000000..362e1bd --- /dev/null +++ b/vendor/nim-sqlite3-abi @@ -0,0 +1 @@ +Subproject commit 362e1bd9f689ad9f5380d9d27f0705b3d4dfc7d3 diff --git a/vendor/nim-stew b/vendor/nim-stew new file mode 160000 index 0000000..e18f5a6 --- /dev/null +++ b/vendor/nim-stew @@ -0,0 +1 @@ +Subproject commit e18f5a62af2ade7a1fd1d39635d4e04d944def08 diff --git a/vendor/nim-stint b/vendor/nim-stint new file mode 160000 index 0000000..036c71d --- /dev/null +++ b/vendor/nim-stint @@ -0,0 +1 @@ +Subproject commit 036c71d06a6b22f8f967ba9d54afd2189c3872ca diff --git a/vendor/nim-unittest2 b/vendor/nim-unittest2 new file mode 160000 index 0000000..b178f47 --- /dev/null +++ b/vendor/nim-unittest2 @@ -0,0 +1 @@ +Subproject commit b178f47527074964f76c395ad0dfc81cf118f379 diff --git a/vendor/nim-websock b/vendor/nim-websock new file mode 160000 index 0000000..2c3ae31 --- /dev/null +++ b/vendor/nim-websock @@ -0,0 +1 @@ +Subproject commit 2c3ae3137f3c9cb48134285bd4a47186fa51f0e8 diff --git a/vendor/nimcrypto b/vendor/nimcrypto new file mode 160000 index 0000000..a5742a9 --- /dev/null +++ b/vendor/nimcrypto @@ -0,0 +1 @@ +Subproject commit a5742a9a214ac33f91615f3862c7b099aec43b00 diff --git a/vendor/questionable b/vendor/questionable new file mode 160000 index 0000000..0d7ce8e --- /dev/null +++ b/vendor/questionable @@ -0,0 +1 @@ +Subproject commit 0d7ce8efdedaf184680cb7268721fca0af947a74 diff --git a/vendor/upraises b/vendor/upraises new file mode 160000 index 0000000..bc26289 --- /dev/null +++ b/vendor/upraises @@ -0,0 +1 @@ +Subproject commit bc2628989b63854d980e92dadbd58f83e34b6f25