adding dependencies

.github/workflows/ci.yml

@@ -156,12 +156,4 @@ jobs:
             # https://github.com/status-im/nimbus-eth2/issues/3121
             export NIMFLAGS="-d:nimRawSetjmp"
           fi
-          nim --version
-          nimble --version
-          nimble install -y --depsOnly
-          nimble test
-          if [[ "${{ matrix.branch }}" == "version-1-6" || "${{ matrix.branch }}" == "devel" ]]; then
-            echo -e "\nTesting with '--gc:orc':\n"
-            export NIMFLAGS="${NIMFLAGS} --gc:orc"
-            nimble test
-          fi;
+          make test

.gitmodules

@@ -1,3 +1,125 @@
 [submodule "vendor/nimbus-build-system"]
 	path = vendor/nimbus-build-system
 	url = https://github.com/status-im/nimbus-build-system.git
+	ignore = untracked
+	branch = master
+[submodule "vendor/asynctest"]
+	path = vendor/asynctest
+	url = https://github.com/status-im/asynctest.git
+	ignore = untracked
+	branch = master
+[submodule "vendor/libp2p"]
+	ignore = untracked
+	branch = master
+[submodule "vendor/nim-libp2p"]
+	path = vendor/nim-libp2p
+	url = https://github.com/status-im/nim-libp2p.git
+	ignore = untracked
+	branch = master
+[submodule "vendor/nimcrypto"]
+	path = vendor/nimcrypto
+	url = https://github.com/status-im/nimcrypto.git
+	ignore = untracked
+	branch = master
+[submodule "vendor/nim-stint"]
+	path = vendor/nim-stint
+	url = https://github.com/status-im/nim-stint.git
+	ignore = untracked
+	branch = master
+[submodule "vendor/nim-secp256k1"]
+	path = vendor/nim-secp256k1
+	url = https://github.com/status-im/nim-secp256k1.git
+	ignore = untracked
+	branch = master
+[submodule "vendor/nim-stew"]
+	path = vendor/nim-stew
+	url = https://github.com/status-im/nim-stew.git
+	ignore = untracked
+	branch = master
+[submodule "vendor/nim-datastore"]
+	path = vendor/nim-datastore
+	url = https://github.com/codex-storage/nim-datastore.git
+	ignore = untracked
+	branch = master
+[submodule "vendor/nim-nat-traversal"]
+	path = vendor/nim-nat-traversal
+	url = https://github.com/status-im/nim-nat-traversal.git
+	ignore = untracked
+	branch = master
+[submodule "vendor/nim-metrics"]
+	path = vendor/nim-metrics
+	url = https://github.com/status-im/nim-metrics.git
+	ignore = untracked
+	branch = master
+[submodule "vendor/nim-bearssl"]
+	path = vendor/nim-bearssl
+	url = https://github.com/status-im/nim-bearssl.git
+	ignore = untracked
+	branch = master
+[submodule "vendor/nim-chronicles"]
+	path = vendor/nim-chronicles
+	url = https://github.com/status-im/nim-chronicles.git
+	ignore = untracked
+	branch = master
+[submodule "vendor/nim-chronos"]
+	path = vendor/nim-chronos
+	url = https://github.com/status-im/nim-chronos.git
+	ignore = untracked
+	branch = master
+[submodule "vendor/nim-faststream"]
+	ignore = untracked
+	branch = master
+[submodule "vendor/nim-faststreams"]
+	path = vendor/nim-faststreams
+	url = https://github.com/status-im/nim-faststreams.git
+	ignore = untracked
+	branch = master
+[submodule "vendor/nim-json-serialization"]
+	path = vendor/nim-json-serialization
+	url = https://github.com/status-im/nim-json-serialization.git
+	ignore = untracked
+	branch = master
+[submodule "vendor/nim-serialization"]
+	path = vendor/nim-serialization
+	url = https://github.com/status-im/nim-serialization.git
+	ignore = untracked
+	branch = master
+[submodule "vendor/nim-websock"]
+	path = vendor/nim-websock
+	url = https://github.com/status-im/nim-websock.git
+	ignore = untracked
+	branch = master
+[submodule "vendor/nim-httputils"]
+	ignore = untracked
+	branch = master
+[submodule "vendor/nimhttputils"]
+	ignore = untracked
+	branch = master
+[submodule "vendor/nim-http-utils"]
+	path = vendor/nim-http-utils
+	url = https://github.com/status-im/nim-http-utils.git
+	ignore = untracked
+	branch = master
+[submodule "vendor/questionable"]
+	path = vendor/questionable
+	url = https://github.com/status-im/questionable.git
+	ignore = untracked
+	branch = master
+[submodule "vendor/upraises"]
+	ignore = untracked
+	branch = master
+	path = vendor/upraises
+	url = https://github.com/markspanbroek/upraises.git
+[submodule "vendor/nim-sqlite3-abi"]
+	path = vendor/nim-sqlite3-abi
+	url = https://github.com/arnetheduck/nim-sqlite3-abi.git
+	ignore = untracked
+	branch = master
+[submodule "vendor/unittest2"]
+	ignore = untracked
+	branch = master
+[submodule "vendor/nim-unittest2"]
+	path = vendor/nim-unittest2
+	url = https://github.com/status-im/nim-unittest2.git
+	ignore = untracked
+	branch = master

Makefile

@@ -0,0 +1,75 @@
+# Copyright (c) 2020 Status Research & Development GmbH. Licensed under
+# either of:
+# - Apache License, version 2.0
+# - MIT license
+# at your option. This file may not be copied, modified, or distributed except
+# according to those terms.
+
+SHELL := bash # the shell used internally by Make
+
+# used inside the included makefiles
+BUILD_SYSTEM_DIR := vendor/nimbus-build-system
+
+# -d:insecure - Necessary to enable Prometheus HTTP endpoint for metrics
+# -d:chronicles_colors:none - Necessary to disable colors in logs for Docker
+DOCKER_IMAGE_NIM_PARAMS ?= -d:chronicles_colors:none -d:insecure
+
+LINK_PCRE := 0
+
+# we don't want an error here, so we can handle things later, in the ".DEFAULT" target
+-include $(BUILD_SYSTEM_DIR)/makefiles/variables.mk
+
+.PHONY: \
+	all \
+	clean \
+	coverage \
+	deps \
+	libbacktrace \
+	test \
+	update
+
+ifeq ($(NIM_PARAMS),)
+# "variables.mk" was not included, so we update the submodules.
+GIT_SUBMODULE_UPDATE := git submodule update --init --recursive
+.DEFAULT:
+	+@ echo -e "Git submodules not found. Running '$(GIT_SUBMODULE_UPDATE)'.\n"; \
+		$(GIT_SUBMODULE_UPDATE); \
+		echo
+# Now that the included *.mk files appeared, and are newer than this file, Make will restart itself:
+# https://www.gnu.org/software/make/manual/make.html#Remaking-Makefiles
+#
+# After restarting, it will execute its original goal, so we don't have to start a child Make here
+# with "$(MAKE) $(MAKECMDGOALS)". Isn't hidden control flow great?
+
+else # "variables.mk" was included. Business as usual until the end of this file.
+
+# default target, because it's the first one that doesn't start with '.'
+
+# Builds the codex binary
+all: | build deps
+	echo -e $(BUILD_MSG) "$@" && \
+		$(ENV_SCRIPT) nim codex $(NIM_PARAMS) codexdht.nims
+
+# must be included after the default target
+-include $(BUILD_SYSTEM_DIR)/makefiles/targets.mk
+
+deps: | deps-common nat-libs codexdht.nims
+
+#- deletes and recreates "codexdht.nims" which on Windows is a copy instead of a proper symlink
+update: | update-common codexdht.nims
+	rm -rf codexdht.nims && \
+		$(MAKE) codexdht.nims $(HANDLE_OUTPUT)
+
+# Builds and run a part of the test suite
+test: | build deps
+	echo -e $(BUILD_MSG) "$@" && \
+		$(ENV_SCRIPT) nim testAll $(NIM_PARAMS) codexdht.nims
+
+# symlink
+codexdht.nims:
+	ln -s codexdht.nimble $@
+
+# usual cleaning
+clean: | clean-common
+
+endif # "variables.mk" was not included

codexdht.nimble

@@ -23,40 +23,43 @@ requires "nim >= 1.2.0",
          "https://github.com/status-im/nim-datastore#head",
          "questionable"
 
-task coverage, "generates code coverage report":
-  var (output, exitCode) = gorgeEx("which lcov")
-  if exitCode != 0:
-    echo ""
-    echo "  ************************** ⛔️ ERROR ⛔️ **************************"
-    echo "  **                                                             **"
-    echo "  **   ERROR: lcov not found, it must be installed to run code   **"
-    echo "  **   coverage locally                                          **"
-    echo "  **                                                             **"
-    echo "  *****************************************************************"
-    echo ""
-    quit 1
+task testAll, "Run DHT tests":
+  exec "nim c -r tests/testAll.nim"
 
-  (output, exitCode) = gorgeEx("gcov --version")
-  if output.contains("Apple LLVM"):
-    echo ""
-    echo "  ************************* ⚠️  WARNING ⚠️  *************************"
-    echo "  **                                                             **"
-    echo "  **   WARNING: Using Apple's llvm-cov in place of gcov, which   **"
-    echo "  **   emulates an old version of gcov (4.2.0) and therefore     **"
-    echo "  **   coverage results will differ than those on CI (which      **"
-    echo "  **   uses a much newer version of gcov).                       **"
-    echo "  **                                                             **"
-    echo "  *****************************************************************"
-    echo ""
+# task coverage, "generates code coverage report":
+#   var (output, exitCode) = gorgeEx("which lcov")
+#   if exitCode != 0:
+#     echo ""
+#     echo "  ************************** ⛔️ ERROR ⛔️ **************************"
+#     echo "  **                                                             **"
+#     echo "  **   ERROR: lcov not found, it must be installed to run code   **"
+#     echo "  **   coverage locally                                          **"
+#     echo "  **                                                             **"
+#     echo "  *****************************************************************"
+#     echo ""
+#     quit 1
 
-  exec("nimble --verbose test --opt:speed -d:debug --verbosity:0 --hints:off --lineDir:on -d:chronicles_log_level=INFO --nimcache:nimcache --passC:-fprofile-arcs --passC:-ftest-coverage --passL:-fprofile-arcs --passL:-ftest-coverage")
-  exec("cd nimcache; rm *.c; cd ..")
-  mkDir("coverage")
-  exec("lcov --capture --directory nimcache --output-file coverage/coverage.info")
-  exec("$(which bash) -c 'shopt -s globstar; ls $(pwd)/libp2pdht/{*,**/*}.nim'")
-  exec("$(which bash) -c 'shopt -s globstar; lcov --extract coverage/coverage.info  $(pwd)/libp2pdht/{*,**/*}.nim --output-file coverage/coverage.f.info'")
-  echo "Generating HTML coverage report"
-  exec("genhtml coverage/coverage.f.info --output-directory coverage/report")
-  echo "Opening HTML coverage report in browser..."
-  exec("open coverage/report/index.html")
+#   (output, exitCode) = gorgeEx("gcov --version")
+#   if output.contains("Apple LLVM"):
+#     echo ""
+#     echo "  ************************* ⚠️  WARNING ⚠️  *************************"
+#     echo "  **                                                             **"
+#     echo "  **   WARNING: Using Apple's llvm-cov in place of gcov, which   **"
+#     echo "  **   emulates an old version of gcov (4.2.0) and therefore     **"
+#     echo "  **   coverage results will differ than those on CI (which      **"
+#     echo "  **   uses a much newer version of gcov).                       **"
+#     echo "  **                                                             **"
+#     echo "  *****************************************************************"
+#     echo ""
+
+#   exec("nimble --verbose test --opt:speed -d:debug --verbosity:0 --hints:off --lineDir:on -d:chronicles_log_level=INFO --nimcache:nimcache --passC:-fprofile-arcs --passC:-ftest-coverage --passL:-fprofile-arcs --passL:-ftest-coverage")
+#   exec("cd nimcache; rm *.c; cd ..")
+#   mkDir("coverage")
+#   exec("lcov --capture --directory nimcache --output-file coverage/coverage.info")
+#   exec("$(which bash) -c 'shopt -s globstar; ls $(pwd)/libp2pdht/{*,**/*}.nim'")
+#   exec("$(which bash) -c 'shopt -s globstar; lcov --extract coverage/coverage.info  $(pwd)/libp2pdht/{*,**/*}.nim --output-file coverage/coverage.f.info'")
+#   echo "Generating HTML coverage report"
+#   exec("genhtml coverage/coverage.f.info --output-directory coverage/report")
+#   echo "Opening HTML coverage report in browser..."
+#   exec("open coverage/report/index.html")
 

env.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+# We use ${BASH_SOURCE[0]} instead of $0 to allow sourcing this file
+# and we fall back to a Zsh-specific special var to also support Zsh.
+REL_PATH="$(dirname ${BASH_SOURCE[0]:-${(%):-%x}})"
+ABS_PATH="$(cd ${REL_PATH}; pwd)"
+source ${ABS_PATH}/vendor/nimbus-build-system/scripts/env.sh

libp2pdht/private/eth/p2p/discoveryv5/protocol.nim

@@ -1032,6 +1032,76 @@ func init*(
     bitsPerHop: bitsPerHop
   )
 
+proc newProtocol*(
+    privKey: PrivateKey,
+    enrIp: Option[ValidIpAddress],
+    enrTcpPort, enrUdpPort: Option[Port],
+    localEnrFields: openArray[(string, seq[byte])] = [],
+    bootstrapRecords: openArray[SignedPeerRecord] = [],
+    previousRecord = none[SignedPeerRecord](),
+    bindPort: Port,
+    bindIp = IPv4_any(),
+    enrAutoUpdate = false,
+    config = defaultDiscoveryConfig,
+    rng = newRng(),
+    providers = ProvidersManager.new(
+      SQLiteDatastore.new(Memory)
+      .expect("Should not fail!"))):
+    Protocol =
+  # TODO: Tried adding bindPort = udpPort as parameter but that gave
+  # "Error: internal error: environment misses: udpPort" in nim-beacon-chain.
+  # Anyhow, nim-beacon-chain would also require some changes to support port
+  # remapping through NAT and this API is also subject to change once we
+  # introduce support for ipv4 + ipv6 binding/listening.
+
+  # TODO: Implement SignedPeerRecord custom fields?
+  # let extraFields = mapIt(localEnrFields, toFieldPair(it[0], it[1]))
+
+  # TODO:
+  # - Defect as is now or return a result for spr errors?
+  # - In case incorrect key, allow for new spr based on new key (new node id)?
+  var record: SignedPeerRecord
+  if previousRecord.isSome():
+    record = previousRecord.get()
+    record.update(privKey, enrIp, enrTcpPort, enrUdpPort)
+            .expect("SignedPeerRecord within size limits and correct key")
+  else:
+    record = SignedPeerRecord.init(1, privKey, enrIp, enrTcpPort, enrUdpPort)
+               .expect("SignedPeerRecord within size limits")
+
+  info "SPR initialized", ip = enrIp, tcp = enrTcpPort, udp = enrUdpPort,
+    seqNum = record.seqNum, uri = toURI(record)
+  if enrIp.isNone():
+    if enrAutoUpdate:
+      notice "No external IP provided for the SPR, this node will not be " &
+        "discoverable until the SPR is updated with the discovered external IP address"
+    else:
+      warn "No external IP provided for the SPR, this node will not be discoverable"
+
+  let node = newNode(record).expect("Properly initialized record")
+
+  # TODO Consider whether this should be a Defect
+  doAssert rng != nil, "RNG initialization failed"
+
+  let
+    routingTable = RoutingTable.init(
+      node,
+      config.bitsPerHop,
+      config.tableIpLimits,
+      rng)
+
+  result = Protocol(
+    privateKey: privKey,
+    localNode: node,
+    bootstrapRecords: @bootstrapRecords,
+    ipVote: IpVote.init(),
+    enrAutoUpdate: enrAutoUpdate,
+    routingTable: routingTable,
+    rng: rng,
+    providers: providers)
+
+  result.transport = newTransport(result, privKey, node, bindPort, bindIp, rng)
+
 proc newProtocol*(
     privKey: PrivateKey,
     bindPort: Port,

vendor/asynctest

@@ -0,0 +1 @@
+Subproject commit a236a5f0f3031573ac2cb082b63dbf6e170e06e7

vendor/nim-bearssl

@@ -0,0 +1 @@
+Subproject commit f4c4233de453cb7eac0ce3f3ffad6496295f83ab

vendor/nim-chronicles

@@ -0,0 +1 @@
+Subproject commit 7631f7b2ee03398cb1512a79923264e8f9410af6

vendor/nim-chronos

@@ -0,0 +1 @@
+Subproject commit 6525f4ce1d1a7eba146e5f1a53f6f105077ae686

vendor/nim-datastore

@@ -0,0 +1 @@
+Subproject commit 0cde8aeb67c59fd0ac95496dc6b5e1168d6632aa

vendor/nim-faststreams

@@ -0,0 +1 @@
+Subproject commit 1b561a9e71b6bdad1c1cdff753418906037e9d09

vendor/nim-http-utils

@@ -0,0 +1 @@
+Subproject commit 3b491a40c60aad9e8d3407443f46f62511e63b18

vendor/nim-json-serialization

@@ -0,0 +1 @@
+Subproject commit e5b18fb710c3d0167ec79f3b892f5a7a1bc6d1a4

vendor/nim-libp2p

@@ -0,0 +1 @@
+Subproject commit a3e9d1ed80c048cd5abc839cbe0863cefcedc702

vendor/nim-metrics

@@ -0,0 +1 @@
+Subproject commit 743f81d4f6c6ebf0ac02389f2392ff8b4235bee5

vendor/nim-nat-traversal

@@ -0,0 +1 @@
+Subproject commit 27d314d65c9078924b3239fe4e2f5af0c512b28c

vendor/nim-secp256k1

@@ -0,0 +1 @@
+Subproject commit 5340cf188168d6afcafc8023770d880f067c0b2f

vendor/nim-serialization

@@ -0,0 +1 @@
+Subproject commit 493d18b8292fc03aa4f835fd825dea1183f97466

vendor/nim-sqlite3-abi

@@ -0,0 +1 @@
+Subproject commit 362e1bd9f689ad9f5380d9d27f0705b3d4dfc7d3

vendor/nim-stew

@@ -0,0 +1 @@
+Subproject commit e18f5a62af2ade7a1fd1d39635d4e04d944def08

vendor/nim-stint

@@ -0,0 +1 @@
+Subproject commit 036c71d06a6b22f8f967ba9d54afd2189c3872ca

vendor/nim-unittest2

@@ -0,0 +1 @@
+Subproject commit b178f47527074964f76c395ad0dfc81cf118f379

vendor/nim-websock

@@ -0,0 +1 @@
+Subproject commit 2c3ae3137f3c9cb48134285bd4a47186fa51f0e8

vendor/nimcrypto

@@ -0,0 +1 @@
+Subproject commit a5742a9a214ac33f91615f3862c7b099aec43b00

vendor/questionable

@@ -0,0 +1 @@
+Subproject commit 0d7ce8efdedaf184680cb7268721fca0af947a74

vendor/upraises

@@ -0,0 +1 @@
+Subproject commit bc2628989b63854d980e92dadbd58f83e34b6f25