From b6a227f189cf6b3b06e9ce8e5a8a5a22aa63c06f Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 11 Jul 2023 15:19:45 -0700 Subject: [PATCH 01/30] import nimbus build tools --- .github/workflows/ci-nimbus.yml | 165 ++++++++++++++++++++++++++++++++ .gitignore | 3 + Makefile | 75 +++++++++++++++ atlas.lock | 139 +++++++++++++++++++++++++++ libp2pdht.nim => codexdht.nim | 0 codexdht.nimble | 65 +++++++++++++ config.nims | 26 ++++- env.sh | 7 ++ libp2pdht.nimble | 63 ------------ vendor/atlas.workspace | 2 + 10 files changed, 478 insertions(+), 67 deletions(-) create mode 100644 .github/workflows/ci-nimbus.yml create mode 100644 Makefile create mode 100644 atlas.lock rename libp2pdht.nim => codexdht.nim (100%) create mode 100644 codexdht.nimble create mode 100755 env.sh delete mode 100644 libp2pdht.nimble create mode 100644 vendor/atlas.workspace diff --git a/.github/workflows/ci-nimbus.yml b/.github/workflows/ci-nimbus.yml new file mode 100644 index 0000000..06db9d4 --- /dev/null +++ b/.github/workflows/ci-nimbus.yml @@ -0,0 +1,165 @@ +name: CI +on: + push: + # branches: + # - main + pull_request: + workflow_dispatch: + +jobs: + build: + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + target: + - os: linux + cpu: amd64 + # - os: linux + # cpu: i386 + - os: macos + cpu: amd64 + - os: windows + cpu: amd64 + #- os: windows + #cpu: i386 + branch: [version-1-6] + include: + - target: + os: linux + builder: ubuntu-20.04 + shell: bash + - target: + os: macos + builder: macos-10.15 + shell: bash + - target: + os: windows + builder: windows-2019 + shell: msys2 {0} + + defaults: + run: + shell: ${{ matrix.shell }} + + name: '${{ matrix.target.os }}-${{ matrix.target.cpu }} (Nim ${{ matrix.branch }})' + runs-on: ${{ matrix.builder }} + continue-on-error: ${{ matrix.branch == 'version-1-6' || matrix.branch == 'devel' }} + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + submodules: true + + - name: Install build dependencies (Linux i386) + if: runner.os == 'Linux' && matrix.target.cpu == 'i386' + run: | + sudo dpkg --add-architecture i386 + sudo apt-get update -qq + sudo DEBIAN_FRONTEND='noninteractive' apt-get install \ + --no-install-recommends -yq gcc-multilib g++-multilib \ + libssl-dev:i386 + mkdir -p external/bin + cat << EOF > external/bin/gcc + #!/bin/bash + exec $(which gcc) -m32 "\$@" + EOF + cat << EOF > external/bin/g++ + #!/bin/bash + exec $(which g++) -m32 "\$@" + EOF + chmod 755 external/bin/gcc external/bin/g++ + echo '${{ github.workspace }}/external/bin' >> $GITHUB_PATH + + - name: MSYS2 (Windows i386) + if: runner.os == 'Windows' && matrix.target.cpu == 'i386' + uses: msys2/setup-msys2@v2 + with: + path-type: inherit + msystem: MINGW32 + install: >- + base-devel + git + mingw-w64-i686-toolchain + + - name: MSYS2 (Windows amd64) + if: runner.os == 'Windows' && matrix.target.cpu == 'amd64' + uses: msys2/setup-msys2@v2 + with: + path-type: inherit + install: >- + base-devel + git + mingw-w64-x86_64-toolchain + + - name: Restore Nim DLLs dependencies (Windows) from cache + if: runner.os == 'Windows' + id: windows-dlls-cache + uses: actions/cache@v2 + with: + path: external/dlls + key: 'dlls' + + - name: Install DLL dependencies (Windows) + if: > + steps.windows-dlls-cache.outputs.cache-hit != 'true' && + runner.os == 'Windows' + run: | + mkdir external + curl -L "https://nim-lang.org/download/windeps.zip" -o external/windeps.zip + 7z x external/windeps.zip -oexternal/dlls + + - name: Path to cached dependencies (Windows) + if: > + runner.os == 'Windows' + run: | + echo '${{ github.workspace }}'"/external/dlls" >> $GITHUB_PATH + + - name: Derive environment variables + run: | + if [[ '${{ matrix.target.cpu }}' == 'amd64' ]]; then + PLATFORM=x64 + else + PLATFORM=x86 + fi + echo "PLATFORM=$PLATFORM" >> $GITHUB_ENV + + ncpu= + MAKE_CMD="make" + case '${{ runner.os }}' in + 'Linux') + ncpu=$(nproc) + ;; + 'macOS') + ncpu=$(sysctl -n hw.ncpu) + ;; + 'Windows') + ncpu=$NUMBER_OF_PROCESSORS + MAKE_CMD="mingw32-make" + ;; + esac + [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1 + echo "ncpu=$ncpu" >> $GITHUB_ENV + echo "MAKE_CMD=${MAKE_CMD}" >> $GITHUB_ENV + + - name: Restore Nim toolchain binaries from cache + id: nim-cache + uses: actions/cache@v3 + with: + path: NimBinaries + key: ${{ matrix.target.os }}-${{ matrix.target.cpu }}-nim-${{ matrix.branch }}-cache-${{ env.cache_nonce }}-${{ github.run_id }} + restore-keys: ${{ matrix.target.os }}-${{ matrix.target.cpu }}-nim-${{ matrix.branch }}-cache-${{ env.cache_nonce }} + + # - name: Set NIM_COMMIT + # shell: ${{ matrix.shell }} {0} + # run: echo "NIM_COMMIT=${{ matrix.branch }}" >> ${GITHUB_ENV} + + - name: Run tests + run: | + if [[ "${{ matrix.target.os }}" == "windows" ]]; then + # https://github.com/status-im/nimbus-eth2/issues/3121 + export NIMFLAGS="-d:nimRawSetjmp" + fi + export NIM_COMMIT=${{ matrix.branch }} + make -j${ncpu} CI_CACHE=NimBinaries ARCH_OVERRIDE=${PLATFORM} QUICK_AND_DIRTY_COMPILER=1 update + make test -j${ncpu} diff --git a/.gitignore b/.gitignore index e36a278..e5a54f0 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,6 @@ nimcache tests/testAll nimble.develop nimble.paths +nim.cfg +nimbus-build-system.paths +vendor/* diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..4cb31ba --- /dev/null +++ b/Makefile @@ -0,0 +1,75 @@ +# Copyright (c) 2020 Status Research & Development GmbH. Licensed under +# either of: +# - Apache License, version 2.0 +# - MIT license +# at your option. This file may not be copied, modified, or distributed except +# according to those terms. + +SHELL := bash # the shell used internally by Make + +# used inside the included makefiles +BUILD_SYSTEM_DIR := vendor/nimbus-build-system + +# -d:insecure - Necessary to enable Prometheus HTTP endpoint for metrics +# -d:chronicles_colors:none - Necessary to disable colors in logs for Docker +DOCKER_IMAGE_NIM_PARAMS ?= -d:chronicles_colors:none -d:insecure + +LINK_PCRE := 0 + +# we don't want an error here, so we can handle things later, in the ".DEFAULT" target +-include $(BUILD_SYSTEM_DIR)/makefiles/variables.mk + +.PHONY: \ + all \ + clean \ + coverage \ + deps \ + libbacktrace \ + test \ + update + +ifeq ($(NIM_PARAMS),) +# "variables.mk" was not included, so we update the submodules. +GIT_SUBMODULE_UPDATE := git submodule update --init --recursive +.DEFAULT: + +@ echo -e "Git submodules not found. Running '$(GIT_SUBMODULE_UPDATE)'.\n"; \ + $(GIT_SUBMODULE_UPDATE); \ + echo +# Now that the included *.mk files appeared, and are newer than this file, Make will restart itself: +# https://www.gnu.org/software/make/manual/make.html#Remaking-Makefiles +# +# After restarting, it will execute its original goal, so we don't have to start a child Make here +# with "$(MAKE) $(MAKECMDGOALS)". Isn't hidden control flow great? + +else # "variables.mk" was included. Business as usual until the end of this file. + +# default target, because it's the first one that doesn't start with '.' + +# Builds the codex binary +all: | build deps + echo -e $(BUILD_MSG) "$@" && \ + $(ENV_SCRIPT) nim codex $(NIM_PARAMS) codexdht.nims + +# must be included after the default target +-include $(BUILD_SYSTEM_DIR)/makefiles/targets.mk + +deps: | deps-common nat-libs codexdht.nims + +#- deletes and recreates "codexdht.nims" which on Windows is a copy instead of a proper symlink +update: | update-common codexdht.nims + rm -rf codexdht.nims && \ + $(MAKE) codexdht.nims $(HANDLE_OUTPUT) + +# Builds and run a part of the test suite +test: | build deps + echo -e $(BUILD_MSG) "$@" && \ + $(ENV_SCRIPT) nim testAll $(NIM_PARAMS) codexdht.nims + +# symlink +codexdht.nims: + ln -s codexdht.nimble $@ + +# usual cleaning +clean: | clean-common + +endif # "variables.mk" was not included diff --git a/atlas.lock b/atlas.lock new file mode 100644 index 0000000..04169bc --- /dev/null +++ b/atlas.lock @@ -0,0 +1,139 @@ +{ + "items": { + "nim-zlib": { + "dir": "vendor/nim-zlib", + "url": "https://github.com/status-im/nim-zlib", + "commit": "f34ca261efd90f118dc1647beefd2f7a69b05d93" + }, + "nim-stew": { + "dir": "vendor/nim-stew", + "url": "https://github.com/status-im/nim-stew.git", + "commit": "e18f5a62af2ade7a1fd1d39635d4e04d944def08" + }, + "nim-http-utils": { + "dir": "vendor/nim-http-utils", + "url": "https://github.com/status-im/nim-http-utils.git", + "commit": "3b491a40c60aad9e8d3407443f46f62511e63b18" + }, + "nim-chronos": { + "dir": "vendor/nim-chronos", + "url": "https://github.com/status-im/nim-chronos.git", + "commit": "6525f4ce1d1a7eba146e5f1a53f6f105077ae686" + }, + "upraises": { + "dir": "vendor/upraises", + "url": "https://github.com/markspanbroek/upraises.git", + "commit": "bc2628989b63854d980e92dadbd58f83e34b6f25" + }, + "nim-sqlite3-abi": { + "dir": "vendor/nim-sqlite3-abi", + "url": "https://github.com/arnetheduck/nim-sqlite3-abi.git", + "commit": "362e1bd9f689ad9f5380d9d27f0705b3d4dfc7d3" + }, + "questionable": { + "dir": "vendor/questionable", + "url": "https://github.com/status-im/questionable.git", + "commit": "0d7ce8efdedaf184680cb7268721fca0af947a74" + }, + "nim-websock": { + "dir": "vendor/nim-websock", + "url": "https://github.com/status-im/nim-websock.git", + "commit": "2c3ae3137f3c9cb48134285bd4a47186fa51f0e8" + }, + "nim-secp256k1": { + "dir": "vendor/nim-secp256k1", + "url": "https://github.com/status-im/nim-secp256k1.git", + "commit": "5340cf188168d6afcafc8023770d880f067c0b2f" + }, + "nim-bearssl": { + "dir": "vendor/nim-bearssl", + "url": "https://github.com/status-im/nim-bearssl.git", + "commit": "f4c4233de453cb7eac0ce3f3ffad6496295f83ab" + }, + "dnsclient.nim": { + "dir": "vendor/dnsclient.nim", + "url": "https://github.com/ba0f3/dnsclient.nim", + "commit": "23214235d4784d24aceed99bbfe153379ea557c8" + }, + "nimcrypto": { + "dir": "vendor/nimcrypto", + "url": "https://github.com/status-im/nimcrypto.git", + "commit": "a5742a9a214ac33f91615f3862c7b099aec43b00" + }, + "nim-json-serialization": { + "dir": "vendor/nim-json-serialization", + "url": "https://github.com/status-im/nim-json-serialization.git", + "commit": "e5b18fb710c3d0167ec79f3b892f5a7a1bc6d1a4" + }, + "nim-testutils": { + "dir": "vendor/nim-testutils", + "url": "https://github.com/status-im/nim-testutils", + "commit": "b56a5953e37fc5117bd6ea6dfa18418c5e112815" + }, + "nim-unittest2": { + "dir": "vendor/nim-unittest2", + "url": "https://github.com/status-im/nim-unittest2.git", + "commit": "b178f47527074964f76c395ad0dfc81cf118f379" + }, + "npeg": { + "dir": "vendor/npeg", + "url": "https://github.com/zevv/npeg", + "commit": "b15a10e388b91b898c581dbbcb6a718d46b27d2f" + }, + "nim-serialization": { + "dir": "vendor/nim-serialization", + "url": "https://github.com/status-im/nim-serialization.git", + "commit": "493d18b8292fc03aa4f835fd825dea1183f97466" + }, + "nim-faststreams": { + "dir": "vendor/nim-faststreams", + "url": "https://github.com/status-im/nim-faststreams.git", + "commit": "1b561a9e71b6bdad1c1cdff753418906037e9d09" + }, + "nim-datastore": { + "dir": "vendor/nim-datastore", + "url": "https://github.com/codex-storage/nim-datastore.git", + "commit": "0cde8aeb67c59fd0ac95496dc6b5e1168d6632aa" + }, + "asynctest": { + "dir": "vendor/asynctest", + "url": "https://github.com/markspanbroek/asynctest", + "commit": "a236a5f0f3031573ac2cb082b63dbf6e170e06e7" + }, + "nim-stint": { + "dir": "vendor/nim-stint", + "url": "https://github.com/status-im/nim-stint.git", + "commit": "036c71d06a6b22f8f967ba9d54afd2189c3872ca" + }, + "nim-metrics": { + "dir": "vendor/nim-metrics", + "url": "https://github.com/status-im/nim-metrics.git", + "commit": "743f81d4f6c6ebf0ac02389f2392ff8b4235bee5" + }, + "nim-libp2p": { + "dir": "vendor/nim-libp2p", + "url": "https://github.com/status-im/nim-libp2p.git", + "commit": "a3e9d1ed80c048cd5abc839cbe0863cefcedc702" + }, + "nim-chronicles": { + "dir": "vendor/nim-chronicles", + "url": "https://github.com/status-im/nim-chronicles.git", + "commit": "7631f7b2ee03398cb1512a79923264e8f9410af6" + }, + "nim-protobuf-serialization": { + "dir": "vendor/nim-protobuf-serialization", + "url": "https://github.com/status-im/nim-protobuf-serialization", + "commit": "28214b3e40c755a9886d2ec8f261ec48fbb6bec6" + } + }, + "nimcfg": "############# begin Atlas config section ##########\n--noNimblePath\n--path:\"vendor/nim-secp256k1\"\n--path:\"vendor/nim-protobuf-serialization\"\n--path:\"vendor/nimcrypto\"\n--path:\"vendor/nim-bearssl\"\n--path:\"vendor/nim-chronicles\"\n--path:\"vendor/nim-chronos\"\n--path:\"vendor/nim-libp2p\"\n--path:\"vendor/nim-metrics\"\n--path:\"vendor/nim-stew\"\n--path:\"vendor/nim-stint\"\n--path:\"vendor/asynctest\"\n--path:\"vendor/nim-datastore\"\n--path:\"vendor/questionable\"\n--path:\"vendor/nim-faststreams\"\n--path:\"vendor/nim-serialization\"\n--path:\"vendor/npeg/src\"\n--path:\"vendor/nim-unittest2\"\n--path:\"vendor/nim-testutils\"\n--path:\"vendor/nim-json-serialization\"\n--path:\"vendor/nim-http-utils\"\n--path:\"vendor/dnsclient.nim/src\"\n--path:\"vendor/nim-websock\"\n--path:\"vendor/nim-sqlite3-abi\"\n--path:\"vendor/upraises\"\n--path:\"vendor/nim-zlib\"\n############# end Atlas config section ##########\n", + "nimbleFile": { + "filename": "nim-codex-dht.nimble", + "content": "# Package\n\nversion = \"0.0.1\"\nauthor = \"Status Research & Development GmbH\"\ndescription = \"DHT based on the libp2p Kademlia spec\"\nlicense = \"MIT\"\nskipDirs = @[\"tests\"]\n\n\n# Dependencies\nrequires \"nim >= 1.2.0\"\nrequires \"secp256k1#b3f38e2795e805743b299dc5d96d332db375b520\" # >= 0.5.2 & < 0.6.0\nrequires \"protobuf_serialization#27b400fdf3bd8ce7120ca66fc1de39d3f1a5804a\" # >= 0.2.0 & < 0.3.0\nrequires \"nimcrypto == 0.5.4\"\nrequires \"bearssl#head\"\nrequires \"chronicles >= 0.10.2 & < 0.11.0\"\nrequires \"chronos#1394c9e04957928afc1db33d2e0965cfb677a1e0\" # >= 3.0.11 & < 3.1.0\nrequires \"libp2p#unstable\"\nrequires \"metrics\"\nrequires \"stew#head\"\nrequires \"stint\"\nrequires \"asynctest >= 0.3.1 & < 0.4.0\"\nrequires \"https://github.com/status-im/nim-datastore#head\"\nrequires \"questionable\"\n\ntask testAll, \"Run DHT tests\":\n exec \"nim c -r tests/testAll.nim\"\n\n# task coverage, \"generates code coverage report\":\n# var (output, exitCode) = gorgeEx(\"which lcov\")\n# if exitCode != 0:\n# echo \"\"\n# echo \" ************************** ⛔️ ERROR ⛔️ **************************\"\n# echo \" ** **\"\n# echo \" ** ERROR: lcov not found, it must be installed to run code **\"\n# echo \" ** coverage locally **\"\n# echo \" ** **\"\n# echo \" *****************************************************************\"\n# echo \"\"\n# quit 1\n\n# (output, exitCode) = gorgeEx(\"gcov --version\")\n# if output.contains(\"Apple LLVM\"):\n# echo \"\"\n# echo \" ************************* ⚠️ WARNING ⚠️ *************************\"\n# echo \" ** **\"\n# echo \" ** WARNING: Using Apple's llvm-cov in place of gcov, which **\"\n# echo \" ** emulates an old version of gcov (4.2.0) and therefore **\"\n# echo \" ** coverage results will differ than those on CI (which **\"\n# echo \" ** uses a much newer version of gcov). **\"\n# echo \" ** **\"\n# echo \" *****************************************************************\"\n# echo \"\"\n\n# exec(\"nimble --verbose test --opt:speed -d:debug --verbosity:0 --hints:off --lineDir:on -d:chronicles_log_level=INFO --nimcache:nimcache --passC:-fprofile-arcs --passC:-ftest-coverage --passL:-fprofile-arcs --passL:-ftest-coverage\")\n# exec(\"cd nimcache; rm *.c; cd ..\")\n# mkDir(\"coverage\")\n# exec(\"lcov --capture --directory nimcache --output-file coverage/coverage.info\")\n# exec(\"$(which bash) -c 'shopt -s globstar; ls $(pwd)/libp2pdht/{*,**/*}.nim'\")\n# exec(\"$(which bash) -c 'shopt -s globstar; lcov --extract coverage/coverage.info $(pwd)/libp2pdht/{*,**/*}.nim --output-file coverage/coverage.f.info'\")\n# echo \"Generating HTML coverage report\"\n# exec(\"genhtml coverage/coverage.f.info --output-directory coverage/report\")\n# echo \"Opening HTML coverage report in browser...\"\n# exec(\"open coverage/report/index.html\")\n\n" + }, + "hostOS": "macosx", + "hostCPU": "arm64", + "nimVersion": "1.6.14", + "gccVersion": "", + "clangVersion": "" +} \ No newline at end of file diff --git a/libp2pdht.nim b/codexdht.nim similarity index 100% rename from libp2pdht.nim rename to codexdht.nim diff --git a/codexdht.nimble b/codexdht.nimble new file mode 100644 index 0000000..75023d4 --- /dev/null +++ b/codexdht.nimble @@ -0,0 +1,65 @@ +# Package + +version = "0.2.0" +author = "Status Research & Development GmbH" +description = "DHT based on the libp2p Kademlia spec" +license = "MIT" +skipDirs = @["tests"] + + +# Dependencies +requires "nim >= 1.2.0" +requires "secp256k1#b3f38e2795e805743b299dc5d96d332db375b520" # >= 0.5.2 & < 0.6.0 +requires "protobuf_serialization#27b400fdf3bd8ce7120ca66fc1de39d3f1a5804a" # >= 0.2.0 & < 0.3.0 +requires "nimcrypto == 0.5.4" +requires "bearssl#head" +requires "chronicles >= 0.10.2 & < 0.11.0" +requires "chronos#1394c9e04957928afc1db33d2e0965cfb677a1e0" # >= 3.0.11 & < 3.1.0 +requires "libp2p#unstable" +requires "metrics" +requires "stew#head" +requires "stint" +requires "asynctest >= 0.3.1 & < 0.4.0" +requires "https://github.com/status-im/nim-datastore#head" +requires "questionable" + +task testAll, "Run DHT tests": + exec "nim c -r tests/testAll.nim" + +# task coverage, "generates code coverage report": +# var (output, exitCode) = gorgeEx("which lcov") +# if exitCode != 0: +# echo "" +# echo " ************************** ⛔️ ERROR ⛔️ **************************" +# echo " ** **" +# echo " ** ERROR: lcov not found, it must be installed to run code **" +# echo " ** coverage locally **" +# echo " ** **" +# echo " *****************************************************************" +# echo "" +# quit 1 + +# (output, exitCode) = gorgeEx("gcov --version") +# if output.contains("Apple LLVM"): +# echo "" +# echo " ************************* ⚠️ WARNING ⚠️ *************************" +# echo " ** **" +# echo " ** WARNING: Using Apple's llvm-cov in place of gcov, which **" +# echo " ** emulates an old version of gcov (4.2.0) and therefore **" +# echo " ** coverage results will differ than those on CI (which **" +# echo " ** uses a much newer version of gcov). **" +# echo " ** **" +# echo " *****************************************************************" +# echo "" + +# exec("nimble --verbose test --opt:speed -d:debug --verbosity:0 --hints:off --lineDir:on -d:chronicles_log_level=INFO --nimcache:nimcache --passC:-fprofile-arcs --passC:-ftest-coverage --passL:-fprofile-arcs --passL:-ftest-coverage") +# exec("cd nimcache; rm *.c; cd ..") +# mkDir("coverage") +# exec("lcov --capture --directory nimcache --output-file coverage/coverage.info") +# exec("$(which bash) -c 'shopt -s globstar; ls $(pwd)/libp2pdht/{*,**/*}.nim'") +# exec("$(which bash) -c 'shopt -s globstar; lcov --extract coverage/coverage.info $(pwd)/libp2pdht/{*,**/*}.nim --output-file coverage/coverage.f.info'") +# echo "Generating HTML coverage report" +# exec("genhtml coverage/coverage.f.info --output-directory coverage/report") +# echo "Opening HTML coverage report in browser..." +# exec("open coverage/report/index.html") + diff --git a/config.nims b/config.nims index 068054d..4d5fef5 100644 --- a/config.nims +++ b/config.nims @@ -1,7 +1,25 @@ +import std/os + +const currentDir = currentSourcePath()[0 .. ^(len("config.nims") + 1)] + switch("define", "libp2p_pki_schemes=secp256k1") -# begin Nimble config (version 2) ---noNimblePath -when withDir(thisDir(), system.fileExists("nimble.paths")): +task testAll, "Run DHT tests": + exec "nim c -r tests/testAll.nim" + +task test, "Run DHT tests": + testAllTask() + +when getEnv("NIMBUS_BUILD_SYSTEM") == "yes" and + # BEWARE + # In Nim 1.6, config files are evaluated with a working directory + # matching where the Nim command was invocated. This means that we + # must do all file existance checks with full absolute paths: + system.fileExists(currentDir & "nimbus-build-system.paths"): + echo "Using Nimbus Paths" + include "nimbus-build-system.paths" +elif fileExists("nimble.paths"): + echo "Using Nimble Paths" + # begin Nimble config (version 1) include "nimble.paths" -# end Nimble config + # end Nimble config diff --git a/env.sh b/env.sh new file mode 100755 index 0000000..697a426 --- /dev/null +++ b/env.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +# We use ${BASH_SOURCE[0]} instead of $0 to allow sourcing this file +# and we fall back to a Zsh-specific special var to also support Zsh. +REL_PATH="$(dirname ${BASH_SOURCE[0]:-${(%):-%x}})" +ABS_PATH="$(cd ${REL_PATH}; pwd)" +source ${ABS_PATH}/vendor/nimbus-build-system/scripts/env.sh diff --git a/libp2pdht.nimble b/libp2pdht.nimble deleted file mode 100644 index 29712ca..0000000 --- a/libp2pdht.nimble +++ /dev/null @@ -1,63 +0,0 @@ -# Package - -version = "0.1.0" -author = "Status Research & Development GmbH" -description = "DHT based on the libp2p Kademlia spec" -license = "MIT" -skipDirs = @["tests"] - - -# TODO: fix versions after tagging the version in the corresponding package -# Dependencies -requires "nim >= 1.2.0" -requires "secp256k1#b3f38e2795e805743b299dc5d96d332db375b520" # >= 0.5.2 & < 0.6.0 -requires "protobuf_serialization#27b400fdf3bd8ce7120ca66fc1de39d3f1a5804a" # >= 0.2.0 & < 0.3.0 -requires "nimcrypto == 0.5.4" -requires "bearssl#head" -requires "chronicles >= 0.10.2 & < 0.11.0" -requires "chronos#1394c9e04957928afc1db33d2e0965cfb677a1e0" # >= 3.0.11 & < 3.1.0 -requires "libp2p#unstable" -requires "metrics" -requires "stew#head" -requires "stint" -requires "asynctest >= 0.3.1 & < 0.4.0" -requires "https://github.com/status-im/nim-datastore#head" -requires "questionable" - -task coverage, "generates code coverage report": - var (output, exitCode) = gorgeEx("which lcov") - if exitCode != 0: - echo "" - echo " ************************** ⛔️ ERROR ⛔️ **************************" - echo " ** **" - echo " ** ERROR: lcov not found, it must be installed to run code **" - echo " ** coverage locally **" - echo " ** **" - echo " *****************************************************************" - echo "" - quit 1 - - (output, exitCode) = gorgeEx("gcov --version") - if output.contains("Apple LLVM"): - echo "" - echo " ************************* ⚠️ WARNING ⚠️ *************************" - echo " ** **" - echo " ** WARNING: Using Apple's llvm-cov in place of gcov, which **" - echo " ** emulates an old version of gcov (4.2.0) and therefore **" - echo " ** coverage results will differ than those on CI (which **" - echo " ** uses a much newer version of gcov). **" - echo " ** **" - echo " *****************************************************************" - echo "" - - exec("nimble --verbose test --opt:speed -d:debug --verbosity:0 --hints:off --lineDir:on -d:chronicles_log_level=INFO --nimcache:nimcache --passC:-fprofile-arcs --passC:-ftest-coverage --passL:-fprofile-arcs --passL:-ftest-coverage") - exec("cd nimcache; rm *.c; cd ..") - mkDir("coverage") - exec("lcov --capture --directory nimcache --output-file coverage/coverage.info") - exec("$(which bash) -c 'shopt -s globstar; ls $(pwd)/libp2pdht/{*,**/*}.nim'") - exec("$(which bash) -c 'shopt -s globstar; lcov --extract coverage/coverage.info $(pwd)/libp2pdht/{*,**/*}.nim --output-file coverage/coverage.f.info'") - echo "Generating HTML coverage report" - exec("genhtml coverage/coverage.f.info --output-directory coverage/report") - echo "Opening HTML coverage report in browser..." - exec("open coverage/report/index.html") - diff --git a/vendor/atlas.workspace b/vendor/atlas.workspace new file mode 100644 index 0000000..79173a1 --- /dev/null +++ b/vendor/atlas.workspace @@ -0,0 +1,2 @@ +deps="" +resolver="MaxVer" From bd72bc1e60643236d702a812af324f92126baf55 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 11 Jul 2023 15:24:54 -0700 Subject: [PATCH 02/30] add nimbus build --- atlas.lock | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/atlas.lock b/atlas.lock index 04169bc..e019c4a 100644 --- a/atlas.lock +++ b/atlas.lock @@ -1,5 +1,10 @@ { "items": { + "nimbus-build-system": { + "dir": "vendor/nimbus-build-system", + "url": "https://github.com/status-im/nimbus-build-system", + "commit": "239c3a7fbb88fd241da0ade3246fd2e5fcff4f25" + }, "nim-zlib": { "dir": "vendor/nim-zlib", "url": "https://github.com/status-im/nim-zlib", From 169e1d76e520bbf14d735923c6b6a6da29c55244 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 11 Jul 2023 15:48:47 -0700 Subject: [PATCH 03/30] updates --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 4cb31ba..78ffdbc 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,7 @@ LINK_PCRE := 0 ifeq ($(NIM_PARAMS),) # "variables.mk" was not included, so we update the submodules. -GIT_SUBMODULE_UPDATE := git submodule update --init --recursive +GIT_SUBMODULE_UPDATE := nimble install https://github.com/elcritch/atlas && atlas rep atlas.lock .DEFAULT: +@ echo -e "Git submodules not found. Running '$(GIT_SUBMODULE_UPDATE)'.\n"; \ $(GIT_SUBMODULE_UPDATE); \ From 58f91c48c048bc60eff08e3e704de954e218b72e Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 11 Jul 2023 15:59:27 -0700 Subject: [PATCH 04/30] add nat traversal to make nimbus happy --- atlas.lock | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/atlas.lock b/atlas.lock index e019c4a..0e94752 100644 --- a/atlas.lock +++ b/atlas.lock @@ -5,6 +5,11 @@ "url": "https://github.com/status-im/nimbus-build-system", "commit": "239c3a7fbb88fd241da0ade3246fd2e5fcff4f25" }, + "nim-nat-traversal": { + "dir": "vendor/nim-nat-traversal", + "url": "https://github.com/status-im/nim-nat-traversal", + "commit": "802d75edcc656e616120fb27f950ff1285ddcbba" + }, "nim-zlib": { "dir": "vendor/nim-zlib", "url": "https://github.com/status-im/nim-zlib", From 5e5851c1e24234348e3494fde0ce2038873aa507 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 11 Jul 2023 16:07:00 -0700 Subject: [PATCH 05/30] clone nimbus vendor using atlas --- Makefile | 13 ++----------- atlas.lock | 6 ------ 2 files changed, 2 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index 78ffdbc..fe7e9ae 100644 --- a/Makefile +++ b/Makefile @@ -45,15 +45,10 @@ else # "variables.mk" was included. Business as usual until the end of this file # default target, because it's the first one that doesn't start with '.' -# Builds the codex binary -all: | build deps - echo -e $(BUILD_MSG) "$@" && \ - $(ENV_SCRIPT) nim codex $(NIM_PARAMS) codexdht.nims - # must be included after the default target -include $(BUILD_SYSTEM_DIR)/makefiles/targets.mk -deps: | deps-common nat-libs codexdht.nims +deps: | deps-common nat-libs #- deletes and recreates "codexdht.nims" which on Windows is a copy instead of a proper symlink update: | update-common codexdht.nims @@ -63,11 +58,7 @@ update: | update-common codexdht.nims # Builds and run a part of the test suite test: | build deps echo -e $(BUILD_MSG) "$@" && \ - $(ENV_SCRIPT) nim testAll $(NIM_PARAMS) codexdht.nims - -# symlink -codexdht.nims: - ln -s codexdht.nimble $@ + $(ENV_SCRIPT) nim testAll $(NIM_PARAMS) config.nims # usual cleaning clean: | clean-common diff --git a/atlas.lock b/atlas.lock index 0e94752..15fafdc 100644 --- a/atlas.lock +++ b/atlas.lock @@ -137,12 +137,6 @@ } }, "nimcfg": "############# begin Atlas config section ##########\n--noNimblePath\n--path:\"vendor/nim-secp256k1\"\n--path:\"vendor/nim-protobuf-serialization\"\n--path:\"vendor/nimcrypto\"\n--path:\"vendor/nim-bearssl\"\n--path:\"vendor/nim-chronicles\"\n--path:\"vendor/nim-chronos\"\n--path:\"vendor/nim-libp2p\"\n--path:\"vendor/nim-metrics\"\n--path:\"vendor/nim-stew\"\n--path:\"vendor/nim-stint\"\n--path:\"vendor/asynctest\"\n--path:\"vendor/nim-datastore\"\n--path:\"vendor/questionable\"\n--path:\"vendor/nim-faststreams\"\n--path:\"vendor/nim-serialization\"\n--path:\"vendor/npeg/src\"\n--path:\"vendor/nim-unittest2\"\n--path:\"vendor/nim-testutils\"\n--path:\"vendor/nim-json-serialization\"\n--path:\"vendor/nim-http-utils\"\n--path:\"vendor/dnsclient.nim/src\"\n--path:\"vendor/nim-websock\"\n--path:\"vendor/nim-sqlite3-abi\"\n--path:\"vendor/upraises\"\n--path:\"vendor/nim-zlib\"\n############# end Atlas config section ##########\n", - "nimbleFile": { - "filename": "nim-codex-dht.nimble", - "content": "# Package\n\nversion = \"0.0.1\"\nauthor = \"Status Research & Development GmbH\"\ndescription = \"DHT based on the libp2p Kademlia spec\"\nlicense = \"MIT\"\nskipDirs = @[\"tests\"]\n\n\n# Dependencies\nrequires \"nim >= 1.2.0\"\nrequires \"secp256k1#b3f38e2795e805743b299dc5d96d332db375b520\" # >= 0.5.2 & < 0.6.0\nrequires \"protobuf_serialization#27b400fdf3bd8ce7120ca66fc1de39d3f1a5804a\" # >= 0.2.0 & < 0.3.0\nrequires \"nimcrypto == 0.5.4\"\nrequires \"bearssl#head\"\nrequires \"chronicles >= 0.10.2 & < 0.11.0\"\nrequires \"chronos#1394c9e04957928afc1db33d2e0965cfb677a1e0\" # >= 3.0.11 & < 3.1.0\nrequires \"libp2p#unstable\"\nrequires \"metrics\"\nrequires \"stew#head\"\nrequires \"stint\"\nrequires \"asynctest >= 0.3.1 & < 0.4.0\"\nrequires \"https://github.com/status-im/nim-datastore#head\"\nrequires \"questionable\"\n\ntask testAll, \"Run DHT tests\":\n exec \"nim c -r tests/testAll.nim\"\n\n# task coverage, \"generates code coverage report\":\n# var (output, exitCode) = gorgeEx(\"which lcov\")\n# if exitCode != 0:\n# echo \"\"\n# echo \" ************************** ⛔️ ERROR ⛔️ **************************\"\n# echo \" ** **\"\n# echo \" ** ERROR: lcov not found, it must be installed to run code **\"\n# echo \" ** coverage locally **\"\n# echo \" ** **\"\n# echo \" *****************************************************************\"\n# echo \"\"\n# quit 1\n\n# (output, exitCode) = gorgeEx(\"gcov --version\")\n# if output.contains(\"Apple LLVM\"):\n# echo \"\"\n# echo \" ************************* ⚠️ WARNING ⚠️ *************************\"\n# echo \" ** **\"\n# echo \" ** WARNING: Using Apple's llvm-cov in place of gcov, which **\"\n# echo \" ** emulates an old version of gcov (4.2.0) and therefore **\"\n# echo \" ** coverage results will differ than those on CI (which **\"\n# echo \" ** uses a much newer version of gcov). **\"\n# echo \" ** **\"\n# echo \" *****************************************************************\"\n# echo \"\"\n\n# exec(\"nimble --verbose test --opt:speed -d:debug --verbosity:0 --hints:off --lineDir:on -d:chronicles_log_level=INFO --nimcache:nimcache --passC:-fprofile-arcs --passC:-ftest-coverage --passL:-fprofile-arcs --passL:-ftest-coverage\")\n# exec(\"cd nimcache; rm *.c; cd ..\")\n# mkDir(\"coverage\")\n# exec(\"lcov --capture --directory nimcache --output-file coverage/coverage.info\")\n# exec(\"$(which bash) -c 'shopt -s globstar; ls $(pwd)/libp2pdht/{*,**/*}.nim'\")\n# exec(\"$(which bash) -c 'shopt -s globstar; lcov --extract coverage/coverage.info $(pwd)/libp2pdht/{*,**/*}.nim --output-file coverage/coverage.f.info'\")\n# echo \"Generating HTML coverage report\"\n# exec(\"genhtml coverage/coverage.f.info --output-directory coverage/report\")\n# echo \"Opening HTML coverage report in browser...\"\n# exec(\"open coverage/report/index.html\")\n\n" - }, - "hostOS": "macosx", - "hostCPU": "arm64", "nimVersion": "1.6.14", "gccVersion": "", "clangVersion": "" From 3f552af2cec2cbe01eeae66fac186ca30998a10b Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 11 Jul 2023 16:16:54 -0700 Subject: [PATCH 06/30] disable ci-nimbus --- .github/workflows/ci-nimbus.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/ci-nimbus.yml b/.github/workflows/ci-nimbus.yml index 06db9d4..a96b06f 100644 --- a/.github/workflows/ci-nimbus.yml +++ b/.github/workflows/ci-nimbus.yml @@ -1,8 +1,6 @@ -name: CI +name: CI-nimbus on: push: - # branches: - # - main pull_request: workflow_dispatch: From 6cda13db5d135f09e2583af5c3fe7d6a63c8df52 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 11 Jul 2023 16:21:48 -0700 Subject: [PATCH 07/30] disable ci-nimbus --- .github/workflows/ci-nimbus.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-nimbus.yml b/.github/workflows/ci-nimbus.yml index a96b06f..1a2ddce 100644 --- a/.github/workflows/ci-nimbus.yml +++ b/.github/workflows/ci-nimbus.yml @@ -1,8 +1,9 @@ name: CI-nimbus on: push: - pull_request: - workflow_dispatch: + # total hack... can remove later but I want to preserve for now + branches: + - disable_this_ci jobs: build: From 32cbb41a92bbb05ca5c1edb088fac96b05cf7406 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 11 Jul 2023 16:28:04 -0700 Subject: [PATCH 08/30] only run ci-nimbus on lockfile change --- .github/workflows/ci-nimbus.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-nimbus.yml b/.github/workflows/ci-nimbus.yml index 1a2ddce..d2dad5a 100644 --- a/.github/workflows/ci-nimbus.yml +++ b/.github/workflows/ci-nimbus.yml @@ -1,9 +1,8 @@ name: CI-nimbus on: push: - # total hack... can remove later but I want to preserve for now - branches: - - disable_this_ci + paths: + - atlas.lock jobs: build: @@ -30,7 +29,7 @@ jobs: shell: bash - target: os: macos - builder: macos-10.15 + builder: macos-12 shell: bash - target: os: windows From 8fdf907811b36e84b3774e61f032f3a272f457e2 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 11 Jul 2023 16:29:27 -0700 Subject: [PATCH 09/30] fake change to test ci --- atlas.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/atlas.lock b/atlas.lock index 15fafdc..f672e7f 100644 --- a/atlas.lock +++ b/atlas.lock @@ -140,4 +140,4 @@ "nimVersion": "1.6.14", "gccVersion": "", "clangVersion": "" -} \ No newline at end of file +} From 992cf83e3d2678528279aa5444a92c2ffd66c1a3 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 11 Jul 2023 16:41:29 -0700 Subject: [PATCH 10/30] fake change to test ci-nimbus --- .github/workflows/ci-nimbus.yml | 4 ++++ atlas.lock | 1 + 2 files changed, 5 insertions(+) diff --git a/.github/workflows/ci-nimbus.yml b/.github/workflows/ci-nimbus.yml index d2dad5a..ac8e038 100644 --- a/.github/workflows/ci-nimbus.yml +++ b/.github/workflows/ci-nimbus.yml @@ -158,6 +158,10 @@ jobs: # https://github.com/status-im/nimbus-eth2/issues/3121 export NIMFLAGS="-d:nimRawSetjmp" fi + echo "ENV: " + env + + echo "BUILD: " export NIM_COMMIT=${{ matrix.branch }} make -j${ncpu} CI_CACHE=NimBinaries ARCH_OVERRIDE=${PLATFORM} QUICK_AND_DIRTY_COMPILER=1 update make test -j${ncpu} diff --git a/atlas.lock b/atlas.lock index f672e7f..c4a045b 100644 --- a/atlas.lock +++ b/atlas.lock @@ -141,3 +141,4 @@ "gccVersion": "", "clangVersion": "" } + \ No newline at end of file From 6fadf32ee80881bba43cae3352ef6ae088152c49 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 11 Jul 2023 17:08:27 -0700 Subject: [PATCH 11/30] tweaks --- .github/workflows/ci-nimbus.yml | 1 + Makefile | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-nimbus.yml b/.github/workflows/ci-nimbus.yml index ac8e038..94b6164 100644 --- a/.github/workflows/ci-nimbus.yml +++ b/.github/workflows/ci-nimbus.yml @@ -3,6 +3,7 @@ on: push: paths: - atlas.lock + - .github/workflows/ci-nimbus.yml jobs: build: diff --git a/Makefile b/Makefile index fe7e9ae..7f018c4 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,7 @@ LINK_PCRE := 0 ifeq ($(NIM_PARAMS),) # "variables.mk" was not included, so we update the submodules. -GIT_SUBMODULE_UPDATE := nimble install https://github.com/elcritch/atlas && atlas rep atlas.lock +GIT_SUBMODULE_UPDATE := nimble install https://github.com/elcritch/atlas && atlas rep --noexec atlas.lock .DEFAULT: +@ echo -e "Git submodules not found. Running '$(GIT_SUBMODULE_UPDATE)'.\n"; \ $(GIT_SUBMODULE_UPDATE); \ @@ -45,6 +45,11 @@ else # "variables.mk" was included. Business as usual until the end of this file # default target, because it's the first one that doesn't start with '.' +# Builds the codex binary +all: | build deps + echo -e $(BUILD_MSG) "$@" && \ + $(ENV_SCRIPT) nim test $(NIM_PARAMS) + # must be included after the default target -include $(BUILD_SYSTEM_DIR)/makefiles/targets.mk From 59d45be04597dbeac28df76ac2b39ef6eab0be8a Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 11 Jul 2023 17:15:42 -0700 Subject: [PATCH 12/30] add nim setup --- .github/workflows/ci-nimbus.yml | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci-nimbus.yml b/.github/workflows/ci-nimbus.yml index 94b6164..2ad69d1 100644 --- a/.github/workflows/ci-nimbus.yml +++ b/.github/workflows/ci-nimbus.yml @@ -141,13 +141,18 @@ jobs: echo "ncpu=$ncpu" >> $GITHUB_ENV echo "MAKE_CMD=${MAKE_CMD}" >> $GITHUB_ENV - - name: Restore Nim toolchain binaries from cache - id: nim-cache - uses: actions/cache@v3 + - uses: jiro4989/setup-nim-action@v1 with: - path: NimBinaries - key: ${{ matrix.target.os }}-${{ matrix.target.cpu }}-nim-${{ matrix.branch }}-cache-${{ env.cache_nonce }}-${{ github.run_id }} - restore-keys: ${{ matrix.target.os }}-${{ matrix.target.cpu }}-nim-${{ matrix.branch }}-cache-${{ env.cache_nonce }} + nim-version: 1.6.14 + repo-token: ${{ secrets.GITHUB_TOKEN }} + + # - name: Restore Nim toolchain binaries from cache + # id: nim-cache + # uses: actions/cache@v3 + # with: + # path: NimBinaries + # key: ${{ matrix.target.os }}-${{ matrix.target.cpu }}-nim-${{ matrix.branch }}-cache-${{ env.cache_nonce }}-${{ github.run_id }} + # restore-keys: ${{ matrix.target.os }}-${{ matrix.target.cpu }}-nim-${{ matrix.branch }}-cache-${{ env.cache_nonce }} # - name: Set NIM_COMMIT # shell: ${{ matrix.shell }} {0} From 3fd3008a829a107ea1dcb40dffb24ac29b12d4af Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 11 Jul 2023 17:22:17 -0700 Subject: [PATCH 13/30] updates --- .github/workflows/ci-nimbus.yml | 5 ++++- Makefile | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-nimbus.yml b/.github/workflows/ci-nimbus.yml index 2ad69d1..e22210c 100644 --- a/.github/workflows/ci-nimbus.yml +++ b/.github/workflows/ci-nimbus.yml @@ -158,6 +158,8 @@ jobs: # shell: ${{ matrix.shell }} {0} # run: echo "NIM_COMMIT=${{ matrix.branch }}" >> ${GITHUB_ENV} + - name: Run tests + run: | - name: Run tests run: | if [[ "${{ matrix.target.os }}" == "windows" ]]; then @@ -169,5 +171,6 @@ jobs: echo "BUILD: " export NIM_COMMIT=${{ matrix.branch }} - make -j${ncpu} CI_CACHE=NimBinaries ARCH_OVERRIDE=${PLATFORM} QUICK_AND_DIRTY_COMPILER=1 update + # make -j${ncpu} CI_CACHE=NimBinaries ARCH_OVERRIDE=${PLATFORM} QUICK_AND_DIRTY_COMPILER=1 update + make -j${ncpu} make test -j${ncpu} diff --git a/Makefile b/Makefile index 7f018c4..a507610 100644 --- a/Makefile +++ b/Makefile @@ -56,7 +56,7 @@ all: | build deps deps: | deps-common nat-libs #- deletes and recreates "codexdht.nims" which on Windows is a copy instead of a proper symlink -update: | update-common codexdht.nims +update: | update-common rm -rf codexdht.nims && \ $(MAKE) codexdht.nims $(HANDLE_OUTPUT) From 4a241ffa8c73f5777a25061fae1ca54fe295e6f7 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 11 Jul 2023 17:24:57 -0700 Subject: [PATCH 14/30] don't use update (?) target --- .github/workflows/ci-nimbus.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci-nimbus.yml b/.github/workflows/ci-nimbus.yml index e22210c..e6a93c5 100644 --- a/.github/workflows/ci-nimbus.yml +++ b/.github/workflows/ci-nimbus.yml @@ -171,6 +171,5 @@ jobs: echo "BUILD: " export NIM_COMMIT=${{ matrix.branch }} - # make -j${ncpu} CI_CACHE=NimBinaries ARCH_OVERRIDE=${PLATFORM} QUICK_AND_DIRTY_COMPILER=1 update - make -j${ncpu} + make -j${ncpu} CI_CACHE=NimBinaries ARCH_OVERRIDE=${PLATFORM} QUICK_AND_DIRTY_COMPILER=1 make test -j${ncpu} From a3d050fc18f156c206ef90e3388ac95cc6f872aa Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 11 Jul 2023 17:37:34 -0700 Subject: [PATCH 15/30] caching --- .github/workflows/ci-nimbus.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci-nimbus.yml b/.github/workflows/ci-nimbus.yml index e6a93c5..b92a01c 100644 --- a/.github/workflows/ci-nimbus.yml +++ b/.github/workflows/ci-nimbus.yml @@ -146,20 +146,20 @@ jobs: nim-version: 1.6.14 repo-token: ${{ secrets.GITHUB_TOKEN }} - # - name: Restore Nim toolchain binaries from cache - # id: nim-cache - # uses: actions/cache@v3 - # with: - # path: NimBinaries - # key: ${{ matrix.target.os }}-${{ matrix.target.cpu }}-nim-${{ matrix.branch }}-cache-${{ env.cache_nonce }}-${{ github.run_id }} - # restore-keys: ${{ matrix.target.os }}-${{ matrix.target.cpu }}-nim-${{ matrix.branch }}-cache-${{ env.cache_nonce }} + - name: Restore Nim toolchain binaries from cache + id: nim-cache + uses: actions/cache@v3 + with: + path: NimBinaries + key: ${{ matrix.target.os }}-${{ matrix.target.cpu }}-nim-${{ hashFiles('atlas.lock') }} - # - name: Set NIM_COMMIT - # shell: ${{ matrix.shell }} {0} - # run: echo "NIM_COMMIT=${{ matrix.branch }}" >> ${GITHUB_ENV} + - name: Restore Vendor Clones from cache + id: nim-cache + uses: actions/cache@v3 + with: + path: vendor/*/ + key: ${{ matrix.target.os }}-${{ matrix.target.cpu }}-vendor-${{ hashFiles('atlas.lock') }} - - name: Run tests - run: | - name: Run tests run: | if [[ "${{ matrix.target.os }}" == "windows" ]]; then From ff5853780cebbdcaf65d159dd9a97a71194c0e08 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 11 Jul 2023 17:39:48 -0700 Subject: [PATCH 16/30] caching --- atlas.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/atlas.lock b/atlas.lock index c4a045b..4ff1795 100644 --- a/atlas.lock +++ b/atlas.lock @@ -141,4 +141,4 @@ "gccVersion": "", "clangVersion": "" } - \ No newline at end of file + \ No newline at end of file From ef68f139387e30de0004fd6a1262e9ea531f33d7 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 11 Jul 2023 17:40:26 -0700 Subject: [PATCH 17/30] caching --- .github/workflows/ci-nimbus.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-nimbus.yml b/.github/workflows/ci-nimbus.yml index b92a01c..34a2ab4 100644 --- a/.github/workflows/ci-nimbus.yml +++ b/.github/workflows/ci-nimbus.yml @@ -154,7 +154,7 @@ jobs: key: ${{ matrix.target.os }}-${{ matrix.target.cpu }}-nim-${{ hashFiles('atlas.lock') }} - name: Restore Vendor Clones from cache - id: nim-cache + id: vendor-cache uses: actions/cache@v3 with: path: vendor/*/ From f9c98e7de73534af1c796e150b1dd00637ab038e Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 11 Jul 2023 17:44:54 -0700 Subject: [PATCH 18/30] caching - change names --- .github/workflows/ci-nimbus.yml | 4 ++-- atlas.lock | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-nimbus.yml b/.github/workflows/ci-nimbus.yml index 34a2ab4..1e8d6aa 100644 --- a/.github/workflows/ci-nimbus.yml +++ b/.github/workflows/ci-nimbus.yml @@ -151,14 +151,14 @@ jobs: uses: actions/cache@v3 with: path: NimBinaries - key: ${{ matrix.target.os }}-${{ matrix.target.cpu }}-nim-${{ hashFiles('atlas.lock') }} + key: nim-${{ matrix.target.os }}-${{ matrix.target.cpu }}-${{ hashFiles('atlas.lock') }} - name: Restore Vendor Clones from cache id: vendor-cache uses: actions/cache@v3 with: path: vendor/*/ - key: ${{ matrix.target.os }}-${{ matrix.target.cpu }}-vendor-${{ hashFiles('atlas.lock') }} + key: vendor-${{ matrix.target.os }}-${{ matrix.target.cpu }}-${{ hashFiles('atlas.lock') }} - name: Run tests run: | diff --git a/atlas.lock b/atlas.lock index 4ff1795..f672e7f 100644 --- a/atlas.lock +++ b/atlas.lock @@ -141,4 +141,3 @@ "gccVersion": "", "clangVersion": "" } - \ No newline at end of file From 31413856f6ba97f7d66dddb14f7707121624eacd Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 11 Jul 2023 17:50:44 -0700 Subject: [PATCH 19/30] Revert "caching" This reverts commit ef68f139387e30de0004fd6a1262e9ea531f33d7. --- .github/workflows/ci-nimbus.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-nimbus.yml b/.github/workflows/ci-nimbus.yml index 1e8d6aa..914dbd2 100644 --- a/.github/workflows/ci-nimbus.yml +++ b/.github/workflows/ci-nimbus.yml @@ -154,7 +154,7 @@ jobs: key: nim-${{ matrix.target.os }}-${{ matrix.target.cpu }}-${{ hashFiles('atlas.lock') }} - name: Restore Vendor Clones from cache - id: vendor-cache + id: nim-cache uses: actions/cache@v3 with: path: vendor/*/ From ff18bf6fc3b4b6cf9133aa864f18a50d17cd0991 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 11 Jul 2023 17:53:01 -0700 Subject: [PATCH 20/30] restore names --- .github/workflows/ci-nimbus.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-nimbus.yml b/.github/workflows/ci-nimbus.yml index 914dbd2..34a2ab4 100644 --- a/.github/workflows/ci-nimbus.yml +++ b/.github/workflows/ci-nimbus.yml @@ -151,14 +151,14 @@ jobs: uses: actions/cache@v3 with: path: NimBinaries - key: nim-${{ matrix.target.os }}-${{ matrix.target.cpu }}-${{ hashFiles('atlas.lock') }} + key: ${{ matrix.target.os }}-${{ matrix.target.cpu }}-nim-${{ hashFiles('atlas.lock') }} - name: Restore Vendor Clones from cache - id: nim-cache + id: vendor-cache uses: actions/cache@v3 with: path: vendor/*/ - key: vendor-${{ matrix.target.os }}-${{ matrix.target.cpu }}-${{ hashFiles('atlas.lock') }} + key: ${{ matrix.target.os }}-${{ matrix.target.cpu }}-vendor-${{ hashFiles('atlas.lock') }} - name: Run tests run: | From 0f7732cb66da300279782217e99653071e52a56e Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 11 Jul 2023 17:58:06 -0700 Subject: [PATCH 21/30] try cache again --- .github/workflows/ci.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3786a2f..54f584c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -140,14 +140,14 @@ jobs: echo "ncpu=$ncpu" >> $GITHUB_ENV echo "MAKE_CMD=${MAKE_CMD}" >> $GITHUB_ENV - # - name: Restore nimble dependencies from cache - # id: nimble_deps - # uses: actions/cache@v3 - # with: - # path: | - # ~/.nimble/pkgs2 - # ~/.nimble/packages_official.json - # key: ${{ inputs.os }}-${{ inputs.cpu }}-${{ env.cache_nonce }} + - name: Restore nimble dependencies from cache + id: nimble_deps + uses: actions/cache@v3 + with: + path: | + ~/.nimble/pkgs2 + ~/.nimble/packages_official.json + key: ${{ matrix.target.os }}-${{ matrix.target.cpu }}-nimble-${{ hashFiles('nimble.lock') }} - name: Setup Nimble uses: "./.github/actions/install_nimble" From 39f1364982de118fb4b66c3700e78eaa754e2690 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 11 Jul 2023 18:42:41 -0700 Subject: [PATCH 22/30] cleanup & test --- .github/workflows/ci-nimbus.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci-nimbus.yml b/.github/workflows/ci-nimbus.yml index 34a2ab4..11fea1b 100644 --- a/.github/workflows/ci-nimbus.yml +++ b/.github/workflows/ci-nimbus.yml @@ -166,8 +166,6 @@ jobs: # https://github.com/status-im/nimbus-eth2/issues/3121 export NIMFLAGS="-d:nimRawSetjmp" fi - echo "ENV: " - env echo "BUILD: " export NIM_COMMIT=${{ matrix.branch }} From 5dae32b0a7048474e4c7dd18637362984c05eba1 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 11 Jul 2023 18:48:22 -0700 Subject: [PATCH 23/30] ignores --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index e5a54f0..584c0a3 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,5 @@ nimble.paths nim.cfg nimbus-build-system.paths vendor/* +NimBinaries +.update.timestamp From 7a5639d57014a8601b8c6d896576f883e682c88f Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 12 Jul 2023 14:33:31 -0700 Subject: [PATCH 24/30] bump version --- libp2pdht.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libp2pdht.nimble b/libp2pdht.nimble index 29712ca..80421fe 100644 --- a/libp2pdht.nimble +++ b/libp2pdht.nimble @@ -1,6 +1,6 @@ # Package -version = "0.1.0" +version = "0.2.0" author = "Status Research & Development GmbH" description = "DHT based on the libp2p Kademlia spec" license = "MIT" From c516a02a65b567b9fc2d997106fdf5ae9c4895d5 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 12 Jul 2023 14:47:07 -0700 Subject: [PATCH 25/30] bump version --- codexdht.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codexdht.nimble b/codexdht.nimble index 75023d4..6ad94a8 100644 --- a/codexdht.nimble +++ b/codexdht.nimble @@ -1,6 +1,6 @@ # Package -version = "0.2.0" +version = "0.2.1" author = "Status Research & Development GmbH" description = "DHT based on the libp2p Kademlia spec" license = "MIT" From c3d073c3da62af81498f68e7e93f7bfce6e8b6fe Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 12 Jul 2023 14:57:26 -0700 Subject: [PATCH 26/30] finishing renaming --- {libp2pdht => codexdht}/dht.nim | 0 {libp2pdht => codexdht}/dht/providers_encoding.nim | 0 {libp2pdht => codexdht}/dht/providers_messages.nim | 0 {libp2pdht => codexdht}/discv5.nim | 0 {libp2pdht => codexdht}/discv5/crypto.nim | 0 {libp2pdht => codexdht}/discv5/encoding.nim | 0 {libp2pdht => codexdht}/discv5/messages.nim | 0 {libp2pdht => codexdht}/discv5/messages_encoding.nim | 0 {libp2pdht => codexdht}/discv5/node.nim | 0 {libp2pdht => codexdht}/discv5/nodes_verification.nim | 0 {libp2pdht => codexdht}/discv5/protocol.nim | 0 {libp2pdht => codexdht}/discv5/routing_table.nim | 0 {libp2pdht => codexdht}/discv5/sessions.nim | 0 {libp2pdht => codexdht}/discv5/spr.nim | 0 {libp2pdht => codexdht}/discv5/transport.nim | 0 {libp2pdht => codexdht}/private/eth/p2p/discoveryv5/crypto.nim | 0 {libp2pdht => codexdht}/private/eth/p2p/discoveryv5/encoding.nim | 0 {libp2pdht => codexdht}/private/eth/p2p/discoveryv5/hkdf.nim | 0 {libp2pdht => codexdht}/private/eth/p2p/discoveryv5/ip_vote.nim | 0 {libp2pdht => codexdht}/private/eth/p2p/discoveryv5/lru.nim | 0 {libp2pdht => codexdht}/private/eth/p2p/discoveryv5/messages.nim | 0 .../private/eth/p2p/discoveryv5/messages_encoding.nim | 0 {libp2pdht => codexdht}/private/eth/p2p/discoveryv5/node.nim | 0 .../private/eth/p2p/discoveryv5/nodes_verification.nim | 0 {libp2pdht => codexdht}/private/eth/p2p/discoveryv5/protocol.nim | 0 {libp2pdht => codexdht}/private/eth/p2p/discoveryv5/providers.nim | 0 .../private/eth/p2p/discoveryv5/providers/cache.nim | 0 .../private/eth/p2p/discoveryv5/providers/common.nim | 0 .../private/eth/p2p/discoveryv5/providers/maintenance.nim | 0 .../private/eth/p2p/discoveryv5/providers/manager.nim | 0 {libp2pdht => codexdht}/private/eth/p2p/discoveryv5/random2.nim | 0 .../private/eth/p2p/discoveryv5/routing_table.nim | 0 {libp2pdht => codexdht}/private/eth/p2p/discoveryv5/sessions.nim | 0 {libp2pdht => codexdht}/private/eth/p2p/discoveryv5/spr.nim | 0 {libp2pdht => codexdht}/private/eth/p2p/discoveryv5/transport.nim | 0 35 files changed, 0 insertions(+), 0 deletions(-) rename {libp2pdht => codexdht}/dht.nim (100%) rename {libp2pdht => codexdht}/dht/providers_encoding.nim (100%) rename {libp2pdht => codexdht}/dht/providers_messages.nim (100%) rename {libp2pdht => codexdht}/discv5.nim (100%) rename {libp2pdht => codexdht}/discv5/crypto.nim (100%) rename {libp2pdht => codexdht}/discv5/encoding.nim (100%) rename {libp2pdht => codexdht}/discv5/messages.nim (100%) rename {libp2pdht => codexdht}/discv5/messages_encoding.nim (100%) rename {libp2pdht => codexdht}/discv5/node.nim (100%) rename {libp2pdht => codexdht}/discv5/nodes_verification.nim (100%) rename {libp2pdht => codexdht}/discv5/protocol.nim (100%) rename {libp2pdht => codexdht}/discv5/routing_table.nim (100%) rename {libp2pdht => codexdht}/discv5/sessions.nim (100%) rename {libp2pdht => codexdht}/discv5/spr.nim (100%) rename {libp2pdht => codexdht}/discv5/transport.nim (100%) rename {libp2pdht => codexdht}/private/eth/p2p/discoveryv5/crypto.nim (100%) rename {libp2pdht => codexdht}/private/eth/p2p/discoveryv5/encoding.nim (100%) rename {libp2pdht => codexdht}/private/eth/p2p/discoveryv5/hkdf.nim (100%) rename {libp2pdht => codexdht}/private/eth/p2p/discoveryv5/ip_vote.nim (100%) rename {libp2pdht => codexdht}/private/eth/p2p/discoveryv5/lru.nim (100%) rename {libp2pdht => codexdht}/private/eth/p2p/discoveryv5/messages.nim (100%) rename {libp2pdht => codexdht}/private/eth/p2p/discoveryv5/messages_encoding.nim (100%) rename {libp2pdht => codexdht}/private/eth/p2p/discoveryv5/node.nim (100%) rename {libp2pdht => codexdht}/private/eth/p2p/discoveryv5/nodes_verification.nim (100%) rename {libp2pdht => codexdht}/private/eth/p2p/discoveryv5/protocol.nim (100%) rename {libp2pdht => codexdht}/private/eth/p2p/discoveryv5/providers.nim (100%) rename {libp2pdht => codexdht}/private/eth/p2p/discoveryv5/providers/cache.nim (100%) rename {libp2pdht => codexdht}/private/eth/p2p/discoveryv5/providers/common.nim (100%) rename {libp2pdht => codexdht}/private/eth/p2p/discoveryv5/providers/maintenance.nim (100%) rename {libp2pdht => codexdht}/private/eth/p2p/discoveryv5/providers/manager.nim (100%) rename {libp2pdht => codexdht}/private/eth/p2p/discoveryv5/random2.nim (100%) rename {libp2pdht => codexdht}/private/eth/p2p/discoveryv5/routing_table.nim (100%) rename {libp2pdht => codexdht}/private/eth/p2p/discoveryv5/sessions.nim (100%) rename {libp2pdht => codexdht}/private/eth/p2p/discoveryv5/spr.nim (100%) rename {libp2pdht => codexdht}/private/eth/p2p/discoveryv5/transport.nim (100%) diff --git a/libp2pdht/dht.nim b/codexdht/dht.nim similarity index 100% rename from libp2pdht/dht.nim rename to codexdht/dht.nim diff --git a/libp2pdht/dht/providers_encoding.nim b/codexdht/dht/providers_encoding.nim similarity index 100% rename from libp2pdht/dht/providers_encoding.nim rename to codexdht/dht/providers_encoding.nim diff --git a/libp2pdht/dht/providers_messages.nim b/codexdht/dht/providers_messages.nim similarity index 100% rename from libp2pdht/dht/providers_messages.nim rename to codexdht/dht/providers_messages.nim diff --git a/libp2pdht/discv5.nim b/codexdht/discv5.nim similarity index 100% rename from libp2pdht/discv5.nim rename to codexdht/discv5.nim diff --git a/libp2pdht/discv5/crypto.nim b/codexdht/discv5/crypto.nim similarity index 100% rename from libp2pdht/discv5/crypto.nim rename to codexdht/discv5/crypto.nim diff --git a/libp2pdht/discv5/encoding.nim b/codexdht/discv5/encoding.nim similarity index 100% rename from libp2pdht/discv5/encoding.nim rename to codexdht/discv5/encoding.nim diff --git a/libp2pdht/discv5/messages.nim b/codexdht/discv5/messages.nim similarity index 100% rename from libp2pdht/discv5/messages.nim rename to codexdht/discv5/messages.nim diff --git a/libp2pdht/discv5/messages_encoding.nim b/codexdht/discv5/messages_encoding.nim similarity index 100% rename from libp2pdht/discv5/messages_encoding.nim rename to codexdht/discv5/messages_encoding.nim diff --git a/libp2pdht/discv5/node.nim b/codexdht/discv5/node.nim similarity index 100% rename from libp2pdht/discv5/node.nim rename to codexdht/discv5/node.nim diff --git a/libp2pdht/discv5/nodes_verification.nim b/codexdht/discv5/nodes_verification.nim similarity index 100% rename from libp2pdht/discv5/nodes_verification.nim rename to codexdht/discv5/nodes_verification.nim diff --git a/libp2pdht/discv5/protocol.nim b/codexdht/discv5/protocol.nim similarity index 100% rename from libp2pdht/discv5/protocol.nim rename to codexdht/discv5/protocol.nim diff --git a/libp2pdht/discv5/routing_table.nim b/codexdht/discv5/routing_table.nim similarity index 100% rename from libp2pdht/discv5/routing_table.nim rename to codexdht/discv5/routing_table.nim diff --git a/libp2pdht/discv5/sessions.nim b/codexdht/discv5/sessions.nim similarity index 100% rename from libp2pdht/discv5/sessions.nim rename to codexdht/discv5/sessions.nim diff --git a/libp2pdht/discv5/spr.nim b/codexdht/discv5/spr.nim similarity index 100% rename from libp2pdht/discv5/spr.nim rename to codexdht/discv5/spr.nim diff --git a/libp2pdht/discv5/transport.nim b/codexdht/discv5/transport.nim similarity index 100% rename from libp2pdht/discv5/transport.nim rename to codexdht/discv5/transport.nim diff --git a/libp2pdht/private/eth/p2p/discoveryv5/crypto.nim b/codexdht/private/eth/p2p/discoveryv5/crypto.nim similarity index 100% rename from libp2pdht/private/eth/p2p/discoveryv5/crypto.nim rename to codexdht/private/eth/p2p/discoveryv5/crypto.nim diff --git a/libp2pdht/private/eth/p2p/discoveryv5/encoding.nim b/codexdht/private/eth/p2p/discoveryv5/encoding.nim similarity index 100% rename from libp2pdht/private/eth/p2p/discoveryv5/encoding.nim rename to codexdht/private/eth/p2p/discoveryv5/encoding.nim diff --git a/libp2pdht/private/eth/p2p/discoveryv5/hkdf.nim b/codexdht/private/eth/p2p/discoveryv5/hkdf.nim similarity index 100% rename from libp2pdht/private/eth/p2p/discoveryv5/hkdf.nim rename to codexdht/private/eth/p2p/discoveryv5/hkdf.nim diff --git a/libp2pdht/private/eth/p2p/discoveryv5/ip_vote.nim b/codexdht/private/eth/p2p/discoveryv5/ip_vote.nim similarity index 100% rename from libp2pdht/private/eth/p2p/discoveryv5/ip_vote.nim rename to codexdht/private/eth/p2p/discoveryv5/ip_vote.nim diff --git a/libp2pdht/private/eth/p2p/discoveryv5/lru.nim b/codexdht/private/eth/p2p/discoveryv5/lru.nim similarity index 100% rename from libp2pdht/private/eth/p2p/discoveryv5/lru.nim rename to codexdht/private/eth/p2p/discoveryv5/lru.nim diff --git a/libp2pdht/private/eth/p2p/discoveryv5/messages.nim b/codexdht/private/eth/p2p/discoveryv5/messages.nim similarity index 100% rename from libp2pdht/private/eth/p2p/discoveryv5/messages.nim rename to codexdht/private/eth/p2p/discoveryv5/messages.nim diff --git a/libp2pdht/private/eth/p2p/discoveryv5/messages_encoding.nim b/codexdht/private/eth/p2p/discoveryv5/messages_encoding.nim similarity index 100% rename from libp2pdht/private/eth/p2p/discoveryv5/messages_encoding.nim rename to codexdht/private/eth/p2p/discoveryv5/messages_encoding.nim diff --git a/libp2pdht/private/eth/p2p/discoveryv5/node.nim b/codexdht/private/eth/p2p/discoveryv5/node.nim similarity index 100% rename from libp2pdht/private/eth/p2p/discoveryv5/node.nim rename to codexdht/private/eth/p2p/discoveryv5/node.nim diff --git a/libp2pdht/private/eth/p2p/discoveryv5/nodes_verification.nim b/codexdht/private/eth/p2p/discoveryv5/nodes_verification.nim similarity index 100% rename from libp2pdht/private/eth/p2p/discoveryv5/nodes_verification.nim rename to codexdht/private/eth/p2p/discoveryv5/nodes_verification.nim diff --git a/libp2pdht/private/eth/p2p/discoveryv5/protocol.nim b/codexdht/private/eth/p2p/discoveryv5/protocol.nim similarity index 100% rename from libp2pdht/private/eth/p2p/discoveryv5/protocol.nim rename to codexdht/private/eth/p2p/discoveryv5/protocol.nim diff --git a/libp2pdht/private/eth/p2p/discoveryv5/providers.nim b/codexdht/private/eth/p2p/discoveryv5/providers.nim similarity index 100% rename from libp2pdht/private/eth/p2p/discoveryv5/providers.nim rename to codexdht/private/eth/p2p/discoveryv5/providers.nim diff --git a/libp2pdht/private/eth/p2p/discoveryv5/providers/cache.nim b/codexdht/private/eth/p2p/discoveryv5/providers/cache.nim similarity index 100% rename from libp2pdht/private/eth/p2p/discoveryv5/providers/cache.nim rename to codexdht/private/eth/p2p/discoveryv5/providers/cache.nim diff --git a/libp2pdht/private/eth/p2p/discoveryv5/providers/common.nim b/codexdht/private/eth/p2p/discoveryv5/providers/common.nim similarity index 100% rename from libp2pdht/private/eth/p2p/discoveryv5/providers/common.nim rename to codexdht/private/eth/p2p/discoveryv5/providers/common.nim diff --git a/libp2pdht/private/eth/p2p/discoveryv5/providers/maintenance.nim b/codexdht/private/eth/p2p/discoveryv5/providers/maintenance.nim similarity index 100% rename from libp2pdht/private/eth/p2p/discoveryv5/providers/maintenance.nim rename to codexdht/private/eth/p2p/discoveryv5/providers/maintenance.nim diff --git a/libp2pdht/private/eth/p2p/discoveryv5/providers/manager.nim b/codexdht/private/eth/p2p/discoveryv5/providers/manager.nim similarity index 100% rename from libp2pdht/private/eth/p2p/discoveryv5/providers/manager.nim rename to codexdht/private/eth/p2p/discoveryv5/providers/manager.nim diff --git a/libp2pdht/private/eth/p2p/discoveryv5/random2.nim b/codexdht/private/eth/p2p/discoveryv5/random2.nim similarity index 100% rename from libp2pdht/private/eth/p2p/discoveryv5/random2.nim rename to codexdht/private/eth/p2p/discoveryv5/random2.nim diff --git a/libp2pdht/private/eth/p2p/discoveryv5/routing_table.nim b/codexdht/private/eth/p2p/discoveryv5/routing_table.nim similarity index 100% rename from libp2pdht/private/eth/p2p/discoveryv5/routing_table.nim rename to codexdht/private/eth/p2p/discoveryv5/routing_table.nim diff --git a/libp2pdht/private/eth/p2p/discoveryv5/sessions.nim b/codexdht/private/eth/p2p/discoveryv5/sessions.nim similarity index 100% rename from libp2pdht/private/eth/p2p/discoveryv5/sessions.nim rename to codexdht/private/eth/p2p/discoveryv5/sessions.nim diff --git a/libp2pdht/private/eth/p2p/discoveryv5/spr.nim b/codexdht/private/eth/p2p/discoveryv5/spr.nim similarity index 100% rename from libp2pdht/private/eth/p2p/discoveryv5/spr.nim rename to codexdht/private/eth/p2p/discoveryv5/spr.nim diff --git a/libp2pdht/private/eth/p2p/discoveryv5/transport.nim b/codexdht/private/eth/p2p/discoveryv5/transport.nim similarity index 100% rename from libp2pdht/private/eth/p2p/discoveryv5/transport.nim rename to codexdht/private/eth/p2p/discoveryv5/transport.nim From d7cc0ae7836ecba183b0769f03de8a741d494f07 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 12 Jul 2023 14:58:29 -0700 Subject: [PATCH 27/30] renaming imports --- codexdht.nim | 4 ++-- codexdht.nimble | 4 ++-- tests/dht/test_helper.nim | 6 +++--- tests/dht/test_providermngr.nim | 10 +++++----- tests/dht/test_providers.nim | 6 +++--- tests/discv5/test_discoveryv5.nim | 6 +++--- tests/discv5/test_discoveryv5_encoding.nim | 4 ++-- 7 files changed, 20 insertions(+), 20 deletions(-) diff --git a/codexdht.nim b/codexdht.nim index a907dd1..a4a4a9d 100644 --- a/codexdht.nim +++ b/codexdht.nim @@ -1,5 +1,5 @@ import - ./libp2pdht/dht, - ./libp2pdht/discv5 + ./codexdht/dht, + ./codexdht/discv5 export dht, discv5 diff --git a/codexdht.nimble b/codexdht.nimble index 6ad94a8..dc47d8d 100644 --- a/codexdht.nimble +++ b/codexdht.nimble @@ -56,8 +56,8 @@ task testAll, "Run DHT tests": # exec("cd nimcache; rm *.c; cd ..") # mkDir("coverage") # exec("lcov --capture --directory nimcache --output-file coverage/coverage.info") -# exec("$(which bash) -c 'shopt -s globstar; ls $(pwd)/libp2pdht/{*,**/*}.nim'") -# exec("$(which bash) -c 'shopt -s globstar; lcov --extract coverage/coverage.info $(pwd)/libp2pdht/{*,**/*}.nim --output-file coverage/coverage.f.info'") +# exec("$(which bash) -c 'shopt -s globstar; ls $(pwd)/codexdht/{*,**/*}.nim'") +# exec("$(which bash) -c 'shopt -s globstar; lcov --extract coverage/coverage.info $(pwd)/codexdht/{*,**/*}.nim --output-file coverage/coverage.f.info'") # echo "Generating HTML coverage report" # exec("genhtml coverage/coverage.f.info --output-directory coverage/report") # echo "Opening HTML coverage report in browser..." diff --git a/tests/dht/test_helper.nim b/tests/dht/test_helper.nim index 89f5797..df1fd86 100644 --- a/tests/dht/test_helper.nim +++ b/tests/dht/test_helper.nim @@ -3,9 +3,9 @@ import chronos, libp2p/crypto/[crypto, secp], libp2p/multiaddress, - libp2pdht/discv5/[node, routing_table, spr], - libp2pdht/discv5/crypto as dhtcrypto, - libp2pdht/discv5/protocol as discv5_protocol, + codexdht/discv5/[node, routing_table, spr], + codexdht/discv5/crypto as dhtcrypto, + codexdht/discv5/protocol as discv5_protocol, stew/shims/net export net diff --git a/tests/dht/test_providermngr.nim b/tests/dht/test_providermngr.nim index 7ec742c..ea2456d 100644 --- a/tests/dht/test_providermngr.nim +++ b/tests/dht/test_providermngr.nim @@ -6,11 +6,11 @@ import pkg/asynctest import pkg/datastore import pkg/libp2p -import libp2pdht/dht -import libp2pdht/private/eth/p2p/discoveryv5/spr -import libp2pdht/private/eth/p2p/discoveryv5/providers -import libp2pdht/discv5/node -import libp2pdht/private/eth/p2p/discoveryv5/lru +import codexdht/dht +import codexdht/private/eth/p2p/discoveryv5/spr +import codexdht/private/eth/p2p/discoveryv5/providers +import codexdht/discv5/node +import codexdht/private/eth/p2p/discoveryv5/lru import ./test_helper suite "Test Providers Manager simple": diff --git a/tests/dht/test_providers.nim b/tests/dht/test_providers.nim index fecb9b9..dab5174 100644 --- a/tests/dht/test_providers.nim +++ b/tests/dht/test_providers.nim @@ -18,9 +18,9 @@ import nimcrypto, libp2p/crypto/[crypto, secp], libp2p/[multiaddress, multicodec, multihash, routing_record, signed_envelope], - libp2pdht/dht, - libp2pdht/discv5/crypto as dhtcrypto, - libp2pdht/discv5/protocol as discv5_protocol, + codexdht/dht, + codexdht/discv5/crypto as dhtcrypto, + codexdht/discv5/protocol as discv5_protocol, stew/byteutils, test_helper diff --git a/tests/discv5/test_discoveryv5.nim b/tests/discv5/test_discoveryv5.nim index ad0066d..5be5b39 100644 --- a/tests/discv5/test_discoveryv5.nim +++ b/tests/discv5/test_discoveryv5.nim @@ -5,9 +5,9 @@ import chronos, chronicles, stint, asynctest, stew/shims/net, stew/byteutils, bearssl/rand, libp2p/crypto/crypto, - libp2pdht/discv5/[transport, spr, node, routing_table, encoding, sessions, nodes_verification], - libp2pdht/discv5/crypto as dhtcrypto, - libp2pdht/discv5/protocol as discv5_protocol, + codexdht/discv5/[transport, spr, node, routing_table, encoding, sessions, nodes_verification], + codexdht/discv5/crypto as dhtcrypto, + codexdht/discv5/protocol as discv5_protocol, ../dht/test_helper suite "Discovery v5 Tests": diff --git a/tests/discv5/test_discoveryv5_encoding.nim b/tests/discv5/test_discoveryv5_encoding.nim index 3eb2380..68bf95c 100644 --- a/tests/discv5/test_discoveryv5_encoding.nim +++ b/tests/discv5/test_discoveryv5_encoding.nim @@ -6,8 +6,8 @@ import bearssl/rand, chronos, libp2p/crypto/secp, - libp2pdht/discv5/[messages, messages_encoding, encoding, spr, node, sessions], - libp2pdht/discv5/crypto, + codexdht/discv5/[messages, messages_encoding, encoding, spr, node, sessions], + codexdht/discv5/crypto, stew/byteutils, stew/shims/net, stint, From 127d7be4e506b746e6fc20eb5854d8ced20f2240 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 12 Jul 2023 15:47:16 -0700 Subject: [PATCH 28/30] bump version --- codexdht.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codexdht.nimble b/codexdht.nimble index dc47d8d..fb9fa7e 100644 --- a/codexdht.nimble +++ b/codexdht.nimble @@ -1,6 +1,6 @@ # Package -version = "0.2.1" +version = "0.3.0" author = "Status Research & Development GmbH" description = "DHT based on the libp2p Kademlia spec" license = "MIT" From 7464c8e9a63523f6138c972bfc7c4a706bfa3909 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 13 Jul 2023 20:42:43 -0700 Subject: [PATCH 29/30] Split windows tests (#70) Changes the CI to properly cache Nimble deps *including* Nim binaries. * parallelize tests * speed up tests * cache whole nimble * use nimble install without -d * bump version * new nimble cache * fix github_env * compare speed * readd msys2 * don't need make for nimble * ugh renames --- .github/workflows/ci-nimbus.yml | 70 ++++--------- .github/workflows/ci.yml | 112 +++++++-------------- .gitignore | 3 + Makefile | 4 +- codexdht.nimble | 11 +- config.nims | 17 +++- tests/dht/test_helper.nim | 6 +- tests/dht/test_providers.nim | 4 +- tests/discv5/test_discoveryv5.nim | 6 +- tests/discv5/test_discoveryv5_encoding.nim | 16 +-- tests/testAllParallel.nim | 22 ++++ 11 files changed, 124 insertions(+), 147 deletions(-) create mode 100644 tests/testAllParallel.nim diff --git a/.github/workflows/ci-nimbus.yml b/.github/workflows/ci-nimbus.yml index 11fea1b..7f83914 100644 --- a/.github/workflows/ci-nimbus.yml +++ b/.github/workflows/ci-nimbus.yml @@ -1,9 +1,14 @@ name: CI-nimbus on: + # push: + # paths: + # - atlas.lock + # - .github/workflows/ci-nimbus.yml push: - paths: - - atlas.lock - - .github/workflows/ci-nimbus.yml + branches: + - main + pull_request: + workflow_dispatch: jobs: build: @@ -14,28 +19,24 @@ jobs: target: - os: linux cpu: amd64 - # - os: linux - # cpu: i386 - - os: macos - cpu: amd64 - - os: windows - cpu: amd64 - #- os: windows - #cpu: i386 + # - os: macos + # cpu: amd64 + # - os: windows + # cpu: amd64 branch: [version-1-6] include: - target: os: linux builder: ubuntu-20.04 shell: bash - - target: - os: macos - builder: macos-12 - shell: bash - - target: - os: windows - builder: windows-2019 - shell: msys2 {0} + # - target: + # os: macos + # builder: macos-12 + # shell: bash + # - target: + # os: windows + # builder: windows-2019 + # shell: msys2 {0} defaults: run: @@ -50,37 +51,6 @@ jobs: with: submodules: true - - name: Install build dependencies (Linux i386) - if: runner.os == 'Linux' && matrix.target.cpu == 'i386' - run: | - sudo dpkg --add-architecture i386 - sudo apt-get update -qq - sudo DEBIAN_FRONTEND='noninteractive' apt-get install \ - --no-install-recommends -yq gcc-multilib g++-multilib \ - libssl-dev:i386 - mkdir -p external/bin - cat << EOF > external/bin/gcc - #!/bin/bash - exec $(which gcc) -m32 "\$@" - EOF - cat << EOF > external/bin/g++ - #!/bin/bash - exec $(which g++) -m32 "\$@" - EOF - chmod 755 external/bin/gcc external/bin/g++ - echo '${{ github.workspace }}/external/bin' >> $GITHUB_PATH - - - name: MSYS2 (Windows i386) - if: runner.os == 'Windows' && matrix.target.cpu == 'i386' - uses: msys2/setup-msys2@v2 - with: - path-type: inherit - msystem: MINGW32 - install: >- - base-devel - git - mingw-w64-i686-toolchain - - name: MSYS2 (Windows amd64) if: runner.os == 'Windows' && matrix.target.cpu == 'amd64' uses: msys2/setup-msys2@v2 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 54f584c..b9f644c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,12 +15,16 @@ jobs: target: - os: linux cpu: amd64 + tests: all - os: macos cpu: amd64 + tests: all - os: windows cpu: amd64 - #- os: windows - #cpu: i386 + tests: part1 + - os: windows + cpu: amd64 + tests: part2 branch: [version-1-6] include: - target: @@ -33,7 +37,7 @@ jobs: shell: bash - target: os: windows - builder: windows-2019 + builder: windows-latest shell: msys2 {0} defaults: @@ -49,37 +53,6 @@ jobs: with: submodules: true - - name: Install build dependencies (Linux i386) - if: runner.os == 'Linux' && matrix.target.cpu == 'i386' - run: | - sudo dpkg --add-architecture i386 - sudo apt-get update -qq - sudo DEBIAN_FRONTEND='noninteractive' apt-get install \ - --no-install-recommends -yq gcc-multilib g++-multilib \ - libssl-dev:i386 - mkdir -p external/bin - cat << EOF > external/bin/gcc - #!/bin/bash - exec $(which gcc) -m32 "\$@" - EOF - cat << EOF > external/bin/g++ - #!/bin/bash - exec $(which g++) -m32 "\$@" - EOF - chmod 755 external/bin/gcc external/bin/g++ - echo '${{ github.workspace }}/external/bin' >> $GITHUB_PATH - - - name: MSYS2 (Windows i386) - if: runner.os == 'Windows' && matrix.target.cpu == 'i386' - uses: msys2/setup-msys2@v2 - with: - path-type: inherit - msystem: MINGW32 - install: >- - base-devel - git - mingw-w64-i686-toolchain - - name: MSYS2 (Windows amd64) if: runner.os == 'Windows' && matrix.target.cpu == 'amd64' uses: msys2/setup-msys2@v2 @@ -89,7 +62,6 @@ jobs: base-devel git mingw-w64-x86_64-toolchain - - name: Restore Nim DLLs dependencies (Windows) from cache if: runner.os == 'Windows' id: windows-dlls-cache @@ -106,48 +78,21 @@ jobs: mkdir external curl -L "https://nim-lang.org/download/windeps.zip" -o external/windeps.zip 7z x external/windeps.zip -oexternal/dlls - - name: Path to cached dependencies (Windows) if: > runner.os == 'Windows' run: | echo '${{ github.workspace }}'"/external/dlls" >> $GITHUB_PATH - - name: Derive environment variables - run: | - if [[ '${{ matrix.target.cpu }}' == 'amd64' ]]; then - PLATFORM=x64 - else - PLATFORM=x86 - fi - echo "PLATFORM=$PLATFORM" >> $GITHUB_ENV - - ncpu= - MAKE_CMD="make" - case '${{ runner.os }}' in - 'Linux') - ncpu=$(nproc) - ;; - 'macOS') - ncpu=$(sysctl -n hw.ncpu) - ;; - 'Windows') - ncpu=$NUMBER_OF_PROCESSORS - MAKE_CMD="mingw32-make" - ;; - esac - [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1 - echo "ncpu=$ncpu" >> $GITHUB_ENV - echo "MAKE_CMD=${MAKE_CMD}" >> $GITHUB_ENV - + ## Restore nimble deps - name: Restore nimble dependencies from cache id: nimble_deps uses: actions/cache@v3 with: path: | - ~/.nimble/pkgs2 - ~/.nimble/packages_official.json - key: ${{ matrix.target.os }}-${{ matrix.target.cpu }}-nimble-${{ hashFiles('nimble.lock') }} + ~/.nimble + ${{ github.workspace }}/.nimble + key: ${{ matrix.target.os }}-${{ matrix.target.cpu }}-dotnimble-${{ hashFiles('nimble.lock') }} - name: Setup Nimble uses: "./.github/actions/install_nimble" @@ -155,16 +100,35 @@ jobs: os: ${{ matrix.target.os }} cpu: ${{ matrix.target.cpu }} - - name: Run tests + - name: Setup Env + run: | + nimble -v + + - name: Setup Deps + run: | + nimble install -d + nimble setup + + - name: Run tests + if: runner.os != 'Windows' + run: | + nimble test -y + + - name: Run windows tests part1 + if: runner.os == 'Windows' && matrix.target.tests == 'part1' run: | - rm -rf ~/.nimble/ if [[ "${{ matrix.target.os }}" == "windows" ]]; then # https://github.com/status-im/nimbus-eth2/issues/3121 export NIMFLAGS="-d:nimRawSetjmp" fi - nimble test -y - if [[ "${{ matrix.branch }}" == "version-1-6" || "${{ matrix.branch }}" == "devel" ]]; then - echo -e "\nTesting with '--gc:orc':\n" - export NIMFLAGS="${NIMFLAGS} --gc:orc" - nimble test -y - fi; + + nimble testPart1 -y + + - name: Run windows tests part2 + if: runner.os == 'Windows' && matrix.target.tests == 'part2' + run: | + if [[ "${{ matrix.target.os }}" == "windows" ]]; then + export NIMFLAGS="-d:nimRawSetjmp" + fi + + nimble testPart2 -y diff --git a/.gitignore b/.gitignore index 584c0a3..49331ec 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +* +!*.* +!*/ coverage nimcache tests/testAll diff --git a/Makefile b/Makefile index a507610..384bad5 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,7 @@ LINK_PCRE := 0 ifeq ($(NIM_PARAMS),) # "variables.mk" was not included, so we update the submodules. -GIT_SUBMODULE_UPDATE := nimble install https://github.com/elcritch/atlas && atlas rep --noexec atlas.lock +GIT_SUBMODULE_UPDATE := nimble install https://github.com/nim-lang/atlas@\#2ab291c9f37e9d7acce906d1bb7fa49f57f10b22 && atlas rep --noexec atlas.lock .DEFAULT: +@ echo -e "Git submodules not found. Running '$(GIT_SUBMODULE_UPDATE)'.\n"; \ $(GIT_SUBMODULE_UPDATE); \ @@ -63,7 +63,7 @@ update: | update-common # Builds and run a part of the test suite test: | build deps echo -e $(BUILD_MSG) "$@" && \ - $(ENV_SCRIPT) nim testAll $(NIM_PARAMS) config.nims + $(ENV_SCRIPT) nim test $(NIM_PARAMS) config.nims # usual cleaning clean: | clean-common diff --git a/codexdht.nimble b/codexdht.nimble index fb9fa7e..00d943a 100644 --- a/codexdht.nimble +++ b/codexdht.nimble @@ -1,6 +1,6 @@ # Package -version = "0.3.0" +version = "0.3.1" author = "Status Research & Development GmbH" description = "DHT based on the libp2p Kademlia spec" license = "MIT" @@ -26,6 +26,15 @@ requires "questionable" task testAll, "Run DHT tests": exec "nim c -r tests/testAll.nim" +task test, "Run DHT tests": + exec "nim c -r -d:testsAll --verbosity:0 tests/testAllParallel.nim" + +task testPart1, "Run DHT tests A": + exec "nim c -r -d:testsPart1 tests/testAllParallel.nim" + +task testPart2, "Run DHT tests B": + exec "nim c -r -d:testsPart2 tests/testAllParallel.nim" + # task coverage, "generates code coverage report": # var (output, exitCode) = gorgeEx("which lcov") # if exitCode != 0: diff --git a/config.nims b/config.nims index 4d5fef5..13a33d5 100644 --- a/config.nims +++ b/config.nims @@ -8,7 +8,13 @@ task testAll, "Run DHT tests": exec "nim c -r tests/testAll.nim" task test, "Run DHT tests": - testAllTask() + exec "nim c -r -d:testsAll --verbosity:0 tests/testAllParallel.nim" + +task testPart1, "Run DHT tests A": + exec "nim c -r -d:testsPart1 tests/testAllParallel.nim" + +task testPart2, "Run DHT tests B": + exec "nim c -r -d:testsPart2 tests/testAllParallel.nim" when getEnv("NIMBUS_BUILD_SYSTEM") == "yes" and # BEWARE @@ -18,8 +24,11 @@ when getEnv("NIMBUS_BUILD_SYSTEM") == "yes" and system.fileExists(currentDir & "nimbus-build-system.paths"): echo "Using Nimbus Paths" include "nimbus-build-system.paths" -elif fileExists("nimble.paths"): +elif withDir(thisDir(), system.fileExists("nimble.paths")): echo "Using Nimble Paths" - # begin Nimble config (version 1) + +# begin Nimble config (version 2) +--noNimblePath +when withDir(thisDir(), system.fileExists("nimble.paths")): include "nimble.paths" - # end Nimble config +# end Nimble config diff --git a/tests/dht/test_helper.nim b/tests/dht/test_helper.nim index df1fd86..38772ca 100644 --- a/tests/dht/test_helper.nim +++ b/tests/dht/test_helper.nim @@ -53,7 +53,7 @@ proc nodeIdInNodes*(id: NodeId, nodes: openArray[Node]): bool = for n in nodes: if id == n.id: return true -proc generateNode*(privKey: PrivateKey, port: int = 20302, +proc generateNode*(privKey: PrivateKey, port: int, ip: ValidIpAddress = ValidIpAddress.init("127.0.0.1")): Node = let @@ -67,7 +67,7 @@ proc generateNRandomNodes*(rng: ref HmacDrbgContext, n: int): seq[Node] = for i in 1..n: let privKey = PrivateKey.example(rng) - node = privKey.generateNode() + node = privKey.generateNode(port = 20402 + 10*n) res.add(node) res @@ -76,7 +76,7 @@ proc nodeAndPrivKeyAtDistance*(n: Node, rng: var HmacDrbgContext, d: uint32, while true: let privKey = PrivateKey.random(rng).expect("Valid rng for private key") - node = privKey.generateNode(ip = ip) + node = privKey.generateNode(port = 21302 + 10*d.int, ip = ip) if logDistance(n.id, node.id) == d: return (node, privKey) diff --git a/tests/dht/test_providers.nim b/tests/dht/test_providers.nim index dab5174..3f01e52 100644 --- a/tests/dht/test_providers.nim +++ b/tests/dht/test_providers.nim @@ -34,7 +34,7 @@ proc bootstrapNodes( debug "---- STARTING BOOSTRAPS ---" for i in 0.. 0: @@ -53,7 +53,7 @@ proc bootstrapNetwork( bootNodeKey = PrivateKey.fromHex( "a2b50376a79b1a8c8a3296485572bdfbf54708bb46d3c25d73d2723aaaf6a617") .expect("Valid private key hex") - bootNodeAddr = localAddress(20301) + bootNodeAddr = localAddress(25311) bootNode = initDiscoveryNode(rng, bootNodeKey, bootNodeAddr, @[]) # just a shortcut for new and open #waitFor bootNode.bootstrap() # immediate, since no bootnodes are defined above diff --git a/tests/discv5/test_discoveryv5.nim b/tests/discv5/test_discoveryv5.nim index 5be5b39..1255569 100644 --- a/tests/discv5/test_discoveryv5.nim +++ b/tests/discv5/test_discoveryv5.nim @@ -22,13 +22,13 @@ suite "Discovery v5 Tests": pk = PrivateKey.example(rng) targetPk = PrivateKey.example(rng) node = initDiscoveryNode(rng, pk, localAddress(20302)) - targetNode = targetPk.generateNode() + targetNode = targetPk.generateNode(port=26302) check node.addNode(targetNode) for i in 0..<1000: let pk = PrivateKey.example(rng) - discard node.addNode(pk.generateNode()) + discard node.addNode(pk.generateNode(port=27302+i)) let n = node.getNode(targetNode.id) check n.isSome() @@ -265,7 +265,7 @@ suite "Discovery v5 Tests": # Generate 1000 random nodes and add to our main node's routing table for i in 0..<1000: - discard mainNode.addSeenNode(generateNode(PrivateKey.example(rng))) # for testing only! + discard mainNode.addSeenNode(generateNode(PrivateKey.example(rng), port=28302+i)) # for testing only! let neighbours = mainNode.neighbours(mainNode.localNode.id) diff --git a/tests/discv5/test_discoveryv5_encoding.nim b/tests/discv5/test_discoveryv5_encoding.nim index 68bf95c..e2c87bc 100644 --- a/tests/discv5/test_discoveryv5_encoding.nim +++ b/tests/discv5/test_discoveryv5_encoding.nim @@ -275,12 +275,12 @@ suite "Discovery v5.1 Packet Encodings Test Vectors": let enrRecA = SignedPeerRecord.init(1, privKeyA, - some(ValidIpAddress.init("127.0.0.1")), some(Port(9000)), - some(Port(9000))).expect("Properly intialized private key") + some(ValidIpAddress.init("127.0.0.1")), some(Port(9001)), + some(Port(9001))).expect("Properly intialized private key") enrRecB = SignedPeerRecord.init(1, privKeyB, - some(ValidIpAddress.init("127.0.0.1")), some(Port(9000)), - some(Port(9000))).expect("Properly intialized private key") + some(ValidIpAddress.init("127.0.0.1")), some(Port(9001)), + some(Port(9001))).expect("Properly intialized private key") nodeA = newNode(enrRecA).expect("Properly initialized record") nodeB = newNode(enrRecB).expect("Properly initialized record") @@ -508,12 +508,12 @@ suite "Discovery v5.1 Additional Encode/Decode": let enrRecA = SignedPeerRecord.init(1, privKeyA, - some(ValidIpAddress.init("127.0.0.1")), some(Port(9000)), - some(Port(9000))).expect("Properly intialized private key") + some(ValidIpAddress.init("127.0.0.1")), some(Port(9001)), + some(Port(9001))).expect("Properly intialized private key") enrRecB = SignedPeerRecord.init(1, privKeyB, - some(ValidIpAddress.init("127.0.0.1")), some(Port(9000)), - some(Port(9000))).expect("Properly intialized private key") + some(ValidIpAddress.init("127.0.0.1")), some(Port(9001)), + some(Port(9001))).expect("Properly intialized private key") nodeA = newNode(enrRecA).expect("Properly initialized record") nodeB = newNode(enrRecB).expect("Properly initialized record") diff --git a/tests/testAllParallel.nim b/tests/testAllParallel.nim new file mode 100644 index 0000000..9127064 --- /dev/null +++ b/tests/testAllParallel.nim @@ -0,0 +1,22 @@ +# import +# ./dht/[test_providers, test_providermngr], +# ./discv5/[test_discoveryv5, test_discoveryv5_encoding] + +import osproc + +var cmds: seq[string] + +when defined(testsPart1) or defined(testsAll): + cmds.add [ + "nim c -r --verbosity:0 tests/dht/test_providers.nim", + "nim c -r --verbosity:0 tests/dht/test_providermngr.nim", + ] +when defined(testsPart2) or defined(testsAll): + cmds.add [ + "nim c -r --verbosity:0 tests/discv5/test_discoveryv5.nim", + "nim c -r --verbosity:0 tests/discv5/test_discoveryv5_encoding.nim", + ] + +echo "CMDS: ", cmds + +quit execProcesses(cmds) From d4331f80621e1ba5c0eb5b1cdcd8853519fc5516 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 17 Jul 2023 12:43:14 -0700 Subject: [PATCH 30/30] Remove nimbus build (#73) * Remove Nimbus * adds docs for Nimble 0.14 * cleanup --- .github/workflows/ci-nimbus.yml | 143 -------------------------------- .gitignore | 1 + Makefile | 71 ---------------- README.md | 23 +++++ build.nims | 58 +++++++++++++ codexdht.nimble | 48 +---------- config.nims | 28 +------ env.sh | 7 -- tests/coverage.nim | 2 + tests/coverage.nims | 15 ++++ tests/testAll.nim | 7 +- tests/testAllParallel.nim | 10 +-- 12 files changed, 110 insertions(+), 303 deletions(-) delete mode 100644 .github/workflows/ci-nimbus.yml delete mode 100644 Makefile create mode 100644 build.nims delete mode 100755 env.sh create mode 100644 tests/coverage.nim create mode 100644 tests/coverage.nims diff --git a/.github/workflows/ci-nimbus.yml b/.github/workflows/ci-nimbus.yml deleted file mode 100644 index 7f83914..0000000 --- a/.github/workflows/ci-nimbus.yml +++ /dev/null @@ -1,143 +0,0 @@ -name: CI-nimbus -on: - # push: - # paths: - # - atlas.lock - # - .github/workflows/ci-nimbus.yml - push: - branches: - - main - pull_request: - workflow_dispatch: - -jobs: - build: - timeout-minutes: 90 - strategy: - fail-fast: false - matrix: - target: - - os: linux - cpu: amd64 - # - os: macos - # cpu: amd64 - # - os: windows - # cpu: amd64 - branch: [version-1-6] - include: - - target: - os: linux - builder: ubuntu-20.04 - shell: bash - # - target: - # os: macos - # builder: macos-12 - # shell: bash - # - target: - # os: windows - # builder: windows-2019 - # shell: msys2 {0} - - defaults: - run: - shell: ${{ matrix.shell }} - - name: '${{ matrix.target.os }}-${{ matrix.target.cpu }} (Nim ${{ matrix.branch }})' - runs-on: ${{ matrix.builder }} - continue-on-error: ${{ matrix.branch == 'version-1-6' || matrix.branch == 'devel' }} - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - submodules: true - - - name: MSYS2 (Windows amd64) - if: runner.os == 'Windows' && matrix.target.cpu == 'amd64' - uses: msys2/setup-msys2@v2 - with: - path-type: inherit - install: >- - base-devel - git - mingw-w64-x86_64-toolchain - - - name: Restore Nim DLLs dependencies (Windows) from cache - if: runner.os == 'Windows' - id: windows-dlls-cache - uses: actions/cache@v2 - with: - path: external/dlls - key: 'dlls' - - - name: Install DLL dependencies (Windows) - if: > - steps.windows-dlls-cache.outputs.cache-hit != 'true' && - runner.os == 'Windows' - run: | - mkdir external - curl -L "https://nim-lang.org/download/windeps.zip" -o external/windeps.zip - 7z x external/windeps.zip -oexternal/dlls - - - name: Path to cached dependencies (Windows) - if: > - runner.os == 'Windows' - run: | - echo '${{ github.workspace }}'"/external/dlls" >> $GITHUB_PATH - - - name: Derive environment variables - run: | - if [[ '${{ matrix.target.cpu }}' == 'amd64' ]]; then - PLATFORM=x64 - else - PLATFORM=x86 - fi - echo "PLATFORM=$PLATFORM" >> $GITHUB_ENV - - ncpu= - MAKE_CMD="make" - case '${{ runner.os }}' in - 'Linux') - ncpu=$(nproc) - ;; - 'macOS') - ncpu=$(sysctl -n hw.ncpu) - ;; - 'Windows') - ncpu=$NUMBER_OF_PROCESSORS - MAKE_CMD="mingw32-make" - ;; - esac - [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1 - echo "ncpu=$ncpu" >> $GITHUB_ENV - echo "MAKE_CMD=${MAKE_CMD}" >> $GITHUB_ENV - - - uses: jiro4989/setup-nim-action@v1 - with: - nim-version: 1.6.14 - repo-token: ${{ secrets.GITHUB_TOKEN }} - - - name: Restore Nim toolchain binaries from cache - id: nim-cache - uses: actions/cache@v3 - with: - path: NimBinaries - key: ${{ matrix.target.os }}-${{ matrix.target.cpu }}-nim-${{ hashFiles('atlas.lock') }} - - - name: Restore Vendor Clones from cache - id: vendor-cache - uses: actions/cache@v3 - with: - path: vendor/*/ - key: ${{ matrix.target.os }}-${{ matrix.target.cpu }}-vendor-${{ hashFiles('atlas.lock') }} - - - name: Run tests - run: | - if [[ "${{ matrix.target.os }}" == "windows" ]]; then - # https://github.com/status-im/nimbus-eth2/issues/3121 - export NIMFLAGS="-d:nimRawSetjmp" - fi - - echo "BUILD: " - export NIM_COMMIT=${{ matrix.branch }} - make -j${ncpu} CI_CACHE=NimBinaries ARCH_OVERRIDE=${PLATFORM} QUICK_AND_DIRTY_COMPILER=1 - make test -j${ncpu} diff --git a/.gitignore b/.gitignore index 49331ec..9eaaf60 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ nimbus-build-system.paths vendor/* NimBinaries .update.timestamp +*.dSYM diff --git a/Makefile b/Makefile deleted file mode 100644 index 384bad5..0000000 --- a/Makefile +++ /dev/null @@ -1,71 +0,0 @@ -# Copyright (c) 2020 Status Research & Development GmbH. Licensed under -# either of: -# - Apache License, version 2.0 -# - MIT license -# at your option. This file may not be copied, modified, or distributed except -# according to those terms. - -SHELL := bash # the shell used internally by Make - -# used inside the included makefiles -BUILD_SYSTEM_DIR := vendor/nimbus-build-system - -# -d:insecure - Necessary to enable Prometheus HTTP endpoint for metrics -# -d:chronicles_colors:none - Necessary to disable colors in logs for Docker -DOCKER_IMAGE_NIM_PARAMS ?= -d:chronicles_colors:none -d:insecure - -LINK_PCRE := 0 - -# we don't want an error here, so we can handle things later, in the ".DEFAULT" target --include $(BUILD_SYSTEM_DIR)/makefiles/variables.mk - -.PHONY: \ - all \ - clean \ - coverage \ - deps \ - libbacktrace \ - test \ - update - -ifeq ($(NIM_PARAMS),) -# "variables.mk" was not included, so we update the submodules. -GIT_SUBMODULE_UPDATE := nimble install https://github.com/nim-lang/atlas@\#2ab291c9f37e9d7acce906d1bb7fa49f57f10b22 && atlas rep --noexec atlas.lock -.DEFAULT: - +@ echo -e "Git submodules not found. Running '$(GIT_SUBMODULE_UPDATE)'.\n"; \ - $(GIT_SUBMODULE_UPDATE); \ - echo -# Now that the included *.mk files appeared, and are newer than this file, Make will restart itself: -# https://www.gnu.org/software/make/manual/make.html#Remaking-Makefiles -# -# After restarting, it will execute its original goal, so we don't have to start a child Make here -# with "$(MAKE) $(MAKECMDGOALS)". Isn't hidden control flow great? - -else # "variables.mk" was included. Business as usual until the end of this file. - -# default target, because it's the first one that doesn't start with '.' - -# Builds the codex binary -all: | build deps - echo -e $(BUILD_MSG) "$@" && \ - $(ENV_SCRIPT) nim test $(NIM_PARAMS) - -# must be included after the default target --include $(BUILD_SYSTEM_DIR)/makefiles/targets.mk - -deps: | deps-common nat-libs - -#- deletes and recreates "codexdht.nims" which on Windows is a copy instead of a proper symlink -update: | update-common - rm -rf codexdht.nims && \ - $(MAKE) codexdht.nims $(HANDLE_OUTPUT) - -# Builds and run a part of the test suite -test: | build deps - echo -e $(BUILD_MSG) "$@" && \ - $(ENV_SCRIPT) nim test $(NIM_PARAMS) config.nims - -# usual cleaning -clean: | clean-common - -endif # "variables.mk" was not included diff --git a/README.md b/README.md index 780f686..805008a 100644 --- a/README.md +++ b/README.md @@ -19,3 +19,26 @@ This DHT implementation is aiming to provide a DHT for Codex with the following Current implementation is based on nim-eth's Discovery v5 implementation. Base files were copied from [`status-im/nim-eth@779d767b024175a51cf74c79ec7513301ebe2f46`](https://github.com/status-im/nim-eth/commit/779d767b024175a51cf74c79ec7513301ebe2f46) + +## Building + +This repo is setup to use Nimble lockfiles. This requires Nimble 0.14+ which isn't installed by default when this was written. If `nimble -v` reports `0.13.x` then you will need to install Nimble 0.14. Note that using Nimble 0.14 changes how Nimble behaves! + +Nimble 0.14 can be install by: + +```sh +nimble install nimble@0.14.2 +``` + +After this you can setup your Nimble environment. Note that this will build the pinned version of Nim! The first run can take ~15 minutes. + +```sh +nimble setup # creates a nimble.paths used for rest of Nimble commands +nimble testAll +``` + +You can also run tasks directly: + +```sh +nim testAll +``` diff --git a/build.nims b/build.nims new file mode 100644 index 0000000..164c795 --- /dev/null +++ b/build.nims @@ -0,0 +1,58 @@ +import std / [os, strutils, sequtils] + +switch("define", "libp2p_pki_schemes=secp256k1") + +task testAll, "Run DHT tests": + exec "nim c -r tests/testAll.nim" + +task test, "Run DHT tests": + exec "nim c -r -d:testsAll --verbosity:0 tests/testAllParallel.nim" + +task testPart1, "Run DHT tests A": + exec "nim c -r -d:testsPart1 tests/testAllParallel.nim" + +task testPart2, "Run DHT tests B": + exec "nim c -r -d:testsPart2 tests/testAllParallel.nim" + +task coverage, "generates code coverage report": + var (output, exitCode) = gorgeEx("which lcov") + if exitCode != 0: + echo "" + echo " ************************** ⛔️ ERROR ⛔️ **************************" + echo " ** **" + echo " ** ERROR: lcov not found, it must be installed to run code **" + echo " ** coverage locally **" + echo " ** **" + echo " *****************************************************************" + echo "" + quit 1 + + (output, exitCode) = gorgeEx("gcov --version") + if output.contains("Apple LLVM"): + echo "" + echo " ************************* ⚠️ WARNING ⚠️ *************************" + echo " ** **" + echo " ** WARNING: Using Apple's llvm-cov in place of gcov, which **" + echo " ** emulates an old version of gcov (4.2.0) and therefore **" + echo " ** coverage results will differ than those on CI (which **" + echo " ** uses a much newer version of gcov). **" + echo " ** **" + echo " *****************************************************************" + echo "" + + var nimSrcs = "" + for f in walkDirRec(".", {pcFile}): + if f.endswith(".nim"): nimSrcs.add " " & f.quoteShell() + + echo "======== Running Tests ======== " + exec("nim c -r tests/coverage.nim") + # exec("rm nimcache/*.c") + rmDir("coverage"); mkDir("coverage") + echo " ======== Running LCOV ======== " + exec("lcov --capture --directory nimcache --output-file coverage/coverage.info") + exec("lcov --extract coverage/coverage.info --output-file coverage/coverage.f.info " & nimSrcs) + echo " ======== Generating HTML coverage report ======== " + exec("genhtml coverage/coverage.f.info --output-directory coverage/report ") + echo " ======== Opening HTML coverage report in browser... ======== " + exec("open coverage/report/index.html") + diff --git a/codexdht.nimble b/codexdht.nimble index 00d943a..25df1ef 100644 --- a/codexdht.nimble +++ b/codexdht.nimble @@ -23,52 +23,6 @@ requires "asynctest >= 0.3.1 & < 0.4.0" requires "https://github.com/status-im/nim-datastore#head" requires "questionable" -task testAll, "Run DHT tests": - exec "nim c -r tests/testAll.nim" +include "build.nims" -task test, "Run DHT tests": - exec "nim c -r -d:testsAll --verbosity:0 tests/testAllParallel.nim" - -task testPart1, "Run DHT tests A": - exec "nim c -r -d:testsPart1 tests/testAllParallel.nim" - -task testPart2, "Run DHT tests B": - exec "nim c -r -d:testsPart2 tests/testAllParallel.nim" - -# task coverage, "generates code coverage report": -# var (output, exitCode) = gorgeEx("which lcov") -# if exitCode != 0: -# echo "" -# echo " ************************** ⛔️ ERROR ⛔️ **************************" -# echo " ** **" -# echo " ** ERROR: lcov not found, it must be installed to run code **" -# echo " ** coverage locally **" -# echo " ** **" -# echo " *****************************************************************" -# echo "" -# quit 1 - -# (output, exitCode) = gorgeEx("gcov --version") -# if output.contains("Apple LLVM"): -# echo "" -# echo " ************************* ⚠️ WARNING ⚠️ *************************" -# echo " ** **" -# echo " ** WARNING: Using Apple's llvm-cov in place of gcov, which **" -# echo " ** emulates an old version of gcov (4.2.0) and therefore **" -# echo " ** coverage results will differ than those on CI (which **" -# echo " ** uses a much newer version of gcov). **" -# echo " ** **" -# echo " *****************************************************************" -# echo "" - -# exec("nimble --verbose test --opt:speed -d:debug --verbosity:0 --hints:off --lineDir:on -d:chronicles_log_level=INFO --nimcache:nimcache --passC:-fprofile-arcs --passC:-ftest-coverage --passL:-fprofile-arcs --passL:-ftest-coverage") -# exec("cd nimcache; rm *.c; cd ..") -# mkDir("coverage") -# exec("lcov --capture --directory nimcache --output-file coverage/coverage.info") -# exec("$(which bash) -c 'shopt -s globstar; ls $(pwd)/codexdht/{*,**/*}.nim'") -# exec("$(which bash) -c 'shopt -s globstar; lcov --extract coverage/coverage.info $(pwd)/codexdht/{*,**/*}.nim --output-file coverage/coverage.f.info'") -# echo "Generating HTML coverage report" -# exec("genhtml coverage/coverage.f.info --output-directory coverage/report") -# echo "Opening HTML coverage report in browser..." -# exec("open coverage/report/index.html") diff --git a/config.nims b/config.nims index 13a33d5..831d5cd 100644 --- a/config.nims +++ b/config.nims @@ -1,31 +1,5 @@ -import std/os -const currentDir = currentSourcePath()[0 .. ^(len("config.nims") + 1)] - -switch("define", "libp2p_pki_schemes=secp256k1") - -task testAll, "Run DHT tests": - exec "nim c -r tests/testAll.nim" - -task test, "Run DHT tests": - exec "nim c -r -d:testsAll --verbosity:0 tests/testAllParallel.nim" - -task testPart1, "Run DHT tests A": - exec "nim c -r -d:testsPart1 tests/testAllParallel.nim" - -task testPart2, "Run DHT tests B": - exec "nim c -r -d:testsPart2 tests/testAllParallel.nim" - -when getEnv("NIMBUS_BUILD_SYSTEM") == "yes" and - # BEWARE - # In Nim 1.6, config files are evaluated with a working directory - # matching where the Nim command was invocated. This means that we - # must do all file existance checks with full absolute paths: - system.fileExists(currentDir & "nimbus-build-system.paths"): - echo "Using Nimbus Paths" - include "nimbus-build-system.paths" -elif withDir(thisDir(), system.fileExists("nimble.paths")): - echo "Using Nimble Paths" +include "build.nims" # begin Nimble config (version 2) --noNimblePath diff --git a/env.sh b/env.sh deleted file mode 100755 index 697a426..0000000 --- a/env.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash - -# We use ${BASH_SOURCE[0]} instead of $0 to allow sourcing this file -# and we fall back to a Zsh-specific special var to also support Zsh. -REL_PATH="$(dirname ${BASH_SOURCE[0]:-${(%):-%x}})" -ABS_PATH="$(cd ${REL_PATH}; pwd)" -source ${ABS_PATH}/vendor/nimbus-build-system/scripts/env.sh diff --git a/tests/coverage.nim b/tests/coverage.nim new file mode 100644 index 0000000..e9146c9 --- /dev/null +++ b/tests/coverage.nim @@ -0,0 +1,2 @@ + +include ./testAll diff --git a/tests/coverage.nims b/tests/coverage.nims new file mode 100644 index 0000000..f14fce4 --- /dev/null +++ b/tests/coverage.nims @@ -0,0 +1,15 @@ +switch("define", "testsAll") + +switch("debugger", "native") +switch("lineDir", "on") +switch("define", "debug") +# switch("opt", "none") +switch("verbosity", "0") +switch("hints", "off") +switch("warnings", "off") +switch("define", "chronicles_log_level=INFO") +switch("nimcache", "nimcache") +switch("passC", "-fprofile-arcs") +switch("passC", "-ftest-coverage") +switch("passL", "-fprofile-arcs") +switch("passL", "-ftest-coverage") diff --git a/tests/testAll.nim b/tests/testAll.nim index f9fc17c..af432e0 100644 --- a/tests/testAll.nim +++ b/tests/testAll.nim @@ -1,5 +1,6 @@ -import - ./dht/[test_providers, test_providermngr], - ./discv5/[test_discoveryv5, test_discoveryv5_encoding] +import ./dht/test_providers +import ./dht/test_providermngr +import ./discv5/test_discoveryv5 +import ./discv5/test_discoveryv5_encoding {.warning[UnusedImport]: off.} diff --git a/tests/testAllParallel.nim b/tests/testAllParallel.nim index 9127064..7bff2e1 100644 --- a/tests/testAllParallel.nim +++ b/tests/testAllParallel.nim @@ -8,15 +8,15 @@ var cmds: seq[string] when defined(testsPart1) or defined(testsAll): cmds.add [ - "nim c -r --verbosity:0 tests/dht/test_providers.nim", - "nim c -r --verbosity:0 tests/dht/test_providermngr.nim", + "nim c -r tests/dht/test_providers.nim", + "nim c -r tests/dht/test_providermngr.nim", ] when defined(testsPart2) or defined(testsAll): cmds.add [ - "nim c -r --verbosity:0 tests/discv5/test_discoveryv5.nim", - "nim c -r --verbosity:0 tests/discv5/test_discoveryv5_encoding.nim", + "nim c -r tests/discv5/test_discoveryv5.nim", + "nim c -r tests/discv5/test_discoveryv5_encoding.nim", ] -echo "CMDS: ", cmds +echo "Running Test Commands: ", cmds quit execProcesses(cmds)