diff --git a/.github/workflows/ci-nimbus.yml b/.github/workflows/ci-nimbus.yml deleted file mode 100644 index 7f83914..0000000 --- a/.github/workflows/ci-nimbus.yml +++ /dev/null @@ -1,143 +0,0 @@ -name: CI-nimbus -on: - # push: - # paths: - # - atlas.lock - # - .github/workflows/ci-nimbus.yml - push: - branches: - - main - pull_request: - workflow_dispatch: - -jobs: - build: - timeout-minutes: 90 - strategy: - fail-fast: false - matrix: - target: - - os: linux - cpu: amd64 - # - os: macos - # cpu: amd64 - # - os: windows - # cpu: amd64 - branch: [version-1-6] - include: - - target: - os: linux - builder: ubuntu-20.04 - shell: bash - # - target: - # os: macos - # builder: macos-12 - # shell: bash - # - target: - # os: windows - # builder: windows-2019 - # shell: msys2 {0} - - defaults: - run: - shell: ${{ matrix.shell }} - - name: '${{ matrix.target.os }}-${{ matrix.target.cpu }} (Nim ${{ matrix.branch }})' - runs-on: ${{ matrix.builder }} - continue-on-error: ${{ matrix.branch == 'version-1-6' || matrix.branch == 'devel' }} - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - submodules: true - - - name: MSYS2 (Windows amd64) - if: runner.os == 'Windows' && matrix.target.cpu == 'amd64' - uses: msys2/setup-msys2@v2 - with: - path-type: inherit - install: >- - base-devel - git - mingw-w64-x86_64-toolchain - - - name: Restore Nim DLLs dependencies (Windows) from cache - if: runner.os == 'Windows' - id: windows-dlls-cache - uses: actions/cache@v2 - with: - path: external/dlls - key: 'dlls' - - - name: Install DLL dependencies (Windows) - if: > - steps.windows-dlls-cache.outputs.cache-hit != 'true' && - runner.os == 'Windows' - run: | - mkdir external - curl -L "https://nim-lang.org/download/windeps.zip" -o external/windeps.zip - 7z x external/windeps.zip -oexternal/dlls - - - name: Path to cached dependencies (Windows) - if: > - runner.os == 'Windows' - run: | - echo '${{ github.workspace }}'"/external/dlls" >> $GITHUB_PATH - - - name: Derive environment variables - run: | - if [[ '${{ matrix.target.cpu }}' == 'amd64' ]]; then - PLATFORM=x64 - else - PLATFORM=x86 - fi - echo "PLATFORM=$PLATFORM" >> $GITHUB_ENV - - ncpu= - MAKE_CMD="make" - case '${{ runner.os }}' in - 'Linux') - ncpu=$(nproc) - ;; - 'macOS') - ncpu=$(sysctl -n hw.ncpu) - ;; - 'Windows') - ncpu=$NUMBER_OF_PROCESSORS - MAKE_CMD="mingw32-make" - ;; - esac - [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1 - echo "ncpu=$ncpu" >> $GITHUB_ENV - echo "MAKE_CMD=${MAKE_CMD}" >> $GITHUB_ENV - - - uses: jiro4989/setup-nim-action@v1 - with: - nim-version: 1.6.14 - repo-token: ${{ secrets.GITHUB_TOKEN }} - - - name: Restore Nim toolchain binaries from cache - id: nim-cache - uses: actions/cache@v3 - with: - path: NimBinaries - key: ${{ matrix.target.os }}-${{ matrix.target.cpu }}-nim-${{ hashFiles('atlas.lock') }} - - - name: Restore Vendor Clones from cache - id: vendor-cache - uses: actions/cache@v3 - with: - path: vendor/*/ - key: ${{ matrix.target.os }}-${{ matrix.target.cpu }}-vendor-${{ hashFiles('atlas.lock') }} - - - name: Run tests - run: | - if [[ "${{ matrix.target.os }}" == "windows" ]]; then - # https://github.com/status-im/nimbus-eth2/issues/3121 - export NIMFLAGS="-d:nimRawSetjmp" - fi - - echo "BUILD: " - export NIM_COMMIT=${{ matrix.branch }} - make -j${ncpu} CI_CACHE=NimBinaries ARCH_OVERRIDE=${PLATFORM} QUICK_AND_DIRTY_COMPILER=1 - make test -j${ncpu} diff --git a/.gitignore b/.gitignore index 49331ec..9eaaf60 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ nimbus-build-system.paths vendor/* NimBinaries .update.timestamp +*.dSYM diff --git a/Makefile b/Makefile deleted file mode 100644 index 384bad5..0000000 --- a/Makefile +++ /dev/null @@ -1,71 +0,0 @@ -# Copyright (c) 2020 Status Research & Development GmbH. Licensed under -# either of: -# - Apache License, version 2.0 -# - MIT license -# at your option. This file may not be copied, modified, or distributed except -# according to those terms. - -SHELL := bash # the shell used internally by Make - -# used inside the included makefiles -BUILD_SYSTEM_DIR := vendor/nimbus-build-system - -# -d:insecure - Necessary to enable Prometheus HTTP endpoint for metrics -# -d:chronicles_colors:none - Necessary to disable colors in logs for Docker -DOCKER_IMAGE_NIM_PARAMS ?= -d:chronicles_colors:none -d:insecure - -LINK_PCRE := 0 - -# we don't want an error here, so we can handle things later, in the ".DEFAULT" target --include $(BUILD_SYSTEM_DIR)/makefiles/variables.mk - -.PHONY: \ - all \ - clean \ - coverage \ - deps \ - libbacktrace \ - test \ - update - -ifeq ($(NIM_PARAMS),) -# "variables.mk" was not included, so we update the submodules. -GIT_SUBMODULE_UPDATE := nimble install https://github.com/nim-lang/atlas@\#2ab291c9f37e9d7acce906d1bb7fa49f57f10b22 && atlas rep --noexec atlas.lock -.DEFAULT: - +@ echo -e "Git submodules not found. Running '$(GIT_SUBMODULE_UPDATE)'.\n"; \ - $(GIT_SUBMODULE_UPDATE); \ - echo -# Now that the included *.mk files appeared, and are newer than this file, Make will restart itself: -# https://www.gnu.org/software/make/manual/make.html#Remaking-Makefiles -# -# After restarting, it will execute its original goal, so we don't have to start a child Make here -# with "$(MAKE) $(MAKECMDGOALS)". Isn't hidden control flow great? - -else # "variables.mk" was included. Business as usual until the end of this file. - -# default target, because it's the first one that doesn't start with '.' - -# Builds the codex binary -all: | build deps - echo -e $(BUILD_MSG) "$@" && \ - $(ENV_SCRIPT) nim test $(NIM_PARAMS) - -# must be included after the default target --include $(BUILD_SYSTEM_DIR)/makefiles/targets.mk - -deps: | deps-common nat-libs - -#- deletes and recreates "codexdht.nims" which on Windows is a copy instead of a proper symlink -update: | update-common - rm -rf codexdht.nims && \ - $(MAKE) codexdht.nims $(HANDLE_OUTPUT) - -# Builds and run a part of the test suite -test: | build deps - echo -e $(BUILD_MSG) "$@" && \ - $(ENV_SCRIPT) nim test $(NIM_PARAMS) config.nims - -# usual cleaning -clean: | clean-common - -endif # "variables.mk" was not included diff --git a/README.md b/README.md index 780f686..805008a 100644 --- a/README.md +++ b/README.md @@ -19,3 +19,26 @@ This DHT implementation is aiming to provide a DHT for Codex with the following Current implementation is based on nim-eth's Discovery v5 implementation. Base files were copied from [`status-im/nim-eth@779d767b024175a51cf74c79ec7513301ebe2f46`](https://github.com/status-im/nim-eth/commit/779d767b024175a51cf74c79ec7513301ebe2f46) + +## Building + +This repo is setup to use Nimble lockfiles. This requires Nimble 0.14+ which isn't installed by default when this was written. If `nimble -v` reports `0.13.x` then you will need to install Nimble 0.14. Note that using Nimble 0.14 changes how Nimble behaves! + +Nimble 0.14 can be install by: + +```sh +nimble install nimble@0.14.2 +``` + +After this you can setup your Nimble environment. Note that this will build the pinned version of Nim! The first run can take ~15 minutes. + +```sh +nimble setup # creates a nimble.paths used for rest of Nimble commands +nimble testAll +``` + +You can also run tasks directly: + +```sh +nim testAll +``` diff --git a/build.nims b/build.nims new file mode 100644 index 0000000..164c795 --- /dev/null +++ b/build.nims @@ -0,0 +1,58 @@ +import std / [os, strutils, sequtils] + +switch("define", "libp2p_pki_schemes=secp256k1") + +task testAll, "Run DHT tests": + exec "nim c -r tests/testAll.nim" + +task test, "Run DHT tests": + exec "nim c -r -d:testsAll --verbosity:0 tests/testAllParallel.nim" + +task testPart1, "Run DHT tests A": + exec "nim c -r -d:testsPart1 tests/testAllParallel.nim" + +task testPart2, "Run DHT tests B": + exec "nim c -r -d:testsPart2 tests/testAllParallel.nim" + +task coverage, "generates code coverage report": + var (output, exitCode) = gorgeEx("which lcov") + if exitCode != 0: + echo "" + echo " ************************** ⛔️ ERROR ⛔️ **************************" + echo " ** **" + echo " ** ERROR: lcov not found, it must be installed to run code **" + echo " ** coverage locally **" + echo " ** **" + echo " *****************************************************************" + echo "" + quit 1 + + (output, exitCode) = gorgeEx("gcov --version") + if output.contains("Apple LLVM"): + echo "" + echo " ************************* ⚠️ WARNING ⚠️ *************************" + echo " ** **" + echo " ** WARNING: Using Apple's llvm-cov in place of gcov, which **" + echo " ** emulates an old version of gcov (4.2.0) and therefore **" + echo " ** coverage results will differ than those on CI (which **" + echo " ** uses a much newer version of gcov). **" + echo " ** **" + echo " *****************************************************************" + echo "" + + var nimSrcs = "" + for f in walkDirRec(".", {pcFile}): + if f.endswith(".nim"): nimSrcs.add " " & f.quoteShell() + + echo "======== Running Tests ======== " + exec("nim c -r tests/coverage.nim") + # exec("rm nimcache/*.c") + rmDir("coverage"); mkDir("coverage") + echo " ======== Running LCOV ======== " + exec("lcov --capture --directory nimcache --output-file coverage/coverage.info") + exec("lcov --extract coverage/coverage.info --output-file coverage/coverage.f.info " & nimSrcs) + echo " ======== Generating HTML coverage report ======== " + exec("genhtml coverage/coverage.f.info --output-directory coverage/report ") + echo " ======== Opening HTML coverage report in browser... ======== " + exec("open coverage/report/index.html") + diff --git a/codexdht.nimble b/codexdht.nimble index 00d943a..25df1ef 100644 --- a/codexdht.nimble +++ b/codexdht.nimble @@ -23,52 +23,6 @@ requires "asynctest >= 0.3.1 & < 0.4.0" requires "https://github.com/status-im/nim-datastore#head" requires "questionable" -task testAll, "Run DHT tests": - exec "nim c -r tests/testAll.nim" +include "build.nims" -task test, "Run DHT tests": - exec "nim c -r -d:testsAll --verbosity:0 tests/testAllParallel.nim" - -task testPart1, "Run DHT tests A": - exec "nim c -r -d:testsPart1 tests/testAllParallel.nim" - -task testPart2, "Run DHT tests B": - exec "nim c -r -d:testsPart2 tests/testAllParallel.nim" - -# task coverage, "generates code coverage report": -# var (output, exitCode) = gorgeEx("which lcov") -# if exitCode != 0: -# echo "" -# echo " ************************** ⛔️ ERROR ⛔️ **************************" -# echo " ** **" -# echo " ** ERROR: lcov not found, it must be installed to run code **" -# echo " ** coverage locally **" -# echo " ** **" -# echo " *****************************************************************" -# echo "" -# quit 1 - -# (output, exitCode) = gorgeEx("gcov --version") -# if output.contains("Apple LLVM"): -# echo "" -# echo " ************************* ⚠️ WARNING ⚠️ *************************" -# echo " ** **" -# echo " ** WARNING: Using Apple's llvm-cov in place of gcov, which **" -# echo " ** emulates an old version of gcov (4.2.0) and therefore **" -# echo " ** coverage results will differ than those on CI (which **" -# echo " ** uses a much newer version of gcov). **" -# echo " ** **" -# echo " *****************************************************************" -# echo "" - -# exec("nimble --verbose test --opt:speed -d:debug --verbosity:0 --hints:off --lineDir:on -d:chronicles_log_level=INFO --nimcache:nimcache --passC:-fprofile-arcs --passC:-ftest-coverage --passL:-fprofile-arcs --passL:-ftest-coverage") -# exec("cd nimcache; rm *.c; cd ..") -# mkDir("coverage") -# exec("lcov --capture --directory nimcache --output-file coverage/coverage.info") -# exec("$(which bash) -c 'shopt -s globstar; ls $(pwd)/codexdht/{*,**/*}.nim'") -# exec("$(which bash) -c 'shopt -s globstar; lcov --extract coverage/coverage.info $(pwd)/codexdht/{*,**/*}.nim --output-file coverage/coverage.f.info'") -# echo "Generating HTML coverage report" -# exec("genhtml coverage/coverage.f.info --output-directory coverage/report") -# echo "Opening HTML coverage report in browser..." -# exec("open coverage/report/index.html") diff --git a/config.nims b/config.nims index 13a33d5..831d5cd 100644 --- a/config.nims +++ b/config.nims @@ -1,31 +1,5 @@ -import std/os -const currentDir = currentSourcePath()[0 .. ^(len("config.nims") + 1)] - -switch("define", "libp2p_pki_schemes=secp256k1") - -task testAll, "Run DHT tests": - exec "nim c -r tests/testAll.nim" - -task test, "Run DHT tests": - exec "nim c -r -d:testsAll --verbosity:0 tests/testAllParallel.nim" - -task testPart1, "Run DHT tests A": - exec "nim c -r -d:testsPart1 tests/testAllParallel.nim" - -task testPart2, "Run DHT tests B": - exec "nim c -r -d:testsPart2 tests/testAllParallel.nim" - -when getEnv("NIMBUS_BUILD_SYSTEM") == "yes" and - # BEWARE - # In Nim 1.6, config files are evaluated with a working directory - # matching where the Nim command was invocated. This means that we - # must do all file existance checks with full absolute paths: - system.fileExists(currentDir & "nimbus-build-system.paths"): - echo "Using Nimbus Paths" - include "nimbus-build-system.paths" -elif withDir(thisDir(), system.fileExists("nimble.paths")): - echo "Using Nimble Paths" +include "build.nims" # begin Nimble config (version 2) --noNimblePath diff --git a/env.sh b/env.sh deleted file mode 100755 index 697a426..0000000 --- a/env.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash - -# We use ${BASH_SOURCE[0]} instead of $0 to allow sourcing this file -# and we fall back to a Zsh-specific special var to also support Zsh. -REL_PATH="$(dirname ${BASH_SOURCE[0]:-${(%):-%x}})" -ABS_PATH="$(cd ${REL_PATH}; pwd)" -source ${ABS_PATH}/vendor/nimbus-build-system/scripts/env.sh diff --git a/tests/coverage.nim b/tests/coverage.nim new file mode 100644 index 0000000..e9146c9 --- /dev/null +++ b/tests/coverage.nim @@ -0,0 +1,2 @@ + +include ./testAll diff --git a/tests/coverage.nims b/tests/coverage.nims new file mode 100644 index 0000000..f14fce4 --- /dev/null +++ b/tests/coverage.nims @@ -0,0 +1,15 @@ +switch("define", "testsAll") + +switch("debugger", "native") +switch("lineDir", "on") +switch("define", "debug") +# switch("opt", "none") +switch("verbosity", "0") +switch("hints", "off") +switch("warnings", "off") +switch("define", "chronicles_log_level=INFO") +switch("nimcache", "nimcache") +switch("passC", "-fprofile-arcs") +switch("passC", "-ftest-coverage") +switch("passL", "-fprofile-arcs") +switch("passL", "-ftest-coverage") diff --git a/tests/testAll.nim b/tests/testAll.nim index f9fc17c..af432e0 100644 --- a/tests/testAll.nim +++ b/tests/testAll.nim @@ -1,5 +1,6 @@ -import - ./dht/[test_providers, test_providermngr], - ./discv5/[test_discoveryv5, test_discoveryv5_encoding] +import ./dht/test_providers +import ./dht/test_providermngr +import ./discv5/test_discoveryv5 +import ./discv5/test_discoveryv5_encoding {.warning[UnusedImport]: off.} diff --git a/tests/testAllParallel.nim b/tests/testAllParallel.nim index 9127064..7bff2e1 100644 --- a/tests/testAllParallel.nim +++ b/tests/testAllParallel.nim @@ -8,15 +8,15 @@ var cmds: seq[string] when defined(testsPart1) or defined(testsAll): cmds.add [ - "nim c -r --verbosity:0 tests/dht/test_providers.nim", - "nim c -r --verbosity:0 tests/dht/test_providermngr.nim", + "nim c -r tests/dht/test_providers.nim", + "nim c -r tests/dht/test_providermngr.nim", ] when defined(testsPart2) or defined(testsAll): cmds.add [ - "nim c -r --verbosity:0 tests/discv5/test_discoveryv5.nim", - "nim c -r --verbosity:0 tests/discv5/test_discoveryv5_encoding.nim", + "nim c -r tests/discv5/test_discoveryv5.nim", + "nim c -r tests/discv5/test_discoveryv5_encoding.nim", ] -echo "CMDS: ", cmds +echo "Running Test Commands: ", cmds quit execProcesses(cmds)