diff --git a/.github/workflows/ci-nimbus.yml b/.github/workflows/ci-nimbus.yml index 11fea1b..7f83914 100644 --- a/.github/workflows/ci-nimbus.yml +++ b/.github/workflows/ci-nimbus.yml @@ -1,9 +1,14 @@ name: CI-nimbus on: + # push: + # paths: + # - atlas.lock + # - .github/workflows/ci-nimbus.yml push: - paths: - - atlas.lock - - .github/workflows/ci-nimbus.yml + branches: + - main + pull_request: + workflow_dispatch: jobs: build: @@ -14,28 +19,24 @@ jobs: target: - os: linux cpu: amd64 - # - os: linux - # cpu: i386 - - os: macos - cpu: amd64 - - os: windows - cpu: amd64 - #- os: windows - #cpu: i386 + # - os: macos + # cpu: amd64 + # - os: windows + # cpu: amd64 branch: [version-1-6] include: - target: os: linux builder: ubuntu-20.04 shell: bash - - target: - os: macos - builder: macos-12 - shell: bash - - target: - os: windows - builder: windows-2019 - shell: msys2 {0} + # - target: + # os: macos + # builder: macos-12 + # shell: bash + # - target: + # os: windows + # builder: windows-2019 + # shell: msys2 {0} defaults: run: @@ -50,37 +51,6 @@ jobs: with: submodules: true - - name: Install build dependencies (Linux i386) - if: runner.os == 'Linux' && matrix.target.cpu == 'i386' - run: | - sudo dpkg --add-architecture i386 - sudo apt-get update -qq - sudo DEBIAN_FRONTEND='noninteractive' apt-get install \ - --no-install-recommends -yq gcc-multilib g++-multilib \ - libssl-dev:i386 - mkdir -p external/bin - cat << EOF > external/bin/gcc - #!/bin/bash - exec $(which gcc) -m32 "\$@" - EOF - cat << EOF > external/bin/g++ - #!/bin/bash - exec $(which g++) -m32 "\$@" - EOF - chmod 755 external/bin/gcc external/bin/g++ - echo '${{ github.workspace }}/external/bin' >> $GITHUB_PATH - - - name: MSYS2 (Windows i386) - if: runner.os == 'Windows' && matrix.target.cpu == 'i386' - uses: msys2/setup-msys2@v2 - with: - path-type: inherit - msystem: MINGW32 - install: >- - base-devel - git - mingw-w64-i686-toolchain - - name: MSYS2 (Windows amd64) if: runner.os == 'Windows' && matrix.target.cpu == 'amd64' uses: msys2/setup-msys2@v2 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 54f584c..b9f644c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,12 +15,16 @@ jobs: target: - os: linux cpu: amd64 + tests: all - os: macos cpu: amd64 + tests: all - os: windows cpu: amd64 - #- os: windows - #cpu: i386 + tests: part1 + - os: windows + cpu: amd64 + tests: part2 branch: [version-1-6] include: - target: @@ -33,7 +37,7 @@ jobs: shell: bash - target: os: windows - builder: windows-2019 + builder: windows-latest shell: msys2 {0} defaults: @@ -49,37 +53,6 @@ jobs: with: submodules: true - - name: Install build dependencies (Linux i386) - if: runner.os == 'Linux' && matrix.target.cpu == 'i386' - run: | - sudo dpkg --add-architecture i386 - sudo apt-get update -qq - sudo DEBIAN_FRONTEND='noninteractive' apt-get install \ - --no-install-recommends -yq gcc-multilib g++-multilib \ - libssl-dev:i386 - mkdir -p external/bin - cat << EOF > external/bin/gcc - #!/bin/bash - exec $(which gcc) -m32 "\$@" - EOF - cat << EOF > external/bin/g++ - #!/bin/bash - exec $(which g++) -m32 "\$@" - EOF - chmod 755 external/bin/gcc external/bin/g++ - echo '${{ github.workspace }}/external/bin' >> $GITHUB_PATH - - - name: MSYS2 (Windows i386) - if: runner.os == 'Windows' && matrix.target.cpu == 'i386' - uses: msys2/setup-msys2@v2 - with: - path-type: inherit - msystem: MINGW32 - install: >- - base-devel - git - mingw-w64-i686-toolchain - - name: MSYS2 (Windows amd64) if: runner.os == 'Windows' && matrix.target.cpu == 'amd64' uses: msys2/setup-msys2@v2 @@ -89,7 +62,6 @@ jobs: base-devel git mingw-w64-x86_64-toolchain - - name: Restore Nim DLLs dependencies (Windows) from cache if: runner.os == 'Windows' id: windows-dlls-cache @@ -106,48 +78,21 @@ jobs: mkdir external curl -L "https://nim-lang.org/download/windeps.zip" -o external/windeps.zip 7z x external/windeps.zip -oexternal/dlls - - name: Path to cached dependencies (Windows) if: > runner.os == 'Windows' run: | echo '${{ github.workspace }}'"/external/dlls" >> $GITHUB_PATH - - name: Derive environment variables - run: | - if [[ '${{ matrix.target.cpu }}' == 'amd64' ]]; then - PLATFORM=x64 - else - PLATFORM=x86 - fi - echo "PLATFORM=$PLATFORM" >> $GITHUB_ENV - - ncpu= - MAKE_CMD="make" - case '${{ runner.os }}' in - 'Linux') - ncpu=$(nproc) - ;; - 'macOS') - ncpu=$(sysctl -n hw.ncpu) - ;; - 'Windows') - ncpu=$NUMBER_OF_PROCESSORS - MAKE_CMD="mingw32-make" - ;; - esac - [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1 - echo "ncpu=$ncpu" >> $GITHUB_ENV - echo "MAKE_CMD=${MAKE_CMD}" >> $GITHUB_ENV - + ## Restore nimble deps - name: Restore nimble dependencies from cache id: nimble_deps uses: actions/cache@v3 with: path: | - ~/.nimble/pkgs2 - ~/.nimble/packages_official.json - key: ${{ matrix.target.os }}-${{ matrix.target.cpu }}-nimble-${{ hashFiles('nimble.lock') }} + ~/.nimble + ${{ github.workspace }}/.nimble + key: ${{ matrix.target.os }}-${{ matrix.target.cpu }}-dotnimble-${{ hashFiles('nimble.lock') }} - name: Setup Nimble uses: "./.github/actions/install_nimble" @@ -155,16 +100,35 @@ jobs: os: ${{ matrix.target.os }} cpu: ${{ matrix.target.cpu }} - - name: Run tests + - name: Setup Env + run: | + nimble -v + + - name: Setup Deps + run: | + nimble install -d + nimble setup + + - name: Run tests + if: runner.os != 'Windows' + run: | + nimble test -y + + - name: Run windows tests part1 + if: runner.os == 'Windows' && matrix.target.tests == 'part1' run: | - rm -rf ~/.nimble/ if [[ "${{ matrix.target.os }}" == "windows" ]]; then # https://github.com/status-im/nimbus-eth2/issues/3121 export NIMFLAGS="-d:nimRawSetjmp" fi - nimble test -y - if [[ "${{ matrix.branch }}" == "version-1-6" || "${{ matrix.branch }}" == "devel" ]]; then - echo -e "\nTesting with '--gc:orc':\n" - export NIMFLAGS="${NIMFLAGS} --gc:orc" - nimble test -y - fi; + + nimble testPart1 -y + + - name: Run windows tests part2 + if: runner.os == 'Windows' && matrix.target.tests == 'part2' + run: | + if [[ "${{ matrix.target.os }}" == "windows" ]]; then + export NIMFLAGS="-d:nimRawSetjmp" + fi + + nimble testPart2 -y diff --git a/.gitignore b/.gitignore index 584c0a3..49331ec 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +* +!*.* +!*/ coverage nimcache tests/testAll diff --git a/Makefile b/Makefile index a507610..384bad5 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,7 @@ LINK_PCRE := 0 ifeq ($(NIM_PARAMS),) # "variables.mk" was not included, so we update the submodules. -GIT_SUBMODULE_UPDATE := nimble install https://github.com/elcritch/atlas && atlas rep --noexec atlas.lock +GIT_SUBMODULE_UPDATE := nimble install https://github.com/nim-lang/atlas@\#2ab291c9f37e9d7acce906d1bb7fa49f57f10b22 && atlas rep --noexec atlas.lock .DEFAULT: +@ echo -e "Git submodules not found. Running '$(GIT_SUBMODULE_UPDATE)'.\n"; \ $(GIT_SUBMODULE_UPDATE); \ @@ -63,7 +63,7 @@ update: | update-common # Builds and run a part of the test suite test: | build deps echo -e $(BUILD_MSG) "$@" && \ - $(ENV_SCRIPT) nim testAll $(NIM_PARAMS) config.nims + $(ENV_SCRIPT) nim test $(NIM_PARAMS) config.nims # usual cleaning clean: | clean-common diff --git a/codexdht.nimble b/codexdht.nimble index fb9fa7e..00d943a 100644 --- a/codexdht.nimble +++ b/codexdht.nimble @@ -1,6 +1,6 @@ # Package -version = "0.3.0" +version = "0.3.1" author = "Status Research & Development GmbH" description = "DHT based on the libp2p Kademlia spec" license = "MIT" @@ -26,6 +26,15 @@ requires "questionable" task testAll, "Run DHT tests": exec "nim c -r tests/testAll.nim" +task test, "Run DHT tests": + exec "nim c -r -d:testsAll --verbosity:0 tests/testAllParallel.nim" + +task testPart1, "Run DHT tests A": + exec "nim c -r -d:testsPart1 tests/testAllParallel.nim" + +task testPart2, "Run DHT tests B": + exec "nim c -r -d:testsPart2 tests/testAllParallel.nim" + # task coverage, "generates code coverage report": # var (output, exitCode) = gorgeEx("which lcov") # if exitCode != 0: diff --git a/config.nims b/config.nims index 4d5fef5..13a33d5 100644 --- a/config.nims +++ b/config.nims @@ -8,7 +8,13 @@ task testAll, "Run DHT tests": exec "nim c -r tests/testAll.nim" task test, "Run DHT tests": - testAllTask() + exec "nim c -r -d:testsAll --verbosity:0 tests/testAllParallel.nim" + +task testPart1, "Run DHT tests A": + exec "nim c -r -d:testsPart1 tests/testAllParallel.nim" + +task testPart2, "Run DHT tests B": + exec "nim c -r -d:testsPart2 tests/testAllParallel.nim" when getEnv("NIMBUS_BUILD_SYSTEM") == "yes" and # BEWARE @@ -18,8 +24,11 @@ when getEnv("NIMBUS_BUILD_SYSTEM") == "yes" and system.fileExists(currentDir & "nimbus-build-system.paths"): echo "Using Nimbus Paths" include "nimbus-build-system.paths" -elif fileExists("nimble.paths"): +elif withDir(thisDir(), system.fileExists("nimble.paths")): echo "Using Nimble Paths" - # begin Nimble config (version 1) + +# begin Nimble config (version 2) +--noNimblePath +when withDir(thisDir(), system.fileExists("nimble.paths")): include "nimble.paths" - # end Nimble config +# end Nimble config diff --git a/tests/dht/test_helper.nim b/tests/dht/test_helper.nim index df1fd86..38772ca 100644 --- a/tests/dht/test_helper.nim +++ b/tests/dht/test_helper.nim @@ -53,7 +53,7 @@ proc nodeIdInNodes*(id: NodeId, nodes: openArray[Node]): bool = for n in nodes: if id == n.id: return true -proc generateNode*(privKey: PrivateKey, port: int = 20302, +proc generateNode*(privKey: PrivateKey, port: int, ip: ValidIpAddress = ValidIpAddress.init("127.0.0.1")): Node = let @@ -67,7 +67,7 @@ proc generateNRandomNodes*(rng: ref HmacDrbgContext, n: int): seq[Node] = for i in 1..n: let privKey = PrivateKey.example(rng) - node = privKey.generateNode() + node = privKey.generateNode(port = 20402 + 10*n) res.add(node) res @@ -76,7 +76,7 @@ proc nodeAndPrivKeyAtDistance*(n: Node, rng: var HmacDrbgContext, d: uint32, while true: let privKey = PrivateKey.random(rng).expect("Valid rng for private key") - node = privKey.generateNode(ip = ip) + node = privKey.generateNode(port = 21302 + 10*d.int, ip = ip) if logDistance(n.id, node.id) == d: return (node, privKey) diff --git a/tests/dht/test_providers.nim b/tests/dht/test_providers.nim index dab5174..3f01e52 100644 --- a/tests/dht/test_providers.nim +++ b/tests/dht/test_providers.nim @@ -34,7 +34,7 @@ proc bootstrapNodes( debug "---- STARTING BOOSTRAPS ---" for i in 0.. 0: @@ -53,7 +53,7 @@ proc bootstrapNetwork( bootNodeKey = PrivateKey.fromHex( "a2b50376a79b1a8c8a3296485572bdfbf54708bb46d3c25d73d2723aaaf6a617") .expect("Valid private key hex") - bootNodeAddr = localAddress(20301) + bootNodeAddr = localAddress(25311) bootNode = initDiscoveryNode(rng, bootNodeKey, bootNodeAddr, @[]) # just a shortcut for new and open #waitFor bootNode.bootstrap() # immediate, since no bootnodes are defined above diff --git a/tests/discv5/test_discoveryv5.nim b/tests/discv5/test_discoveryv5.nim index 5be5b39..1255569 100644 --- a/tests/discv5/test_discoveryv5.nim +++ b/tests/discv5/test_discoveryv5.nim @@ -22,13 +22,13 @@ suite "Discovery v5 Tests": pk = PrivateKey.example(rng) targetPk = PrivateKey.example(rng) node = initDiscoveryNode(rng, pk, localAddress(20302)) - targetNode = targetPk.generateNode() + targetNode = targetPk.generateNode(port=26302) check node.addNode(targetNode) for i in 0..<1000: let pk = PrivateKey.example(rng) - discard node.addNode(pk.generateNode()) + discard node.addNode(pk.generateNode(port=27302+i)) let n = node.getNode(targetNode.id) check n.isSome() @@ -265,7 +265,7 @@ suite "Discovery v5 Tests": # Generate 1000 random nodes and add to our main node's routing table for i in 0..<1000: - discard mainNode.addSeenNode(generateNode(PrivateKey.example(rng))) # for testing only! + discard mainNode.addSeenNode(generateNode(PrivateKey.example(rng), port=28302+i)) # for testing only! let neighbours = mainNode.neighbours(mainNode.localNode.id) diff --git a/tests/discv5/test_discoveryv5_encoding.nim b/tests/discv5/test_discoveryv5_encoding.nim index 68bf95c..e2c87bc 100644 --- a/tests/discv5/test_discoveryv5_encoding.nim +++ b/tests/discv5/test_discoveryv5_encoding.nim @@ -275,12 +275,12 @@ suite "Discovery v5.1 Packet Encodings Test Vectors": let enrRecA = SignedPeerRecord.init(1, privKeyA, - some(ValidIpAddress.init("127.0.0.1")), some(Port(9000)), - some(Port(9000))).expect("Properly intialized private key") + some(ValidIpAddress.init("127.0.0.1")), some(Port(9001)), + some(Port(9001))).expect("Properly intialized private key") enrRecB = SignedPeerRecord.init(1, privKeyB, - some(ValidIpAddress.init("127.0.0.1")), some(Port(9000)), - some(Port(9000))).expect("Properly intialized private key") + some(ValidIpAddress.init("127.0.0.1")), some(Port(9001)), + some(Port(9001))).expect("Properly intialized private key") nodeA = newNode(enrRecA).expect("Properly initialized record") nodeB = newNode(enrRecB).expect("Properly initialized record") @@ -508,12 +508,12 @@ suite "Discovery v5.1 Additional Encode/Decode": let enrRecA = SignedPeerRecord.init(1, privKeyA, - some(ValidIpAddress.init("127.0.0.1")), some(Port(9000)), - some(Port(9000))).expect("Properly intialized private key") + some(ValidIpAddress.init("127.0.0.1")), some(Port(9001)), + some(Port(9001))).expect("Properly intialized private key") enrRecB = SignedPeerRecord.init(1, privKeyB, - some(ValidIpAddress.init("127.0.0.1")), some(Port(9000)), - some(Port(9000))).expect("Properly intialized private key") + some(ValidIpAddress.init("127.0.0.1")), some(Port(9001)), + some(Port(9001))).expect("Properly intialized private key") nodeA = newNode(enrRecA).expect("Properly initialized record") nodeB = newNode(enrRecB).expect("Properly initialized record") diff --git a/tests/testAllParallel.nim b/tests/testAllParallel.nim new file mode 100644 index 0000000..9127064 --- /dev/null +++ b/tests/testAllParallel.nim @@ -0,0 +1,22 @@ +# import +# ./dht/[test_providers, test_providermngr], +# ./discv5/[test_discoveryv5, test_discoveryv5_encoding] + +import osproc + +var cmds: seq[string] + +when defined(testsPart1) or defined(testsAll): + cmds.add [ + "nim c -r --verbosity:0 tests/dht/test_providers.nim", + "nim c -r --verbosity:0 tests/dht/test_providermngr.nim", + ] +when defined(testsPart2) or defined(testsAll): + cmds.add [ + "nim c -r --verbosity:0 tests/discv5/test_discoveryv5.nim", + "nim c -r --verbosity:0 tests/discv5/test_discoveryv5_encoding.nim", + ] + +echo "CMDS: ", cmds + +quit execProcesses(cmds)