codex-storage
/
nim-codex-dht
mirror of https://github.com/codex-storage/nim-codex-dht.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Split windows tests (#70) 

Changes the CI to properly cache Nimble deps *including* Nim binaries. 

* parallelize tests
* speed up tests
* cache whole nimble
* use nimble install without -d
* bump version
* new nimble cache
* fix github_env
* compare speed
* readd msys2
* don't need make for nimble
* ugh renames
This commit is contained in:
Jaremy Creechley 2023-07-13 20:42:43 -07:00 committed by GitHub
parent 18dbe4cc04
commit 7464c8e9a6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 124 additions and 147 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

70
.github/workflows/ci-nimbus.yml vendored
View File

@ -1,9 +1,14 @@
name: CI-nimbus
on:
# push:
# paths:
# - atlas.lock
# - .github/workflows/ci-nimbus.yml
push:
paths:
- atlas.lock
- .github/workflows/ci-nimbus.yml
branches:
- main
pull_request:
workflow_dispatch:
jobs:
build:
@ -14,28 +19,24 @@ jobs:
target:
- os: linux
cpu: amd64
# - os: linux
# cpu: i386
- os: macos
cpu: amd64
- os: windows
cpu: amd64
#- os: windows
#cpu: i386
# - os: macos
# cpu: amd64
# - os: windows
# cpu: amd64
branch: [version-1-6]
include:
- target:
os: linux
builder: ubuntu-20.04
shell: bash
- target:
os: macos
builder: macos-12
shell: bash
- target:
os: windows
builder: windows-2019
shell: msys2 {0}
# - target:
# os: macos
# builder: macos-12
# shell: bash
# - target:
# os: windows
# builder: windows-2019
# shell: msys2 {0}
defaults:
run:
@ -50,37 +51,6 @@ jobs:
with:
submodules: true
- name: Install build dependencies (Linux i386)
if: runner.os == 'Linux' && matrix.target.cpu == 'i386'
run: |
sudo dpkg --add-architecture i386
sudo apt-get update -qq
sudo DEBIAN_FRONTEND='noninteractive' apt-get install \
--no-install-recommends -yq gcc-multilib g++-multilib \
libssl-dev:i386
mkdir -p external/bin
cat << EOF > external/bin/gcc
#!/bin/bash
exec $(which gcc) -m32 "\$@"
EOF
cat << EOF > external/bin/g++
#!/bin/bash
exec $(which g++) -m32 "\$@"
EOF
chmod 755 external/bin/gcc external/bin/g++
echo '${{ github.workspace }}/external/bin' >> $GITHUB_PATH
- name: MSYS2 (Windows i386)
if: runner.os == 'Windows' && matrix.target.cpu == 'i386'
uses: msys2/setup-msys2@v2
with:
path-type: inherit
msystem: MINGW32
install: >-
base-devel
git
mingw-w64-i686-toolchain
- name: MSYS2 (Windows amd64)
if: runner.os == 'Windows' && matrix.target.cpu == 'amd64'
uses: msys2/setup-msys2@v2

112
.github/workflows/ci.yml vendored
View File

@ -15,12 +15,16 @@ jobs:
target:
- os: linux
cpu: amd64
tests: all
- os: macos
cpu: amd64
tests: all
- os: windows
cpu: amd64
#- os: windows
#cpu: i386
tests: part1
- os: windows
cpu: amd64
tests: part2
branch: [version-1-6]
include:
- target:
@ -33,7 +37,7 @@ jobs:
shell: bash
- target:
os: windows
builder: windows-2019
builder: windows-latest
shell: msys2 {0}
defaults:
@ -49,37 +53,6 @@ jobs:
with:
submodules: true
- name: Install build dependencies (Linux i386)
if: runner.os == 'Linux' && matrix.target.cpu == 'i386'
run: |
sudo dpkg --add-architecture i386
sudo apt-get update -qq
sudo DEBIAN_FRONTEND='noninteractive' apt-get install \
--no-install-recommends -yq gcc-multilib g++-multilib \
libssl-dev:i386
mkdir -p external/bin
cat << EOF > external/bin/gcc
#!/bin/bash
exec $(which gcc) -m32 "\$@"
EOF
cat << EOF > external/bin/g++
#!/bin/bash
exec $(which g++) -m32 "\$@"
EOF
chmod 755 external/bin/gcc external/bin/g++
echo '${{ github.workspace }}/external/bin' >> $GITHUB_PATH
- name: MSYS2 (Windows i386)
if: runner.os == 'Windows' && matrix.target.cpu == 'i386'
uses: msys2/setup-msys2@v2
with:
path-type: inherit
msystem: MINGW32
install: >-
base-devel
git
mingw-w64-i686-toolchain
- name: MSYS2 (Windows amd64)
if: runner.os == 'Windows' && matrix.target.cpu == 'amd64'
uses: msys2/setup-msys2@v2
@ -89,7 +62,6 @@ jobs:
base-devel
git
mingw-w64-x86_64-toolchain
- name: Restore Nim DLLs dependencies (Windows) from cache
if: runner.os == 'Windows'
id: windows-dlls-cache
@ -106,48 +78,21 @@ jobs:
mkdir external
curl -L "https://nim-lang.org/download/windeps.zip" -o external/windeps.zip
7z x external/windeps.zip -oexternal/dlls
- name: Path to cached dependencies (Windows)
if: >
runner.os == 'Windows'
run: |
echo '${{ github.workspace }}'"/external/dlls" >> $GITHUB_PATH
- name: Derive environment variables
run: |
if [[ '${{ matrix.target.cpu }}' == 'amd64' ]]; then
PLATFORM=x64
else
PLATFORM=x86
fi
echo "PLATFORM=$PLATFORM" >> $GITHUB_ENV
ncpu=
MAKE_CMD="make"
case '${{ runner.os }}' in
'Linux')
ncpu=$(nproc)
;;
'macOS')
ncpu=$(sysctl -n hw.ncpu)
;;
'Windows')
ncpu=$NUMBER_OF_PROCESSORS
MAKE_CMD="mingw32-make"
;;
esac
[[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
echo "ncpu=$ncpu" >> $GITHUB_ENV
echo "MAKE_CMD=${MAKE_CMD}" >> $GITHUB_ENV
## Restore nimble deps
- name: Restore nimble dependencies from cache
id: nimble_deps
uses: actions/cache@v3
with:
path: |
~/.nimble/pkgs2
~/.nimble/packages_official.json
key: ${{ matrix.target.os }}-${{ matrix.target.cpu }}-nimble-${{ hashFiles('nimble.lock') }}
~/.nimble
${{ github.workspace }}/.nimble
key: ${{ matrix.target.os }}-${{ matrix.target.cpu }}-dotnimble-${{ hashFiles('nimble.lock') }}
- name: Setup Nimble
uses: "./.github/actions/install_nimble"
@ -155,16 +100,35 @@ jobs:
os: ${{ matrix.target.os }}
cpu: ${{ matrix.target.cpu }}
- name: Run tests
- name: Setup Env
run: |
nimble -v
- name: Setup Deps
run: |
nimble install -d
nimble setup
- name: Run tests
if: runner.os != 'Windows'
run: |
nimble test -y
- name: Run windows tests part1
if: runner.os == 'Windows' && matrix.target.tests == 'part1'
run: |
rm -rf ~/.nimble/
if [[ "${{ matrix.target.os }}" == "windows" ]]; then
# https://github.com/status-im/nimbus-eth2/issues/3121
export NIMFLAGS="-d:nimRawSetjmp"
fi
nimble test -y
if [[ "${{ matrix.branch }}" == "version-1-6" || "${{ matrix.branch }}" == "devel" ]]; then
echo -e "\nTesting with '--gc:orc':\n"
export NIMFLAGS="${NIMFLAGS} --gc:orc"
nimble test -y
fi;
nimble testPart1 -y
- name: Run windows tests part2
if: runner.os == 'Windows' && matrix.target.tests == 'part2'
run: |
if [[ "${{ matrix.target.os }}" == "windows" ]]; then
export NIMFLAGS="-d:nimRawSetjmp"
fi
nimble testPart2 -y

3
.gitignore vendored
View File

@ -1,3 +1,6 @@
*
!*.*
!*/
coverage
nimcache
tests/testAll

4
Makefile
View File

@ -30,7 +30,7 @@ LINK_PCRE := 0
ifeq ($(NIM_PARAMS),)
# "variables.mk" was not included, so we update the submodules.
GIT_SUBMODULE_UPDATE := nimble install https://github.com/elcritch/atlas && atlas rep --noexec atlas.lock
GIT_SUBMODULE_UPDATE := nimble install https://github.com/nim-lang/atlas@\#2ab291c9f37e9d7acce906d1bb7fa49f57f10b22 && atlas rep --noexec atlas.lock
.DEFAULT:
+@ echo -e "Git submodules not found. Running '$(GIT_SUBMODULE_UPDATE)'.\n"; \
$(GIT_SUBMODULE_UPDATE); \
@ -63,7 +63,7 @@ update: | update-common
# Builds and run a part of the test suite
test: | build deps
echo -e $(BUILD_MSG) "$@" && \
$(ENV_SCRIPT) nim testAll $(NIM_PARAMS) config.nims
$(ENV_SCRIPT) nim test $(NIM_PARAMS) config.nims
# usual cleaning
clean: | clean-common

11
codexdht.nimble
View File

@ -1,6 +1,6 @@
# Package
version = "0.3.0"
version = "0.3.1"
author = "Status Research & Development GmbH"
description = "DHT based on the libp2p Kademlia spec"
license = "MIT"
@ -26,6 +26,15 @@ requires "questionable"
task testAll, "Run DHT tests":
exec "nim c -r tests/testAll.nim"
task test, "Run DHT tests":
exec "nim c -r -d:testsAll --verbosity:0 tests/testAllParallel.nim"
task testPart1, "Run DHT tests A":
exec "nim c -r -d:testsPart1 tests/testAllParallel.nim"
task testPart2, "Run DHT tests B":
exec "nim c -r -d:testsPart2 tests/testAllParallel.nim"
# task coverage, "generates code coverage report":
# var (output, exitCode) = gorgeEx("which lcov")
# if exitCode != 0:

17
config.nims
View File

@ -8,7 +8,13 @@ task testAll, "Run DHT tests":
exec "nim c -r tests/testAll.nim"
task test, "Run DHT tests":
testAllTask()
exec "nim c -r -d:testsAll --verbosity:0 tests/testAllParallel.nim"
task testPart1, "Run DHT tests A":
exec "nim c -r -d:testsPart1 tests/testAllParallel.nim"
task testPart2, "Run DHT tests B":
exec "nim c -r -d:testsPart2 tests/testAllParallel.nim"
when getEnv("NIMBUS_BUILD_SYSTEM") == "yes" and
# BEWARE
@ -18,8 +24,11 @@ when getEnv("NIMBUS_BUILD_SYSTEM") == "yes" and
system.fileExists(currentDir & "nimbus-build-system.paths"):
echo "Using Nimbus Paths"
include "nimbus-build-system.paths"
elif fileExists("nimble.paths"):
elif withDir(thisDir(), system.fileExists("nimble.paths")):
echo "Using Nimble Paths"
# begin Nimble config (version 1)
# begin Nimble config (version 2)
--noNimblePath
when withDir(thisDir(), system.fileExists("nimble.paths")):
include "nimble.paths"
# end Nimble config
# end Nimble config

6
tests/dht/test_helper.nim
View File

@ -53,7 +53,7 @@ proc nodeIdInNodes*(id: NodeId, nodes: openArray[Node]): bool =
for n in nodes:
if id == n.id: return true
proc generateNode*(privKey: PrivateKey, port: int = 20302,
proc generateNode*(privKey: PrivateKey, port: int,
ip: ValidIpAddress = ValidIpAddress.init("127.0.0.1")): Node =
let
@ -67,7 +67,7 @@ proc generateNRandomNodes*(rng: ref HmacDrbgContext, n: int): seq[Node] =
for i in 1..n:
let
privKey = PrivateKey.example(rng)
node = privKey.generateNode()
node = privKey.generateNode(port = 20402 + 10*n)
res.add(node)
res
@ -76,7 +76,7 @@ proc nodeAndPrivKeyAtDistance*(n: Node, rng: var HmacDrbgContext, d: uint32,
while true:
let
privKey = PrivateKey.random(rng).expect("Valid rng for private key")
node = privKey.generateNode(ip = ip)
node = privKey.generateNode(port = 21302 + 10*d.int, ip = ip)
if logDistance(n.id, node.id) == d:
return (node, privKey)

4
tests/dht/test_providers.nim
View File

@ -34,7 +34,7 @@ proc bootstrapNodes(
debug "---- STARTING BOOSTRAPS ---"
for i in 0..<nodecount:
let privKey = PrivateKey.example(rng)
let node = initDiscoveryNode(rng, privKey, localAddress(20302 + i), bootnodes)
let node = initDiscoveryNode(rng, privKey, localAddress(23302 + i), bootnodes)
await node.start()
result.add((node, privKey))
if delay > 0:
@ -53,7 +53,7 @@ proc bootstrapNetwork(
bootNodeKey = PrivateKey.fromHex(
"a2b50376a79b1a8c8a3296485572bdfbf54708bb46d3c25d73d2723aaaf6a617")
.expect("Valid private key hex")
bootNodeAddr = localAddress(20301)
bootNodeAddr = localAddress(25311)
bootNode = initDiscoveryNode(rng, bootNodeKey, bootNodeAddr, @[]) # just a shortcut for new and open
#waitFor bootNode.bootstrap() # immediate, since no bootnodes are defined above

6
tests/discv5/test_discoveryv5.nim
View File

@ -22,13 +22,13 @@ suite "Discovery v5 Tests":
pk = PrivateKey.example(rng)
targetPk = PrivateKey.example(rng)
node = initDiscoveryNode(rng, pk, localAddress(20302))
targetNode = targetPk.generateNode()
targetNode = targetPk.generateNode(port=26302)
check node.addNode(targetNode)
for i in 0..<1000:
let pk = PrivateKey.example(rng)
discard node.addNode(pk.generateNode())
discard node.addNode(pk.generateNode(port=27302+i))
let n = node.getNode(targetNode.id)
check n.isSome()
@ -265,7 +265,7 @@ suite "Discovery v5 Tests":
# Generate 1000 random nodes and add to our main node's routing table
for i in 0..<1000:
discard mainNode.addSeenNode(generateNode(PrivateKey.example(rng))) # for testing only!
discard mainNode.addSeenNode(generateNode(PrivateKey.example(rng), port=28302+i)) # for testing only!
let
neighbours = mainNode.neighbours(mainNode.localNode.id)

16
tests/discv5/test_discoveryv5_encoding.nim
View File

@ -275,12 +275,12 @@ suite "Discovery v5.1 Packet Encodings Test Vectors":
let
enrRecA = SignedPeerRecord.init(1, privKeyA,
some(ValidIpAddress.init("127.0.0.1")), some(Port(9000)),
some(Port(9000))).expect("Properly intialized private key")
some(ValidIpAddress.init("127.0.0.1")), some(Port(9001)),
some(Port(9001))).expect("Properly intialized private key")
enrRecB = SignedPeerRecord.init(1, privKeyB,
some(ValidIpAddress.init("127.0.0.1")), some(Port(9000)),
some(Port(9000))).expect("Properly intialized private key")
some(ValidIpAddress.init("127.0.0.1")), some(Port(9001)),
some(Port(9001))).expect("Properly intialized private key")
nodeA = newNode(enrRecA).expect("Properly initialized record")
nodeB = newNode(enrRecB).expect("Properly initialized record")
@ -508,12 +508,12 @@ suite "Discovery v5.1 Additional Encode/Decode":
let
enrRecA = SignedPeerRecord.init(1, privKeyA,
some(ValidIpAddress.init("127.0.0.1")), some(Port(9000)),
some(Port(9000))).expect("Properly intialized private key")
some(ValidIpAddress.init("127.0.0.1")), some(Port(9001)),
some(Port(9001))).expect("Properly intialized private key")
enrRecB = SignedPeerRecord.init(1, privKeyB,
some(ValidIpAddress.init("127.0.0.1")), some(Port(9000)),
some(Port(9000))).expect("Properly intialized private key")
some(ValidIpAddress.init("127.0.0.1")), some(Port(9001)),
some(Port(9001))).expect("Properly intialized private key")
nodeA = newNode(enrRecA).expect("Properly initialized record")
nodeB = newNode(enrRecB).expect("Properly initialized record")

22
tests/testAllParallel.nim Normal file
View File

@ -0,0 +1,22 @@
# import
# ./dht/[test_providers, test_providermngr],
# ./discv5/[test_discoveryv5, test_discoveryv5_encoding]
import osproc
var cmds: seq[string]
when defined(testsPart1) or defined(testsAll):
cmds.add [
"nim c -r --verbosity:0 tests/dht/test_providers.nim",
"nim c -r --verbosity:0 tests/dht/test_providermngr.nim",
]
when defined(testsPart2) or defined(testsAll):
cmds.add [
"nim c -r --verbosity:0 tests/discv5/test_discoveryv5.nim",
"nim c -r --verbosity:0 tests/discv5/test_discoveryv5_encoding.nim",
]
echo "CMDS: ", cmds
quit execProcesses(cmds)