From e72d03bc78d3bc896ae5912ab45e2ecd53849aa5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C8=98tefan=20Talpalaru?= Date: Mon, 10 Jan 2022 12:29:52 +0100 Subject: [PATCH] CI: test with multiple Nim versions by default (#684) * CI: test with 1.2 & devel by default * Skip buggy windows websocket test --- .github/workflows/ci.yml | 150 +++++++++++++++----------------- .github/workflows/multi_nim.yml | 145 ++++++++++++++++-------------- libp2p.nimble | 5 +- tests/commontransport.nim | 4 + tests/testswitch.nim | 4 + 5 files changed, 161 insertions(+), 147 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c6d0bcc2e..a02312e29 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,78 +9,49 @@ on: jobs: build: + timeout-minutes: 90 strategy: fail-fast: false - max-parallel: 20 matrix: target: - # Unit tests - os: linux cpu: amd64 - os: linux cpu: i386 - os: macos cpu: amd64 - - os: windows - cpu: i386 - os: windows cpu: amd64 + #- os: windows + #cpu: i386 + branch: [version-1-2, devel] include: - target: os: linux builder: ubuntu-20.04 + shell: bash - target: os: macos builder: macos-10.15 + shell: bash - target: os: windows builder: windows-2019 + shell: msys2 {0} defaults: run: - shell: bash + shell: ${{ matrix.shell }} - name: '${{ matrix.target.os }}-${{ matrix.target.cpu }}' + name: '${{ matrix.target.os }}-${{ matrix.target.cpu }} (Nim ${{ matrix.branch }})' runs-on: ${{ matrix.builder }} + continue-on-error: ${{ matrix.branch == 'version-1-6' || matrix.branch == 'devel' }} steps: - - name: Checkout nim-libp2p + - name: Checkout uses: actions/checkout@v2 with: submodules: true - - name: Derive environment variables - run: | - if [[ '${{ matrix.target.cpu }}' == 'amd64' ]]; then - ARCH=64 - PLATFORM=x64 - else - ARCH=32 - PLATFORM=x86 - fi - echo "ARCH=$ARCH" >> $GITHUB_ENV - echo "PLATFORM=$PLATFORM" >> $GITHUB_ENV - - ncpu= - ext= - MAKE_CMD="make" - case '${{ runner.os }}' in - 'Linux') - ncpu=$(nproc) - ;; - 'macOS') - ncpu=$(sysctl -n hw.ncpu) - ;; - 'Windows') - ncpu=$NUMBER_OF_PROCESSORS - ext=.exe - MAKE_CMD="mingw32-make" - ;; - esac - [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1 - echo "ncpu=$ncpu" >> $GITHUB_ENV - echo "ext=$ext" >> $GITHUB_ENV - echo "MAKE_CMD=${MAKE_CMD}" >> $GITHUB_ENV - - name: Install build dependencies (Linux i386) if: runner.os == 'Linux' && matrix.target.cpu == 'i386' run: | @@ -101,68 +72,83 @@ jobs: chmod 755 external/bin/gcc external/bin/g++ echo '${{ github.workspace }}/external/bin' >> $GITHUB_PATH - - name: Restore MinGW-W64 (Windows) from cache - if: runner.os == 'Windows' - id: windows-mingw-cache - uses: actions/cache@v2 + - name: MSYS2 (Windows i386) + if: runner.os == 'Windows' && matrix.target.cpu == 'i386' + uses: msys2/setup-msys2@v2 with: - path: external/mingw-${{ matrix.target.cpu }} - key: 'mingw-${{ matrix.target.cpu }}' + path-type: inherit + msystem: MINGW32 + install: >- + base-devel + git + mingw-w64-i686-toolchain + + - name: MSYS2 (Windows amd64) + if: runner.os == 'Windows' && matrix.target.cpu == 'amd64' + uses: msys2/setup-msys2@v2 + with: + path-type: inherit + install: >- + base-devel + git + mingw-w64-x86_64-toolchain - name: Restore Nim DLLs dependencies (Windows) from cache if: runner.os == 'Windows' id: windows-dlls-cache uses: actions/cache@v2 with: - path: external/dlls-${{ matrix.target.cpu }} - key: 'dlls-${{ matrix.target.cpu }}' + path: external/dlls + key: 'dlls' - - name: Install MinGW64 dependency (Windows) - if: > - steps.windows-mingw-cache.outputs.cache-hit != 'true' && - runner.os == 'Windows' - run: | - mkdir -p external - curl -L "https://nim-lang.org/download/mingw$ARCH.7z" -o "external/mingw-${{ matrix.target.cpu }}.7z" - 7z x -y "external/mingw-${{ matrix.target.cpu }}.7z" -oexternal/ - mv external/mingw$ARCH external/mingw-${{ matrix.target.cpu }} - - - name: Install DLLs dependencies (Windows) + - name: Install DLL dependencies (Windows) if: > steps.windows-dlls-cache.outputs.cache-hit != 'true' && runner.os == 'Windows' run: | - mkdir -p external + mkdir external curl -L "https://nim-lang.org/download/windeps.zip" -o external/windeps.zip - 7z x -y external/windeps.zip -oexternal/dlls-${{ matrix.target.cpu }} + 7z x external/windeps.zip -oexternal/dlls - name: Path to cached dependencies (Windows) if: > runner.os == 'Windows' run: | - echo "${{ github.workspace }}/external/mingw-${{ matrix.target.cpu }}/bin" >> $GITHUB_PATH - echo "${{ github.workspace }}/external/dlls-${{ matrix.target.cpu }}" >> $GITHUB_PATH + echo '${{ github.workspace }}'"/external/dlls" >> $GITHUB_PATH - - name: Get latest Nim commit hash - id: versions + - name: Derive environment variables run: | - getHash() { - git ls-remote "https://github.com/$1" "${2:-HEAD}" | cut -f 1 - } - nbsHash=$(getHash status-im/nimbus-build-system) - echo "::set-output name=nimbus_build_system::$nbsHash" + if [[ '${{ matrix.target.cpu }}' == 'amd64' ]]; then + PLATFORM=x64 + else + PLATFORM=x86 + fi + echo "PLATFORM=$PLATFORM" >> $GITHUB_ENV - - name: Restore prebuilt Nim from cache - id: nim-cache - uses: actions/cache@v2 - with: - path: NimBinaries - key: 'NimBinaries-${{ matrix.target.os }}-${{ matrix.target.cpu }}-${{ steps.versions.outputs.nimbus_build_system }}' + ncpu= + MAKE_CMD="make" + case '${{ runner.os }}' in + 'Linux') + ncpu=$(nproc) + ;; + 'macOS') + ncpu=$(sysctl -n hw.ncpu) + ;; + 'Windows') + ncpu=$NUMBER_OF_PROCESSORS + MAKE_CMD="mingw32-make" + ;; + esac + [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1 + echo "ncpu=$ncpu" >> $GITHUB_ENV + echo "MAKE_CMD=${MAKE_CMD}" >> $GITHUB_ENV - - name: Build Nim and associated tools + - name: Build Nim and Nimble run: | curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_nim.sh - env MAKE="${MAKE_CMD} -j${ncpu}" ARCH_OVERRIDE=${PLATFORM} CC=gcc bash build_nim.sh nim csources dist/nimble NimBinaries + env MAKE="${MAKE_CMD} -j${ncpu}" ARCH_OVERRIDE=${PLATFORM} NIM_COMMIT=${{ matrix.branch }} \ + QUICK_AND_DIRTY_COMPILER=1 QUICK_AND_DIRTY_NIMBLE=1 CC=gcc \ + bash build_nim.sh nim csources dist/nimble NimBinaries echo '${{ github.workspace }}/nim/bin' >> $GITHUB_PATH - name: Setup Go @@ -174,8 +160,14 @@ jobs: run: | V=1 bash scripts/build_p2pd.sh p2pdCache 124530a3 - - name: Run nim-libp2p tests + - name: Run tests run: | + if [[ "${{ matrix.target.os }}" == "windows" ]]; then + # https://github.com/status-im/nimbus-eth2/issues/3121 + export NIMFLAGS="-d:nimRawSetjmp" + fi + nim --version + nimble --version nimble install_pinned nimble test diff --git a/.github/workflows/multi_nim.yml b/.github/workflows/multi_nim.yml index 655ca7e89..5d55afb9b 100644 --- a/.github/workflows/multi_nim.yml +++ b/.github/workflows/multi_nim.yml @@ -6,6 +6,7 @@ on: jobs: build: + timeout-minutes: 120 strategy: fail-fast: false matrix: @@ -16,69 +17,39 @@ jobs: cpu: i386 - os: macos cpu: amd64 - #- os: windows - #cpu: i386 - os: windows cpu: amd64 + #- os: windows + #cpu: i386 branch: [version-1-2, version-1-4, version-1-6, devel] include: - target: os: linux builder: ubuntu-20.04 + shell: bash - target: os: macos builder: macos-10.15 + shell: bash - target: os: windows builder: windows-2019 + shell: msys2 {0} defaults: run: - shell: bash + shell: ${{ matrix.shell }} name: '${{ matrix.target.os }}-${{ matrix.target.cpu }} (Nim ${{ matrix.branch }})' runs-on: ${{ matrix.builder }} continue-on-error: ${{ matrix.branch == 'version-1-6' || matrix.branch == 'devel' }} steps: - - name: Checkout nim-libp2p + - name: Checkout uses: actions/checkout@v2 with: - ref: master + ref: unstable submodules: true - - name: Derive environment variables - run: | - if [[ '${{ matrix.target.cpu }}' == 'amd64' ]]; then - ARCH=64 - PLATFORM=x64 - else - ARCH=32 - PLATFORM=x86 - fi - echo "ARCH=$ARCH" >> $GITHUB_ENV - echo "PLATFORM=$PLATFORM" >> $GITHUB_ENV - - ncpu= - ext= - MAKE_CMD="make" - case '${{ runner.os }}' in - 'Linux') - ncpu=$(nproc) - ;; - 'macOS') - ncpu=$(sysctl -n hw.ncpu) - ;; - 'Windows') - ncpu=$NUMBER_OF_PROCESSORS - ext=.exe - MAKE_CMD="mingw32-make" - ;; - esac - [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1 - echo "ncpu=$ncpu" >> $GITHUB_ENV - echo "ext=$ext" >> $GITHUB_ENV - echo "MAKE_CMD=${MAKE_CMD}" >> $GITHUB_ENV - - name: Install build dependencies (Linux i386) if: runner.os == 'Linux' && matrix.target.cpu == 'i386' run: | @@ -99,47 +70,76 @@ jobs: chmod 755 external/bin/gcc external/bin/g++ echo '${{ github.workspace }}/external/bin' >> $GITHUB_PATH - - name: Restore MinGW-W64 (Windows) from cache - if: runner.os == 'Windows' - id: windows-mingw-cache - uses: actions/cache@v2 + - name: MSYS2 (Windows i386) + if: runner.os == 'Windows' && matrix.target.cpu == 'i386' + uses: msys2/setup-msys2@v2 with: - path: external/mingw-${{ matrix.target.cpu }} - key: 'mingw-${{ matrix.target.cpu }}' + path-type: inherit + msystem: MINGW32 + install: >- + base-devel + git + mingw-w64-i686-toolchain + + - name: MSYS2 (Windows amd64) + if: runner.os == 'Windows' && matrix.target.cpu == 'amd64' + uses: msys2/setup-msys2@v2 + with: + path-type: inherit + install: >- + base-devel + git + mingw-w64-x86_64-toolchain - name: Restore Nim DLLs dependencies (Windows) from cache if: runner.os == 'Windows' id: windows-dlls-cache uses: actions/cache@v2 with: - path: external/dlls-${{ matrix.target.cpu }} - key: 'dlls-${{ matrix.target.cpu }}' + path: external/dlls + key: 'dlls' - - name: Install MinGW64 dependency (Windows) - if: > - steps.windows-mingw-cache.outputs.cache-hit != 'true' && - runner.os == 'Windows' - run: | - mkdir -p external - curl -L "https://nim-lang.org/download/mingw$ARCH.7z" -o "external/mingw-${{ matrix.target.cpu }}.7z" - 7z x -y "external/mingw-${{ matrix.target.cpu }}.7z" -oexternal/ - mv external/mingw$ARCH external/mingw-${{ matrix.target.cpu }} - - - name: Install DLLs dependencies (Windows) + - name: Install DLL dependencies (Windows) if: > steps.windows-dlls-cache.outputs.cache-hit != 'true' && runner.os == 'Windows' run: | - mkdir -p external + mkdir external curl -L "https://nim-lang.org/download/windeps.zip" -o external/windeps.zip - 7z x -y external/windeps.zip -oexternal/dlls-${{ matrix.target.cpu }} + 7z x external/windeps.zip -oexternal/dlls - name: Path to cached dependencies (Windows) if: > runner.os == 'Windows' run: | - echo "${{ github.workspace }}/external/mingw-${{ matrix.target.cpu }}/bin" >> $GITHUB_PATH - echo "${{ github.workspace }}/external/dlls-${{ matrix.target.cpu }}" >> $GITHUB_PATH + echo '${{ github.workspace }}'"/external/dlls" >> $GITHUB_PATH + + - name: Derive environment variables + run: | + if [[ '${{ matrix.target.cpu }}' == 'amd64' ]]; then + PLATFORM=x64 + else + PLATFORM=x86 + fi + echo "PLATFORM=$PLATFORM" >> $GITHUB_ENV + + ncpu= + MAKE_CMD="make" + case '${{ runner.os }}' in + 'Linux') + ncpu=$(nproc) + ;; + 'macOS') + ncpu=$(sysctl -n hw.ncpu) + ;; + 'Windows') + ncpu=$NUMBER_OF_PROCESSORS + MAKE_CMD="mingw32-make" + ;; + esac + [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1 + echo "ncpu=$ncpu" >> $GITHUB_ENV + echo "MAKE_CMD=${MAKE_CMD}" >> $GITHUB_ENV - name: Build Nim and Nimble run: | @@ -149,12 +149,27 @@ jobs: bash build_nim.sh nim csources dist/nimble NimBinaries echo '${{ github.workspace }}/nim/bin' >> $GITHUB_PATH - - name: Run nim-libp2p tests + - name: Setup Go + uses: actions/setup-go@v2 + with: + go-version: '^1.15.5' + + - name: Install p2pd run: | + V=1 bash scripts/build_p2pd.sh p2pdCache 124530a3 + + - name: Run tests + run: | + if [[ "${{ matrix.target.os }}" == "windows" ]]; then + # https://github.com/status-im/nimbus-eth2/issues/3121 + export NIMFLAGS="-d:nimRawSetjmp" + fi + nim --version + nimble --version nimble install -y --depsOnly - nimble test_slim + nimble test if [[ "${{ matrix.branch }}" == "version-1-6" || "${{ matrix.branch }}" == "devel" ]]; then echo -e "\nTesting with '--gc:orc':\n" - export NIMFLAGS="--gc:orc" - nimble test_slim + export NIMFLAGS="${NIMFLAGS} --gc:orc" + nimble test fi diff --git a/libp2p.nimble b/libp2p.nimble index 22cfce873..a5483acab 100644 --- a/libp2p.nimble +++ b/libp2p.nimble @@ -27,9 +27,8 @@ const nimflags = proc runTest(filename: string, verify: bool = true, sign: bool = true, moreoptions: string = "") = - let env_nimflags = getEnv("NIMFLAGS") var excstr = "nim c --opt:speed -d:debug -d:libp2p_agents_metrics -d:libp2p_protobuf_metrics -d:libp2p_network_protocols_metrics " - excstr.add(" " & env_nimflags & " ") + excstr.add(" " & getEnv("NIMFLAGS") & " ") excstr.add(" " & nimflags & " ") excstr.add(" -d:libp2p_pubsub_sign=" & $sign) excstr.add(" -d:libp2p_pubsub_verify=" & $verify) @@ -91,7 +90,7 @@ task test, "Runs the test suite": exec "nimble testfilter" exec "nimble examples_build" -task test_slim, "Runs the test suite": +task test_slim, "Runs the (slimmed down) test suite": exec "nimble testnative" exec "nimble testpubsub_slim" exec "nimble testfilter" diff --git a/tests/commontransport.nim b/tests/commontransport.nim index 76aa2bbb8..532689e04 100644 --- a/tests/commontransport.nim +++ b/tests/commontransport.nim @@ -137,6 +137,10 @@ proc commonTransportTest*(name: string, prov: TransportProvider, ma: string) = await transport1.stop() asyncTest "e2e should allow multiple local addresses": + when defined(windows): + # this randomly locks the Windows CI job + skip() + return let addrs = @[MultiAddress.init(ma).tryGet(), MultiAddress.init(ma).tryGet()] diff --git a/tests/testswitch.nim b/tests/testswitch.nim index 8d21a0f0a..c24c9b89f 100644 --- a/tests/testswitch.nim +++ b/tests/testswitch.nim @@ -841,6 +841,10 @@ suite "Switch": switch2.peerStore.protoBook.get(switch1.peerInfo.peerId) == switch1.peerInfo.protocols.toHashSet() asyncTest "e2e should allow multiple local addresses": + when defined(windows): + # this randomly locks the Windows CI job + skip() + return proc handle(conn: Connection, proto: string) {.async, gcsafe.} = try: let msg = string.fromBytes(await conn.readLp(1024))