From ebba0933627710bab1e7cb0ee07819b4543c29a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C8=98tefan=20Talpalaru?= Date: Fri, 25 Feb 2022 09:19:12 +0100 Subject: [PATCH] Nim-1.6 compatibility (#3434) --- .github/workflows/ci.yml | 273 ++++++++++------------------- .github/workflows/cron.yml | 20 ++- .gitmodules | 4 + beacon_chain/eth1/eth1_monitor.nim | 14 +- scripts/compile_nim_program.sh | 14 +- 5 files changed, 122 insertions(+), 203 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9a797975c..46048d43e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,81 +2,48 @@ name: CI on: push: paths-ignore: ['media/**', 'docs/**', '**/*.md'] + branches: + - stable + - testing + - unstable pull_request: paths-ignore: ['media/**', 'docs/**', '**/*.md'] + workflow_dispatch: jobs: build: strategy: fail-fast: false - max-parallel: 20 matrix: target: - # Unit tests - os: linux cpu: amd64 - TEST_KIND: unit-tests - os: linux cpu: i386 - TEST_KIND: unit-tests - os: macos cpu: amd64 - TEST_KIND: unit-tests - os: windows cpu: amd64 - TEST_KIND: unit-tests - # Devel cache corrupted for mingw? missing propidl.h on PR but not push CI - # - os: windows - # cpu: i386 - # TEST_KIND: unit-tests - - # # Minimal integration tests - # - os: linux - # cpu: amd64 - # TEST_KIND: finalization-minimal - # - os: linux - # cpu: i386 - # TEST_KIND: finalization-minimal - # # Requires GNU getopt - # - os: macos - # cpu: amd64 - # TEST_KIND: finalization-minimal - # - os: windows - # cpu: amd64 - # TEST_KIND: finalization-minimal - # # TODO - Bootstrap issue: https://github.com/status-im/nimbus-eth2/issues/1725 - # # - os: windows - # # cpu: i386 - # # TEST_KIND: finalization-minimal - - # Mainnet integration tests - # - os: linux - # cpu: amd64 - # TEST_KIND: finalization-mainnet - # # - os: linux - # # cpu: i386 - # # TEST_KIND: finalization-mainnet - # # - os: macos - # # cpu: amd64 - # # TEST_KIND: finalization-mainnet - # - os: windows - # cpu: amd64 - # TEST_KIND: finalization-mainnet - # # TODO - Bootstrap issue: https://github.com/status-im/nimbus-eth2/issues/1725 - # # - os: windows - # # cpu: i386 - # # TEST_KIND: finalization-mainnet + branch: [version-1-2, version-1-6] include: - target: os: linux builder: ubuntu-18.04 + shell: bash - target: os: macos builder: macos-10.15 + shell: bash - target: os: windows builder: windows-2019 - name: '${{ matrix.target.os }}-${{ matrix.target.cpu }} (${{ matrix.target.TEST_KIND }})' + shell: msys2 {0} + + defaults: + run: + shell: ${{ matrix.shell }} + + name: '${{ matrix.target.os }}-${{ matrix.target.cpu }} (Nim ${{ matrix.branch }})' runs-on: ${{ matrix.builder }} steps: - name: Get branch name @@ -91,65 +58,17 @@ jobs: fi id: get_branch - - name: Cancel Previous Runs (except master/devel) + - name: Cancel Previous Runs (except main branches) if: > - steps.get_branch.outputs.branch_name != 'master' && - steps.get_branch.outputs.branch_name != 'devel' && steps.get_branch.outputs.branch_name != 'stable' && steps.get_branch.outputs.branch_name != 'unstable' && steps.get_branch.outputs.branch_name != 'testing' - uses: styfle/cancel-workflow-action@0.5.0 + uses: styfle/cancel-workflow-action@0.9.1 with: access_token: ${{ github.token }} - - name: Support longpaths (Windows) - if: runner.os == 'Windows' - run: git config --system core.longpaths true - - - name: Checkout nimbus-eth2 + - name: Checkout uses: actions/checkout@v2 - with: - path: nimbus-eth2 - - - name: Derive environment variables - shell: bash - run: | - if [[ '${{ matrix.target.cpu }}' == 'amd64' ]]; then - ARCH=64 - PLATFORM=x64 - else - ARCH=32 - PLATFORM=x86 - fi - echo "ARCH=$ARCH" >> $GITHUB_ENV - echo "PLATFORM=$PLATFORM" >> $GITHUB_ENV - - # Stack usage test on recent enough gcc: - if [[ '${{ runner.os }}' == 'Linux' && '${{ matrix.target.cpu }}' == 'amd64' ]]; then - export NIMFLAGS="${NIMFLAGS} -d:limitStackUsage" - echo "NIMFLAGS=$NIMFLAGS" >> $GITHUB_ENV - fi - - # libminiupnp / natpmp - if [[ '${{ runner.os }}' == 'Linux' && '${{ matrix.target.cpu }}' == 'i386' ]]; then - export CFLAGS="${CFLAGS} -m32 -mno-adx" - echo "CFLAGS=$CFLAGS" >> $GITHUB_ENV - fi - - ncpu= - case '${{ runner.os }}' in - 'Linux') - ncpu=$(nproc) - ;; - 'macOS') - ncpu=$(sysctl -n hw.ncpu) - ;; - 'Windows') - ncpu=$NUMBER_OF_PROCESSORS - ;; - esac - [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1 - echo "ncpu=$ncpu" >> $GITHUB_ENV - name: Install build dependencies (Linux i386) if: runner.os == 'Linux' && matrix.target.cpu == 'i386' @@ -170,154 +89,142 @@ jobs: chmod 755 external/bin/gcc external/bin/g++ echo "${{ github.workspace }}/external/bin" >> $GITHUB_PATH - - name: Restore MinGW-W64 (Windows) from cache - if: runner.os == 'Windows' - id: windows-mingw-cache - uses: actions/cache@v2 + - name: MSYS2 (Windows i386) + if: runner.os == 'Windows' && matrix.target.cpu == 'i386' + uses: msys2/setup-msys2@v2 with: - path: external/mingw-${{ matrix.target.cpu }} - key: 'mingw-${{ matrix.target.cpu }}-cachekey_v2' + path-type: inherit + msystem: MINGW32 + install: >- + base-devel + git + mingw-w64-i686-toolchain + + - name: MSYS2 (Windows amd64) + if: runner.os == 'Windows' && matrix.target.cpu == 'amd64' + uses: msys2/setup-msys2@v2 + with: + path-type: inherit + install: >- + base-devel + git + mingw-w64-x86_64-toolchain - name: Restore Nim DLLs dependencies (Windows) from cache if: runner.os == 'Windows' id: windows-dlls-cache uses: actions/cache@v2 with: - path: external/dlls-${{ matrix.target.cpu }} + path: external/dlls key: 'dlls-${{ matrix.target.cpu }}' - - name: Install MinGW64 dependency (Windows) - if: > - steps.windows-mingw-cache.outputs.cache-hit != 'true' && - runner.os == 'Windows' - shell: bash - run: | - mkdir -p external - curl -L "https://nim-lang.org/download/mingw$ARCH.7z" -o "external/mingw-${{ matrix.target.cpu }}.7z" - 7z x -y "external/mingw-${{ matrix.target.cpu }}.7z" -oexternal/ - mv external/mingw$ARCH external/mingw-${{ matrix.target.cpu }} - - name: Install DLLs dependencies (Windows) if: > steps.windows-dlls-cache.outputs.cache-hit != 'true' && runner.os == 'Windows' - shell: bash run: | mkdir -p external curl -L "https://nim-lang.org/download/windeps.zip" -o external/windeps.zip - 7z x -y external/windeps.zip -oexternal/dlls-${{ matrix.target.cpu }} + 7z x -y external/windeps.zip -oexternal/dlls - name: Path to cached dependencies (Windows) if: > runner.os == 'Windows' - shell: bash run: | - echo "${{ github.workspace }}/external/mingw-${{ matrix.target.cpu }}/bin" >> $GITHUB_PATH - echo "${{ github.workspace }}/external/dlls-${{ matrix.target.cpu }}" >> $GITHUB_PATH + echo "${{ github.workspace }}/external/dlls" >> $GITHUB_PATH + # for miniupnp that runs "wingenminiupnpcstrings.exe" from the current dir + echo "." >> $GITHUB_PATH - - name: Install build dependencies (MacOS) + - name: Install build dependencies (macOS) if: runner.os == 'macOS' - shell: bash run: | brew install gnu-getopt brew link --force gnu-getopt - - name: Get latest nimbus-build-system commit hash - id: versions - shell: bash + - name: Derive environment variables run: | - getHash() { - git ls-remote "https://github.com/$1" "${2:-HEAD}" | cut -f 1 - } - nbsHash=$(getHash status-im/nimbus-build-system) - echo "::set-output name=nimbus_build_system::$nbsHash" + if [[ '${{ matrix.target.cpu }}' == 'amd64' ]]; then + PLATFORM=x64 + else + PLATFORM=x86 + fi + echo "PLATFORM=${PLATFORM}" >> $GITHUB_ENV - - name: Restore prebuilt Nim binaries from cache - id: nim-cache - uses: actions/cache@v2 - with: - path: nimbus-eth2/NimBinaries - key: 'nim-${{ matrix.target.os }}-${{ matrix.target.cpu }}-${{ steps.versions.outputs.nimbus_build_system }}' + # Stack usage test on recent enough gcc: + if [[ '${{ runner.os }}' == 'Linux' && '${{ matrix.target.cpu }}' == 'amd64' ]]; then + export NIMFLAGS="${NIMFLAGS} -d:limitStackUsage" + echo "NIMFLAGS=${NIMFLAGS}" >> $GITHUB_ENV + fi + + # libminiupnp / natpmp + if [[ '${{ runner.os }}' == 'Linux' && '${{ matrix.target.cpu }}' == 'i386' ]]; then + export CFLAGS="${CFLAGS} -m32 -mno-adx" + echo "CFLAGS=${CFLAGS}" >> $GITHUB_ENV + fi + + ncpu="" + case '${{ runner.os }}' in + 'Linux') + ncpu=$(nproc) + ;; + 'macOS') + ncpu=$(sysctl -n hw.ncpu) + ;; + 'Windows') + ncpu=${NUMBER_OF_PROCESSORS} + ;; + esac + [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1 + echo "ncpu=${ncpu}" >> $GITHUB_ENV - name: Build Nim and Nimbus dependencies - shell: bash - working-directory: nimbus-eth2 run: | - if [[ "${{ runner.os }}" == "macOS" ]]; then - ulimit -n 1024 - fi - make -j$ncpu ARCH_OVERRIDE=$PLATFORM CI_CACHE=NimBinaries QUICK_AND_DIRTY_COMPILER=1 update + make -j ${ncpu} NIM_COMMIT=${{ matrix.branch }} ARCH_OVERRIDE=${PLATFORM} QUICK_AND_DIRTY_COMPILER=1 update + ./env.sh nim --version - name: Get latest fixtures commit hash - if: matrix.target.TEST_KIND == 'unit-tests' id: fixtures_version - shell: bash run: | getHash() { git ls-remote "https://github.com/$1" "${2:-HEAD}" | cut -f 1 } fixturesHash=$(getHash status-im/nim-eth2-scenarios) - echo "::set-output name=fixtures::$fixturesHash" + echo "::set-output name=fixtures::${fixturesHash}" - name: Restore Ethereum Foundation fixtures from cache - if: matrix.target.TEST_KIND == 'unit-tests' id: fixtures-cache uses: actions/cache@v2 with: - path: nimbus-eth2/fixturesCache + path: fixturesCache key: 'eth2-scenarios-${{ steps.fixtures_version.outputs.fixtures }}' # Important: even with a cache hit, this should be run # as it symlinks the cached items in their proper place - name: Get the Ethereum Foundation fixtures - if: matrix.target.TEST_KIND == 'unit-tests' - shell: bash - working-directory: nimbus-eth2 run: | scripts/setup_scenarios.sh fixturesCache - name: Smoke test the Beacon Node and Validator Client with all tracing enabled - if: matrix.target.TEST_KIND == 'unit-tests' - shell: bash - working-directory: nimbus-eth2 run: | - if [[ "${{ runner.os }}" == "macOS" ]]; then - ulimit -n 1024 - fi - make -j$ncpu ARCH_OVERRIDE=$PLATFORM LOG_LEVEL=TRACE nimbus_beacon_node nimbus_validator_client + make -j ${ncpu} NIM_COMMIT=${{ matrix.branch }} LOG_LEVEL=TRACE nimbus_beacon_node nimbus_validator_client - - name: Run nimbus-eth2 tests - if: matrix.target.TEST_KIND == 'unit-tests' - shell: bash - working-directory: nimbus-eth2 + - name: Build all tools run: | - if [[ "${{ runner.os }}" == "macOS" ]]; then - ulimit -n 1024 - fi - make -j$ncpu ARCH_OVERRIDE=$PLATFORM DISABLE_TEST_FIXTURES_SCRIPT=1 test + make -j ${ncpu} NIM_COMMIT=${{ matrix.branch }} + # The Windows image runs out of disk space, so make some room + rm -rf nimcache + + - name: Run tests + run: | + make -j ${ncpu} NIM_COMMIT=${{ matrix.branch }} DISABLE_TEST_FIXTURES_SCRIPT=1 test # The upload creates a combined report that gets posted as a comment on the PR # https://github.com/EnricoMi/publish-unit-test-result-action - name: Upload combined results - if: matrix.target.TEST_KIND == 'unit-tests' uses: actions/upload-artifact@v2 with: name: Unit Test Results ${{ matrix.target.os }}-${{ matrix.target.cpu }} - path: nimbus-eth2/build/*.xml - - - name: Run nimbus-eth2 testnet0 (minimal) - if: matrix.target.TEST_KIND == 'finalization-minimal' - shell: bash - working-directory: nimbus-eth2 - run: | - ./scripts/launch_local_testnet.sh --preset minimal --nodes 4 --stop-at-epoch 5 --log-level DEBUG --disable-htop --enable-logtrace --data-dir local_testnet0_data --base-port 9000 --base-rpc-port 7000 --base-metrics-port 8008 --timeout 600 -- --verify-finalization --discv5:no - - - name: Run nimbus-eth2 testnet1 (mainnet) - if: matrix.target.TEST_KIND == 'finalization-mainnet' - shell: bash - working-directory: nimbus-eth2 - run: | - ./scripts/launch_local_testnet.sh --nodes 4 --stop-at-epoch 5 --log-level DEBUG --disable-htop --enable-logtrace --data-dir local_testnet0_data --base-port 9000 --base-rpc-port 7000 --base-metrics-port 8008 --timeout 2400 -- --verify-finalization --discv5:no + path: build/*.xml # https://github.com/EnricoMi/publish-unit-test-result-action event_file: diff --git a/.github/workflows/cron.yml b/.github/workflows/cron.yml index 605f27d7b..b740e7f5e 100644 --- a/.github/workflows/cron.yml +++ b/.github/workflows/cron.yml @@ -9,7 +9,6 @@ jobs: build: strategy: fail-fast: false - max-parallel: 20 matrix: target: - os: linux @@ -45,13 +44,13 @@ jobs: runs-on: ${{ matrix.builder }} continue-on-error: ${{ matrix.branch == 'version-1-6' || matrix.branch == 'devel' }} steps: - - name: Checkout nimbus-eth2 + - name: Checkout if: ${{ github.event_name != 'pull_request' }} uses: actions/checkout@v2 with: ref: unstable - - name: Checkout nimbus-eth2 (pull request) + - name: Checkout (pull request) if: ${{ github.event_name == 'pull_request' }} uses: actions/checkout@v2 @@ -100,7 +99,7 @@ jobs: id: windows-dlls-cache uses: actions/cache@v2 with: - path: external/dlls-${{ matrix.target.cpu }} + path: external/dlls key: 'dlls-${{ matrix.target.cpu }}' - name: Install DLLs dependencies (Windows) @@ -110,13 +109,13 @@ jobs: run: | mkdir -p external curl -L "https://nim-lang.org/download/windeps.zip" -o external/windeps.zip - 7z x -y external/windeps.zip -oexternal/dlls-${{ matrix.target.cpu }} + 7z x -y external/windeps.zip -oexternal/dlls - name: Path to cached dependencies (Windows) if: > runner.os == 'Windows' run: | - echo "${{ github.workspace }}/external/dlls-${{ matrix.target.cpu }}" >> $GITHUB_PATH + echo "${{ github.workspace }}/external/dlls" >> $GITHUB_PATH # for miniupnp that runs "wingenminiupnpcstrings.exe" from the current dir echo "." >> $GITHUB_PATH @@ -135,6 +134,12 @@ jobs: fi echo "PLATFORM=${PLATFORM}" >> $GITHUB_ENV + # Stack usage test on recent enough gcc: + if [[ '${{ runner.os }}' == 'Linux' && '${{ matrix.target.cpu }}' == 'amd64' ]]; then + export NIMFLAGS="${NIMFLAGS} -d:limitStackUsage" + echo "NIMFLAGS=${NIMFLAGS}" >> $GITHUB_ENV + fi + # libminiupnp / natpmp if [[ '${{ runner.os }}' == 'Linux' && '${{ matrix.target.cpu }}' == 'i386' ]]; then export CFLAGS="${CFLAGS} -m32 -mno-adx" @@ -187,7 +192,6 @@ jobs: # The Windows image runs out of disk space, so make some room rm -rf nimcache - - name: Run nimbus-eth2 tests + - name: Run tests run: | make -j ${ncpu} V=1 NIM_COMMIT=${{ matrix.branch }} DISABLE_TEST_FIXTURES_SCRIPT=1 test - diff --git a/.gitmodules b/.gitmodules index 507435e0c..f8965d418 100644 --- a/.gitmodules +++ b/.gitmodules @@ -206,9 +206,13 @@ [submodule "vendor/eth2-networks"] path = vendor/eth2-networks url = https://github.com/eth-clients/eth2-networks.git + ignore = untracked + branch = master [submodule "vendor/nim-taskpools"] path = vendor/nim-taskpools url = https://github.com/status-im/nim-taskpools + ignore = untracked + branch = stable [submodule "vendor/nim-ssz-serialization"] path = vendor/nim-ssz-serialization url = https://github.com/status-im/nim-ssz-serialization.git diff --git a/beacon_chain/eth1/eth1_monitor.nim b/beacon_chain/eth1/eth1_monitor.nim index da71b55be..122590658 100644 --- a/beacon_chain/eth1/eth1_monitor.nim +++ b/beacon_chain/eth1/eth1_monitor.nim @@ -1333,19 +1333,15 @@ proc startEth1Syncing(m: Eth1Monitor, delayBeforeStart: Duration) {.async.} = eth1SyncedTo = targetBlock eth1_synced_head.set eth1SyncedTo.toGaugeValue -proc start(m: Eth1Monitor, delayBeforeStart: Duration) = +proc start(m: Eth1Monitor, delayBeforeStart: Duration) {.gcsafe.} = if m.runFut.isNil: let runFut = m.startEth1Syncing(delayBeforeStart) m.runFut = runFut - runFut.addCallback do (p: pointer): + runFut.addCallback do (p: pointer) {.gcsafe.}: if runFut.failed: - if runFut.error[] of CatchableError: - if runFut == m.runFut: - warn "Eth1 chain monitoring failure, restarting", err = runFut.error.msg - m.state = Failed - else: - fatal "Fatal exception reached", err = runFut.error.msg - quit 1 + if runFut == m.runFut: + warn "Eth1 chain monitoring failure, restarting", err = runFut.error.msg + m.state = Failed safeCancel m.runFut m.start(5.seconds) diff --git a/scripts/compile_nim_program.sh b/scripts/compile_nim_program.sh index 4a4070ed5..aa8163e12 100755 --- a/scripts/compile_nim_program.sh +++ b/scripts/compile_nim_program.sh @@ -12,9 +12,17 @@ shift 2 # verbosity level [[ -z "$V" ]] && V=0 -# According to the Nim compiler, the project name comes from the main source -# file, not the output binary. -PROJECT_NAME="$(basename ${SOURCE%.nim})" +# Nim version (formatted as "{MAJOR}{MINOR}"). +# This weird "sed" invocation is because of macOS. +NIM_VERSION=$(nim --version | head -n1 | sed -E 's/^.* ([0-9])\.([0-9]+).*$/\1\2/') + +# According to old Nim compiler versions, the project name comes from the main +# source file, not the output binary. +if [[ "${NIM_VERSION}" -ge "16" ]]; then + PROJECT_NAME="$(basename ${BINARY%.nim})" +else + PROJECT_NAME="$(basename ${SOURCE%.nim})" +fi # The default nimcache dir is "nimcache/release/${PROJECT_NAME}" which doesn't # allow building different binaries from the same main source file, in