diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..54152c5ef --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,279 @@ +name: Nimbus nim-beacon-chain CI +on: [push, pull_request] + +jobs: + build: + strategy: + fail-fast: false + max-parallel: 20 + matrix: + target: + # Unit tests + - os: linux + cpu: amd64 + TEST_KIND: unit-tests + - os: linux + cpu: i386 + TEST_KIND: unit-tests + - os: macos + cpu: amd64 + TEST_KIND: unit-tests + - os: windows + cpu: amd64 + TEST_KIND: unit-tests + - os: windows + cpu: i386 + TEST_KIND: unit-tests + + # # Minimal integration tests + # - os: linux + # cpu: amd64 + # TEST_KIND: finalization-minimal + # - os: linux + # cpu: i386 + # TEST_KIND: finalization-minimal + # # Requires GNU getopt + # - os: macos + # cpu: amd64 + # TEST_KIND: finalization-minimal + # - os: windows + # cpu: amd64 + # TEST_KIND: finalization-minimal + # # TODO - Bootstrap issue: https://github.com/status-im/nim-beacon-chain/issues/1725 + # # - os: windows + # # cpu: i386 + # # TEST_KIND: finalization-minimal + + # Mainnet integration tests + # - os: linux + # cpu: amd64 + # TEST_KIND: finalization-mainnet + # # - os: linux + # # cpu: i386 + # # TEST_KIND: finalization-mainnet + # # - os: macos + # # cpu: amd64 + # # TEST_KIND: finalization-mainnet + # - os: windows + # cpu: amd64 + # TEST_KIND: finalization-mainnet + # # TODO - Bootstrap issue: https://github.com/status-im/nim-beacon-chain/issues/1725 + # # - os: windows + # # cpu: i386 + # # TEST_KIND: finalization-mainnet + include: + - target: + os: linux + builder: ubuntu-18.04 + - target: + os: macos + builder: macos-10.15 + - target: + os: windows + builder: windows-2019 + name: '${{ matrix.target.os }}-${{ matrix.target.cpu }} (${{ matrix.target.TEST_KIND }})' + runs-on: ${{ matrix.builder }} + steps: + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.5.0 + with: + access_token: ${{ github.token }} + + - name: Checkout nim-beacon-chain + uses: actions/checkout@v2 + with: + path: nim-beacon-chain + + - name: Derive environment variables + shell: bash + run: | + if [[ '${{ matrix.target.cpu }}' == 'amd64' ]]; then + ARCH=64 + PLATFORM=x64 + else + ARCH=32 + PLATFORM=x86 + fi + echo "::set-env name=ARCH::$ARCH" + echo "::set-env name=PLATFORM::$PLATFORM" + + # libminiupnp / natpmp + if [[ '${{ runner.os }}' == 'Linux' && '${{ matrix.target.cpu }}' == 'i386' ]]; then + export CFLAGS="${CFLAGS} -m32" + echo "::set-env name=CFLAGS::$CFLAGS" + fi + + - name: Install build dependencies (Linux i386) + if: runner.os == 'Linux' && matrix.target.cpu == 'i386' + run: | + sudo dpkg --add-architecture i386 + sudo apt-fast update -qq + sudo DEBIAN_FRONTEND='noninteractive' apt-fast install \ + --no-install-recommends -yq gcc-multilib g++-multilib \ + libssl-dev:i386 libpcre3-dev:i386 + mkdir -p external/bin + cat << EOF > external/bin/gcc + #!/bin/bash + exec $(which gcc) -m32 "\$@" + EOF + cat << EOF > external/bin/g++ + #!/bin/bash + exec $(which g++) -m32 "\$@" + EOF + chmod 755 external/bin/gcc external/bin/g++ + echo '::add-path::${{ github.workspace }}/external/bin' + + - name: Restore MinGW-W64 (Windows) from cache + if: runner.os == 'Windows' + id: windows-mingw-cache + uses: actions/cache@v1 + with: + path: /external/mingw-${{ matrix.target.cpu }} + key: 'mingw-${{ matrix.target.cpu }}' + + - name: Restore Nim DLLs dependencies (Windows) from cache + if: runner.os == 'Windows' + id: windows-dlls-cache + uses: actions/cache@v1 + with: + path: /external/dlls-${{ matrix.target.cpu }} + key: 'dlls-${{ matrix.target.cpu }}' + + - name: Install MinGW64 dependency (Windows) + if: > + steps.windows-mingw-cache.outputs.cache-hit != 'true' && + runner.os == 'Windows' + shell: bash + run: | + mkdir -p external + + curl -L "https://nim-lang.org/download/mingw$ARCH-6.3.0.7z" -o "external/mingw-${{ matrix.target.cpu }}.7z" + 7z x -y "external/mingw-${{ matrix.target.cpu }}.7z" -oexternal/ + mv external/mingw$ARCH external/mingw-${{ matrix.target.cpu }} + echo '::add-path::${{ github.workspace }}'"/external/mingw-${{ matrix.target.cpu }}/bin" + + - name: Install DLLs dependencies (Windows) + if: > + steps.windows-dlls-cache.outputs.cache-hit != 'true' && + runner.os == 'Windows' + shell: bash + run: | + mkdir -p external + + curl -L "https://nim-lang.org/download/windeps.zip" -o external/windeps.zip + 7z x -y external/windeps.zip -oexternal/dlls-${{ matrix.target.cpu }} + echo '::add-path::${{ github.workspace }}'"/external/dlls-${{ matrix.target.cpu }}" + + - name: Install build dependencies (MacOS) + if: runner.os == 'macOS' + shell: bash + run: | + brew install gnu-getopt + brew link --force gnu-getopt + + - name: Get latest nimbus-build-system commit hash + id: versions + shell: bash + run: | + getHash() { + git ls-remote "https://github.com/$1" "${2:-HEAD}" | cut -f 1 + } + # nimHash=$(getHash nim-lang/Nim '${{ matrix.branch }}') + # csourcesHash=$(getHash nim-lang/csources) + # echo "::set-output name=nim::$nimHash" + # echo "::set-output name=csources::$csourcesHash" + nbsHash=$(getHash status-im/nimbus-build-system) + echo "::set-output name=nimbus_build_system::$nbsHash" + + - name: Restore prebuilt Nim binaries from cache + id: nim-cache + uses: actions/cache@v1 + with: + path: nim-beacon-chain/NimBinaries + key: 'nim-${{ matrix.target.os }}-${{ matrix.target.cpu }}-${{ steps.versions.outputs.nimbus_build_system }}' + + - name: Build Nim and associated tools + if: steps.nim-cache.outputs.cache-hit != 'true' + shell: bash + working-directory: nim-beacon-chain + run: | + ncpu= + ext= + case '${{ runner.os }}' in + 'Linux') + ncpu=$(nproc) + ;; + 'macOS') + ncpu=$(sysctl -n hw.ncpu) + ;; + 'Windows') + ncpu=$NUMBER_OF_PROCESSORS + ext=.exe + ;; + esac + [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1 + echo "::set-output name=ncpu::$ncpu" + + make -j$ncpu ARCH_OVERRIDE=$PLATFORM CI_CACHE=NimBinaries V=1 update + + # Cached in "windows-dlls-cache" and exported globally + # if [[ '${{ runner.os }}' == 'Windows' ]]; then + # make -j$ncpu ARCH_OVERRIDE=$PLATFORM fetch-dlls + # fi + + - name: Smoke test the Beacon Node and Validator Client with all tracing enabled + if: matrix.target.TEST_KIND == 'unit-tests' + shell: bash + working-directory: nim-beacon-chain + run: | + make -j$ncpu ARCH_OVERRIDE=$PLATFORM LOG_LEVEL=TRACE NIMFLAGS="-d:testnet_servers_image" beacon_node + make -j$ncpu ARCH_OVERRIDE=$PLATFORM LOG_LEVEL=TRACE NIMFLAGS="-d:testnet_servers_image" validator_client + + - name: Get latest fixtures commit hash + if: matrix.target.TEST_KIND == 'unit-tests' + id: fixtures_version + shell: bash + run: | + getHash() { + git ls-remote "https://github.com/$1" "${2:-HEAD}" | cut -f 1 + } + fixturesHash=$(getHash status-im/nim-eth2-scenarios) + echo "::set-output name=fixtures::$fixturesHash" + + - name: Restore Ethereum Foundation fixtures from cache + if: matrix.target.TEST_KIND == 'unit-tests' + id: fixtures-cache + uses: actions/cache@v1 + with: + path: fixturesCache + key: 'scenarios-${{ steps.fixtures_version.outputs.fixtures }}' + + - name: Get the Ethereum Foundation fixtures + if: > + matrix.target.TEST_KIND == 'unit-tests' && + steps.fixtures-cache.outputs.cache-hit != 'true' + shell: bash + working-directory: nim-beacon-chain + run: | + scripts/setup_official_tests.sh fixturesCache + + - name: Run nim-beacon-chain tests + if: matrix.target.TEST_KIND == 'unit-tests' + shell: bash + working-directory: nim-beacon-chain + run: | + make -j$ncpu ARCH_OVERRIDE=$PLATFORM DISABLE_TEST_FIXTURES_SCRIPT=1 test + + - name: Run nim-beacon-chain testnet0 (minimal) + if: matrix.target.TEST_KIND == 'finalization-minimal' + shell: bash + working-directory: nim-beacon-chain + run: | + ./scripts/launch_local_testnet.sh --testnet 0 --nodes 4 --stop-at-epoch 5 --log-level DEBUG --disable-htop --enable-logtrace --data-dir local_testnet0_data --base-port $(( 9000 + EXECUTOR_NUMBER * 100 )) --base-rpc-port $(( 7000 + EXECUTOR_NUMBER * 100 )) --base-metrics-port $(( 8008 + EXECUTOR_NUMBER * 100 )) -- --verify-finalization --discv5:no + + - name: Run nim-beacon-chain testnet1 (mainnet) + if: matrix.target.TEST_KIND == 'finalization-mainnet' + shell: bash + working-directory: nim-beacon-chain + run: | + ./scripts/launch_local_testnet.sh --testnet 1 --nodes 4 --stop-at-epoch 5 --log-level DEBUG --disable-htop --enable-logtrace --data-dir local_testnet0_data --base-port $(( 9000 + EXECUTOR_NUMBER * 100 )) --base-rpc-port $(( 7000 + EXECUTOR_NUMBER * 100 )) --base-metrics-port $(( 8008 + EXECUTOR_NUMBER * 100 )) -- --verify-finalization --discv5:no diff --git a/README.md b/README.md index 011d1123c..a88134cf3 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,7 @@ [![Build Status (Travis)](https://img.shields.io/travis/status-im/nim-beacon-chain/master.svg?label=Linux%20/%20macOS "Linux/macOS build status (Travis)")](https://travis-ci.org/status-im/nim-beacon-chain) [![Build Status (Azure)](https://dev.azure.com/nimbus-dev/nim-beacon-chain/_apis/build/status/status-im.nim-beacon-chain?branchName=master)](https://dev.azure.com/nimbus-dev/nim-beacon-chain/_build/latest?definitionId=3&branchName=master) +[![Github Actions CI](https://github.com/status-im/nim-beacon-chain/workflows/Nimbus%20nim-beacon-chain%20CI/badge.svg)](https://github.com/status-im/nim-blscurve/actions?query=workflow%3A%22BLSCurve+CI%22) [![License: Apache](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT) ![Stability: experimental](https://img.shields.io/badge/stability-experimental-orange.svg) @@ -15,29 +16,31 @@ Nimbus beacon chain is a research implementation of the beacon chain component o -- [Documentation](#documentation) -- [Related](#related) -- [Prerequisites for everyone](#prerequisites-for-everyone) - - [Linux](#linux) - - [MacOS](#macos) - - [Windows](#windows) - - [Android](#android) -- [For users](#for-users) - - [Connecting to testnets](#connecting-to-testnets) - - [Getting metrics from a local testnet client](#getting-metrics-from-a-local-testnet-client) -- [Interop (for other Eth2 clients)](#interop-for-other-eth2-clients) -- [For researchers](#for-researchers) - - [State transition simulation](#state-transition-simulation) - - [Local network simulation](#local-network-simulation) - - [Visualising simulation metrics](#visualising-simulation-metrics) - - [Network inspection](#network-inspection) -- [For developers](#for-developers) - - [Windows dev environment](#windows-dev-environment) - - [Linux, MacOS](#linux-macos) - - [Raspberry Pi](#raspberry-pi) - - [Makefile tips and tricks for developers](#makefile-tips-and-tricks-for-developers) - - [CI setup](#ci-setup) -- [License](#license) +- [Nimbus Eth2 (Beacon Chain)](#nimbus-eth2-beacon-chain) + - [Documentation](#documentation) + - [Related projects](#related-projects) + - [Prerequisites for everyone](#prerequisites-for-everyone) + - [Linux](#linux) + - [MacOS](#macos) + - [Windows](#windows) + - [Android](#android) + - [For users](#for-users) + - [Connecting to testnets](#connecting-to-testnets) + - [Getting metrics from a local testnet client](#getting-metrics-from-a-local-testnet-client) + - [Stress-testing the client by limiting the CPU power](#stress-testing-the-client-by-limiting-the-cpu-power) + - [Interop (for other Eth2 clients)](#interop-for-other-eth2-clients) + - [For researchers](#for-researchers) + - [State transition simulation](#state-transition-simulation) + - [Local network simulation](#local-network-simulation) + - [Visualising simulation metrics](#visualising-simulation-metrics) + - [Network inspection](#network-inspection) + - [For developers](#for-developers) + - [Windows dev environment](#windows-dev-environment) + - [Linux, MacOS](#linux-macos) + - [Raspberry Pi](#raspberry-pi) + - [Makefile tips and tricks for developers](#makefile-tips-and-tricks-for-developers) + - [CI setup](#ci-setup) + - [License](#license) diff --git a/config.nims b/config.nims index aa6248707..a1fa9ffaf 100644 --- a/config.nims +++ b/config.nims @@ -30,7 +30,7 @@ else: if defined(windows): # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=65782 # ("-fno-asynchronous-unwind-tables" breaks Nim's exception raising, sometimes) - switch("passC", "-mno-avx512vl") + switch("passC", "-mno-avx512f") --threads:on --opt:speed diff --git a/vendor/nimbus-build-system b/vendor/nimbus-build-system index 41272d200..79469de15 160000 --- a/vendor/nimbus-build-system +++ b/vendor/nimbus-build-system @@ -1 +1 @@ -Subproject commit 41272d2002a46383265afe20881de8d849eafe01 +Subproject commit 79469de15653d40fc2751f984cf52d0ef6084e7a