diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..adc9ffc --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,171 @@ +name: CI +on: + push: + branches: + - master + pull_request: + workflow_dispatch: + +concurrency: # Cancel stale PR builds (but not push builds) + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} + cancel-in-progress: true + +jobs: + build: + strategy: + fail-fast: false + matrix: + target: + - os: linux + cpu: amd64 + - os: linux + cpu: i386 + - os: macos + cpu: amd64 + - os: windows + cpu: amd64 + #- os: windows + #cpu: i386 + branch: [version-1-6, version-2-0, devel] + include: + - target: + os: linux + builder: ubuntu-20.04 + shell: bash + - target: + os: macos + builder: macos-12 + shell: bash + - target: + os: windows + builder: windows-2019 + shell: msys2 {0} + + defaults: + run: + shell: ${{ matrix.shell }} + + name: '${{ matrix.target.os }}-${{ matrix.target.cpu }} (Nim ${{ matrix.branch }})' + runs-on: ${{ matrix.builder }} + continue-on-error: ${{ matrix.branch == 'devel' }} + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Install build dependencies (Linux i386) + if: runner.os == 'Linux' && matrix.target.cpu == 'i386' + run: | + sudo dpkg --add-architecture i386 + sudo apt-fast update -qq + sudo DEBIAN_FRONTEND='noninteractive' apt-fast install \ + --no-install-recommends -yq gcc-multilib g++-multilib \ + libssl-dev:i386 + mkdir -p external/bin + cat << EOF > external/bin/gcc + #!/bin/bash + exec $(which gcc) -m32 "\$@" + EOF + cat << EOF > external/bin/g++ + #!/bin/bash + exec $(which g++) -m32 "\$@" + EOF + chmod 755 external/bin/gcc external/bin/g++ + echo '${{ github.workspace }}/external/bin' >> $GITHUB_PATH + + - name: 'Install dependencies (macOS)' + if: runner.os == 'macOS' && matrix.branch == 'devel' + run: | + brew install openssl@1.1 + ln -s $(brew --prefix)/opt/openssl/lib/libcrypto.1.1.dylib /usr/local/lib + ln -s $(brew --prefix)/opt/openssl/lib/libssl.1.1.dylib /usr/local/lib/ + + - name: MSYS2 (Windows i386) + if: runner.os == 'Windows' && matrix.target.cpu == 'i386' + uses: msys2/setup-msys2@v2 + with: + path-type: inherit + msystem: MINGW32 + install: >- + base-devel + git + mingw-w64-i686-toolchain + + - name: MSYS2 (Windows amd64) + if: runner.os == 'Windows' && matrix.target.cpu == 'amd64' + uses: msys2/setup-msys2@v2 + with: + path-type: inherit + install: >- + base-devel + git + mingw-w64-x86_64-toolchain + + - name: Restore Nim DLLs dependencies (Windows) from cache + if: runner.os == 'Windows' + id: windows-dlls-cache + uses: actions/cache@v3 + with: + path: external/dlls-${{ matrix.target.cpu }} + key: 'dlls-${{ matrix.target.cpu }}' + + - name: Install DLLs dependencies (Windows) + if: > + steps.windows-dlls-cache.outputs.cache-hit != 'true' && + runner.os == 'Windows' + run: | + mkdir -p external + curl -L "https://nim-lang.org/download/windeps.zip" -o external/windeps.zip + 7z x -y external/windeps.zip -oexternal/dlls-${{ matrix.target.cpu }} + + - name: Path to cached dependencies (Windows) + if: > + runner.os == 'Windows' + run: | + echo "${{ github.workspace }}/external/dlls-${{ matrix.target.cpu }}" >> $GITHUB_PATH + + - name: Derive environment variables + run: | + if [[ '${{ matrix.target.cpu }}' == 'amd64' ]]; then + PLATFORM=x64 + else + PLATFORM=x86 + fi + echo "PLATFORM=$PLATFORM" >> $GITHUB_ENV + + ncpu= + MAKE_CMD="make" + case '${{ runner.os }}' in + 'Linux') + ncpu=$(nproc) + ;; + 'macOS') + ncpu=$(sysctl -n hw.ncpu) + ;; + 'Windows') + ncpu=$NUMBER_OF_PROCESSORS + MAKE_CMD="mingw32-make" + ;; + esac + [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1 + echo "ncpu=$ncpu" >> $GITHUB_ENV + echo "MAKE_CMD=${MAKE_CMD}" >> $GITHUB_ENV + + - name: Build Nim and Nimble + run: | + env MAKE="${MAKE_CMD} -j${ncpu}" ARCH_OVERRIDE=${PLATFORM} \ + NIM_COMMIT=${{ matrix.branch }} \ + NIMBLE_COMMIT=a4fc798838ee753f5485dd19afab22e9367eb0e7 \ + QUICK_AND_DIRTY_COMPILER=1 QUICK_AND_DIRTY_NIMBLE=1 CC=gcc \ + scripts/ci/build_nim.sh nim csources dist/nimble-latest NimBinaries + echo '${{ github.workspace }}/nim/bin' >> $GITHUB_PATH + + - name: Run tests + run: | + if [[ "${{ matrix.target.os }}" == "windows" ]]; then + # https://github.com/status-im/nimbus-eth2/issues/3121 + export NIMFLAGS="-d:nimRawSetjmp" + fi + nim --version + nimble --version + nimble build + nimble test \ No newline at end of file diff --git a/scripts/ci/build_nim.sh b/scripts/ci/build_nim.sh new file mode 100644 index 0000000..6a35962 --- /dev/null +++ b/scripts/ci/build_nim.sh @@ -0,0 +1,297 @@ +#!/usr/bin/env bash +# used in Travis CI and AppVeyor scripts + +# Copyright (c) 2018-2020 Status Research & Development GmbH. Licensed under +# either of: +# - Apache License, version 2.0 +# - MIT license +# at your option. This file may not be copied, modified, or distributed except +# according to those terms. + +set -e + +# NIM_COMMIT could be a (partial) commit hash, a tag, a branch name, etc. Empty by default. +NIM_COMMIT_HASH="" # full hash for NIM_COMMIT, retrieved in "nim_needs_rebuilding()" + +# script arguments +[[ $# -ne 4 ]] && { echo "Usage: $0 nim_dir csources_dir nimble_dir ci_cache_dir"; exit 1; } +NIM_DIR="$1" +CSOURCES_DIR="$2" # can be relative to NIM_DIR; only used when `skipIntegrityCheck` unsupported +NIMBLE_DIR="$3" # can be relative to NIM_DIR; only used when `skipIntegrityCheck` unsupported +CI_CACHE="$4" + +## env vars +# verbosity level +[[ -z "$V" ]] && V=0 +[[ -z "$CC" ]] && CC="gcc" +# to build csources in parallel, set MAKE="make -jN" +[[ -z "$MAKE" ]] && MAKE="make" +# for 32-bit binaries on a 64-bit host +UCPU="" +[[ "$ARCH_OVERRIDE" == "x86" ]] && UCPU="ucpu=i686" +[[ -z "$NIM_BUILD_MSG" ]] && NIM_BUILD_MSG="Building the Nim compiler" +[[ -z "$QUICK_AND_DIRTY_COMPILER" ]] && QUICK_AND_DIRTY_COMPILER=0 +[[ -z "$QUICK_AND_DIRTY_NIMBLE" ]] && QUICK_AND_DIRTY_NIMBLE=0 + +# Windows detection +if uname | grep -qiE "mingw|msys"; then + ON_WINDOWS=1 + EXE_SUFFIX=".exe" + # otherwise it fails in AppVeyor due to https://github.com/git-for-windows/git/issues/2495 + GIT_TIMESTAMP_ARG="--date=unix" # available since Git 2.9.4 +else + ON_WINDOWS=0 + EXE_SUFFIX="" + GIT_TIMESTAMP_ARG="--date=format-local:%s" # available since Git 2.7.0 +fi + +NIM_BINARY="${NIM_DIR}/bin/nim${EXE_SUFFIX}" +MAX_NIM_BINARIES="10" # Old ones get deleted. + +nim_needs_rebuilding() { + REBUILD=0 + NO_REBUILD=1 + + if [[ ! -e "$NIM_DIR" ]]; then + # Shallow clone, optimised for the default NIM_COMMIT value. + git clone -q --depth=1 https://github.com/status-im/Nim.git "$NIM_DIR" + fi + + pushd "${NIM_DIR}" >/dev/null + if [[ -n "${NIM_COMMIT}" ]]; then + # support old Git versions, like the one from Ubuntu-18.04 + git restore . 2>/dev/null || git reset --hard + if ! git checkout -q ${NIM_COMMIT} 2>/dev/null; then + # Pay the price for a non-default NIM_COMMIT here, by fetching everything. + # (This includes upstream branches and tags that might be missing from our fork.) + git remote add upstream https://github.com/nim-lang/Nim + git fetch --all --tags --quiet + git checkout -q ${NIM_COMMIT} + fi + # In case the local branch diverged and a fast-forward merge is not possible. + git fetch || true + git reset -q --hard origin/${NIM_COMMIT} 2>/dev/null || true + # In case NIM_COMMIT is a local branch that's behind the remote one it's tracking. + git pull -q 2>/dev/null || true + git checkout -q ${NIM_COMMIT} + # We can't use "rev-parse" here, because it would return the tag object's + # hash instead of the commit hash, when NIM_COMMIT is a tag. + NIM_COMMIT_HASH="$(git rev-list -n 1 ${NIM_COMMIT})" + else + # NIM_COMMIT is empty, so assume the commit we need is already checked out + NIM_COMMIT_HASH="$(git rev-list -n 1 HEAD)" + fi + popd >/dev/null + + if [[ -n "$CI_CACHE" && -d "$CI_CACHE" ]]; then + cp -a "$CI_CACHE"/* "$NIM_DIR"/bin/ || true # let this one fail with an empty cache dir + fi + + # Delete old Nim binaries, to put a limit on how much storage we use. + for F in "$(ls -t "${NIM_DIR}"/bin/nim_commit_* 2>/dev/null | tail -n +$((MAX_NIM_BINARIES + 1)))"; do + if [[ -e "${F}" ]]; then + rm "${F}" + fi + done + + # Compare the last built commit to the one requested. + # Handle the scenario where our symlink is manually deleted by the user. + if [[ -e "${NIM_DIR}/bin/last_built_commit" && \ + -e "${NIM_DIR}/bin/nim${EXE_SUFFIX}" && \ + "$(cat "${NIM_DIR}/bin/last_built_commit")" == "${NIM_COMMIT_HASH}" ]]; then + return $NO_REBUILD + elif [[ -e "${NIM_DIR}/bin/nim_commit_${NIM_COMMIT_HASH}" ]]; then + # we built the requested commit in the past, so we simply reuse it + rm -f "${NIM_DIR}/bin/nim${EXE_SUFFIX}" + ln -s "nim_commit_${NIM_COMMIT_HASH}" "${NIM_DIR}/bin/nim${EXE_SUFFIX}" + echo ${NIM_COMMIT_HASH} > "${NIM_DIR}/bin/last_built_commit" + return $NO_REBUILD + else + return $REBUILD + fi +} + +build_nim() { + echo -e "$NIM_BUILD_MSG" + # [[ "$V" == "0" ]] && exec &>/dev/null + + # working directory + pushd "$NIM_DIR" + if grep -q "skipIntegrityCheck" koch.nim && [ "${NIM_COMMIT}" != "version-1-6" ]; then + echo "in if" + # Run Nim buildchain, with matching dependency versions + # - CSOURCES_REPO from Nim/config/build_config.txt (nim_csourcesUrl) + # - CSOURCES_COMMIT from Nim/config/build_config.txt (nim_csourcesHash) + # - NIMBLE_REPO from Nim/koch.nim (bundleNimbleExe) + # - NIMBLE_COMMIT from Nim/koch.nim (NimbleStableCommit) + . ci/funs.sh + NIMCORES=1 nimBuildCsourcesIfNeeded $UCPU + bin/nim c --noNimblePath --skipUserCfg --skipParentCfg --warnings:off --hints:off koch + ./koch --skipIntegrityCheck boot -d:release --skipUserCfg --skipParentCfg --warnings:off --hints:off + if [[ "${QUICK_AND_DIRTY_COMPILER}" == "0" ]]; then + # We want tools + ./koch tools -d:release --skipUserCfg --skipParentCfg --warnings:off --hints:off + elif [[ "${QUICK_AND_DIRTY_NIMBLE}" != "0" ]]; then + # We just want nimble + ./koch nimble -d:release --skipUserCfg --skipParentCfg --warnings:off --hints:off + fi + else + # Git commits + echo "in else" + + : ${CSOURCES_V1_COMMIT:=561b417c65791cd8356b5f73620914ceff845d10} + : ${CSOURCES_V2_COMMIT:=86742fb02c6606ab01a532a0085784effb2e753e} + : ${CSOURCES_V1_REPO:=https://github.com/nim-lang/csources_v1.git} + : ${CSOURCES_V2_REPO:=https://github.com/nim-lang/csources_v2.git} + + # After this Nim commit, use csources v2 + : ${CSOURCES_V2_START_COMMIT:=f7c203fb6c89b5cef83c4f326aeb23ef8c4a2c40} + : ${NIMBLE_REPO:=https://github.com/nim-lang/nimble.git} + : ${NIMBLE_COMMIT:=a4fc798838ee753f5485dd19afab22e9367eb0e7} # 0.13.1 + + # Custom buildchain for older versions + # TODO Remove this once the default NIM_COMMIT supports `--skipIntegrityCheck` + # We will still be able to compile older versions by removing the flag, + # which will just waste a bit of CPU + + # Git repos for csources and Nimble + if [[ ! -d "$CSOURCES_DIR" ]]; then + if git merge-base --is-ancestor $CSOURCES_V2_START_COMMIT $NIM_COMMIT_HASH; then + CSOURCES_REPO=$CSOURCES_V2_REPO + CSOURCES_COMMIT=$CSOURCES_V2_COMMIT + else + CSOURCES_REPO=$CSOURCES_V1_REPO + CSOURCES_COMMIT=$CSOURCES_V1_COMMIT + fi + + mkdir -p "$CSOURCES_DIR" + pushd "$CSOURCES_DIR" + git clone $CSOURCES_REPO . + git checkout $CSOURCES_COMMIT + popd + fi + if [[ "$CSOURCES_DIR" != "csources" ]]; then + rm -rf csources + ln -s "$CSOURCES_DIR" csources + fi + + # bootstrap the Nim compiler and build the tools + rm -f bin/{nim,nim_csources} + pushd csources + if [[ "$ON_WINDOWS" == "0" ]]; then + $MAKE $UCPU clean + $MAKE $UCPU LD=$CC + else + $MAKE myos=windows $UCPU clean + $MAKE myos=windows $UCPU CC=gcc LD=gcc + fi + popd + if [[ -e csources/bin ]]; then + rm -f bin/nim bin/nim_csources + cp -a csources/bin/nim bin/nim + cp -a csources/bin/nim bin/nim_csources + rm -rf csources/bin + else + cp -a bin/nim bin/nim_csources + fi + if [[ "$QUICK_AND_DIRTY_COMPILER" == "0" ]]; then + sed \ + -e 's/koch$/--warnings:off --hints:off koch/' \ + -e 's/koch boot/koch boot --warnings:off --hints:off/' \ + -e '/nimBuildCsourcesIfNeeded/d' \ + build_all.sh > build_all_custom.sh + sh build_all_custom.sh + rm build_all_custom.sh + else + # Don't re-build it multiple times until we get identical + # binaries, like "build_all.sh" does. Don't build any tools + # either. This is all about build speed, not developer comfort. + bin/nim_csources \ + c \ + --compileOnly \ + --nimcache:nimcache \ + -d:release \ + --skipUserCfg \ + --skipParentCfg \ + --warnings:off \ + --hints:off \ + compiler/nim.nim + bin/nim_csources \ + jsonscript \ + --nimcache:nimcache \ + --skipUserCfg \ + --skipParentCfg \ + compiler/nim.nim + cp -a compiler/nim bin/nim1 + # If we stop here, we risk ending up with a buggy compiler: + # https://github.com/status-im/nimbus-eth2/pull/2220 + # https://github.com/status-im/nimbus-eth2/issues/2310 + bin/nim1 \ + c \ + --compileOnly \ + --nimcache:nimcache \ + -d:release \ + --skipUserCfg \ + --skipParentCfg \ + --warnings:off \ + --hints:off \ + compiler/nim.nim + bin/nim1 \ + jsonscript \ + --nimcache:nimcache \ + --skipUserCfg \ + --skipParentCfg \ + compiler/nim.nim + rm -f bin/nim + cp -a compiler/nim bin/nim + rm bin/nim1 + + if [[ ! -d "$NIMBLE_DIR" ]]; then + mkdir -p "$NIMBLE_DIR" + pushd "$NIMBLE_DIR" + git clone $NIMBLE_REPO . + git checkout $NIMBLE_COMMIT + pwd + ../../bin/nim r src/nimblepkg/private/clone.nim + # we have to delete .git or koch.nim will checkout a branch tip, overriding our target commit + rm -rf .git + popd + fi + if [[ "$NIMBLE_DIR" != "dist/nimble" ]]; then + mkdir -p dist + rm -rf dist/nimble + ln -s ../"$NIMBLE_DIR" dist/nimble + fi + # Do we want Nimble in this quick build? + if [[ "${QUICK_AND_DIRTY_NIMBLE}" != "0" ]]; then + bin/nim c -d:release --noNimblePath --skipUserCfg --skipParentCfg dist/nimble/src/nimble.nim + mv dist/nimble/src/nimble bin/ + fi + fi + fi + + if [[ "$QUICK_AND_DIRTY_COMPILER" == "0" || "${QUICK_AND_DIRTY_NIMBLE}" != "0" ]]; then + # Nimble needs a CA cert + rm -f bin/cacert.pem + curl -LsS -o bin/cacert.pem https://curl.se/ca/cacert.pem || echo "Warning: 'curl' failed to download a CA cert needed by Nimble. Ignoring it." + fi + + # record the built commit + echo ${NIM_COMMIT_HASH} > bin/last_built_commit + + # create the symlink + mv bin/nim bin/nim_commit_${NIM_COMMIT_HASH} + ln -s nim_commit_${NIM_COMMIT_HASH} bin/nim${EXE_SUFFIX} + + # update the CI cache + popd # we were in $NIM_DIR + if [[ -n "$CI_CACHE" ]]; then + rm -rf "$CI_CACHE" + mkdir "$CI_CACHE" + cp "$NIM_DIR"/bin/* "$CI_CACHE"/ + fi +} + +if nim_needs_rebuilding; then + build_nim +fi \ No newline at end of file