From 711609057c5d76266eedb8939d39b9104a5726af Mon Sep 17 00:00:00 2001 From: Tanguy Date: Mon, 21 Nov 2022 16:56:04 +0100 Subject: [PATCH] Reuseable install_nim action (#803) --- .github/actions/install_nim/action.yml | 131 +++++++++++++++++++++++++ .github/workflows/ci.yml | 122 +++++------------------ .github/workflows/codecov.yml | 37 +++++-- .github/workflows/multi_nim.yml | 113 +++------------------ 4 files changed, 196 insertions(+), 207 deletions(-) create mode 100644 .github/actions/install_nim/action.yml diff --git a/.github/actions/install_nim/action.yml b/.github/actions/install_nim/action.yml new file mode 100644 index 000000000..1bb49fcb8 --- /dev/null +++ b/.github/actions/install_nim/action.yml @@ -0,0 +1,131 @@ +name: Install Nim +inputs: + os: + description: "Operating system to build for" + required: true + cpu: + description: "CPU to build for" + default: "amd64" + nim_branch: + description: "Nim version" + default: "version-1-6" + shell: + description: "Shell to run commands in" + default: "bash --noprofile --norc -e -o pipefail" + +runs: + using: "composite" + steps: + - name: Install build dependencies (Linux i386) + shell: ${{ inputs.shell }} + if: inputs.os == 'Linux' && inputs.cpu == 'i386' + run: | + sudo dpkg --add-architecture i386 + sudo apt-get update -qq + sudo DEBIAN_FRONTEND='noninteractive' apt-get install \ + --no-install-recommends -yq gcc-multilib g++-multilib \ + libssl-dev:i386 + mkdir -p external/bin + cat << EOF > external/bin/gcc + #!/bin/bash + exec $(which gcc) -m32 "\$@" + EOF + cat << EOF > external/bin/g++ + #!/bin/bash + exec $(which g++) -m32 "\$@" + EOF + chmod 755 external/bin/gcc external/bin/g++ + echo '${{ github.workspace }}/external/bin' >> $GITHUB_PATH + + - name: MSYS2 (Windows i386) + if: inputs.os == 'Windows' && inputs.cpu == 'i386' + uses: msys2/setup-msys2@v2 + with: + path-type: inherit + msystem: MINGW32 + install: >- + base-devel + git + mingw-w64-i686-toolchain + + - name: MSYS2 (Windows amd64) + if: inputs.os == 'Windows' && inputs.cpu == 'amd64' + uses: msys2/setup-msys2@v2 + with: + path-type: inherit + install: >- + base-devel + git + mingw-w64-x86_64-toolchain + + - name: Restore Nim DLLs dependencies (Windows) from cache + if: inputs.os == 'Windows' + id: windows-dlls-cache + uses: actions/cache@v3 + with: + path: external/dlls + key: 'dlls' + + - name: Install DLL dependencies (Windows) + shell: ${{ inputs.shell }} + if: > + steps.windows-dlls-cache.outputs.cache-hit != 'true' && + inputs.os == 'Windows' + run: | + mkdir external + curl -L "https://nim-lang.org/download/windeps.zip" -o external/windeps.zip + 7z x external/windeps.zip -oexternal/dlls + + - name: Path to cached dependencies (Windows) + shell: ${{ inputs.shell }} + if: > + inputs.os == 'Windows' + run: | + echo '${{ github.workspace }}'"/external/dlls" >> $GITHUB_PATH + + - name: Derive environment variables + shell: ${{ inputs.shell }} + run: | + if [[ '${{ inputs.cpu }}' == 'amd64' ]]; then + PLATFORM=x64 + else + PLATFORM=x86 + fi + echo "PLATFORM=$PLATFORM" >> $GITHUB_ENV + + ncpu= + MAKE_CMD="make" + case '${{ inputs.os }}' in + 'Linux') + ncpu=$(nproc) + ;; + 'macOS') + ncpu=$(sysctl -n hw.ncpu) + ;; + 'Windows') + ncpu=$NUMBER_OF_PROCESSORS + MAKE_CMD="mingw32-make" + ;; + esac + [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1 + echo "ncpu=$ncpu" >> $GITHUB_ENV + echo "MAKE_CMD=${MAKE_CMD}" >> $GITHUB_ENV + echo '${{ github.workspace }}/nim/bin' >> $GITHUB_PATH + + - name: Restore Nim from cache + id: nim-cache + uses: actions/cache@v3 + with: + path: '${{ github.workspace }}/nim' + key: ${{ inputs.os }}-${{ inputs.cpu }}-nim-${{ inputs.nim_branch }}-cache-${{ env.cache_nonce }} + + - name: Build Nim and Nimble + shell: ${{ inputs.shell }} + if: ${{ steps.nim-cache.outputs.cache-hit != 'true' }} + run: | + # We don't want partial matches of the cache restored + rm -rf nim + curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_nim.sh + env MAKE="${MAKE_CMD} -j${ncpu}" ARCH_OVERRIDE=${PLATFORM} NIM_COMMIT=${{ inputs.nim_branch }} \ + QUICK_AND_DIRTY_COMPILER=1 QUICK_AND_DIRTY_NIMBLE=1 CC=gcc \ + bash build_nim.sh nim csources dist/nimble NimBinaries diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d671424ba..c5a72b8fd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,6 +7,10 @@ on: pull_request: workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: build: timeout-minutes: 90 @@ -45,111 +49,20 @@ jobs: name: '${{ matrix.target.os }}-${{ matrix.target.cpu }} (Nim ${{ matrix.branch }})' runs-on: ${{ matrix.builder }} - continue-on-error: ${{ matrix.branch == 'version-1-6' || matrix.branch == 'devel' }} + continue-on-error: ${{ matrix.branch == 'devel' }} steps: - name: Checkout uses: actions/checkout@v2 with: submodules: true - - name: Install build dependencies (Linux i386) - if: runner.os == 'Linux' && matrix.target.cpu == 'i386' - run: | - sudo dpkg --add-architecture i386 - sudo apt-get update -qq - sudo DEBIAN_FRONTEND='noninteractive' apt-get install \ - --no-install-recommends -yq gcc-multilib g++-multilib \ - libssl-dev:i386 - mkdir -p external/bin - cat << EOF > external/bin/gcc - #!/bin/bash - exec $(which gcc) -m32 "\$@" - EOF - cat << EOF > external/bin/g++ - #!/bin/bash - exec $(which g++) -m32 "\$@" - EOF - chmod 755 external/bin/gcc external/bin/g++ - echo '${{ github.workspace }}/external/bin' >> $GITHUB_PATH - - - name: MSYS2 (Windows i386) - if: runner.os == 'Windows' && matrix.target.cpu == 'i386' - uses: msys2/setup-msys2@v2 + - name: Setup Nim + uses: "./.github/actions/install_nim" with: - path-type: inherit - msystem: MINGW32 - install: >- - base-devel - git - mingw-w64-i686-toolchain - - - name: MSYS2 (Windows amd64) - if: runner.os == 'Windows' && matrix.target.cpu == 'amd64' - uses: msys2/setup-msys2@v2 - with: - path-type: inherit - install: >- - base-devel - git - mingw-w64-x86_64-toolchain - - - name: Restore Nim DLLs dependencies (Windows) from cache - if: runner.os == 'Windows' - id: windows-dlls-cache - uses: actions/cache@v2 - with: - path: external/dlls - key: 'dlls' - - - name: Install DLL dependencies (Windows) - if: > - steps.windows-dlls-cache.outputs.cache-hit != 'true' && - runner.os == 'Windows' - run: | - mkdir external - curl -L "https://nim-lang.org/download/windeps.zip" -o external/windeps.zip - 7z x external/windeps.zip -oexternal/dlls - - - name: Path to cached dependencies (Windows) - if: > - runner.os == 'Windows' - run: | - echo '${{ github.workspace }}'"/external/dlls" >> $GITHUB_PATH - - - name: Derive environment variables - run: | - if [[ '${{ matrix.target.cpu }}' == 'amd64' ]]; then - PLATFORM=x64 - else - PLATFORM=x86 - fi - echo "PLATFORM=$PLATFORM" >> $GITHUB_ENV - - ncpu= - MAKE_CMD="make" - case '${{ runner.os }}' in - 'Linux') - ncpu=$(nproc) - ;; - 'macOS') - ncpu=$(sysctl -n hw.ncpu) - ;; - 'Windows') - ncpu=$NUMBER_OF_PROCESSORS - MAKE_CMD="mingw32-make" - ;; - esac - [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1 - echo "ncpu=$ncpu" >> $GITHUB_ENV - echo "MAKE_CMD=${MAKE_CMD}" >> $GITHUB_ENV - - - name: Build Nim and Nimble - run: | - curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_nim.sh - env MAKE="${MAKE_CMD} -j${ncpu}" ARCH_OVERRIDE=${PLATFORM} NIM_COMMIT=${{ matrix.branch }} \ - QUICK_AND_DIRTY_COMPILER=1 QUICK_AND_DIRTY_NIMBLE=1 CC=gcc \ - bash build_nim.sh nim csources dist/nimble NimBinaries - echo '${{ github.workspace }}/nim/bin' >> $GITHUB_PATH + os: ${{ matrix.target.os }} + cpu: ${{ matrix.target.cpu }} + shell: ${{ matrix.shell }} + nim_branch: ${{ matrix.branch }} - name: Setup Go uses: actions/setup-go@v2 @@ -160,9 +73,20 @@ jobs: run: | V=1 bash scripts/build_p2pd.sh p2pdCache 124530a3 + - name: Restore deps from cache + id: deps-cache + uses: actions/cache@v3 + with: + path: nimbledeps + key: nimbledeps-${{ hashFiles('.pinned') }} + + - name: Install deps + if: ${{ steps.deps-cache.outputs.cache-hit != 'true' }} + run: | + nimble install_pinned + - name: Run tests run: | nim --version nimble --version - nimble install_pinned nimble test diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml index d30397b5e..de2d27231 100644 --- a/.github/workflows/codecov.yml +++ b/.github/workflows/codecov.yml @@ -9,6 +9,10 @@ on: pull_request: workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: Coverage: runs-on: ubuntu-20.04 @@ -18,15 +22,31 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 + + - name: Setup Nim + uses: "./.github/actions/install_nim" + with: + os: linux + cpu: amd64 + shell: bash + + - name: Restore deps from cache + id: deps-cache + uses: actions/cache@v3 + with: + path: nimbledeps + key: nimbledeps-${{ hashFiles('.pinned') }} + + - name: Install deps + if: ${{ steps.deps-cache.outputs.cache-hit != 'true' }} + run: | + nimble install_pinned + - name: Run run: | sudo apt-get update sudo apt-get install -y lcov build-essential git curl mkdir coverage - curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_nim.sh - env MAKE="make -j${NPROC}" bash build_nim.sh Nim csources dist/nimble NimBinaries - export PATH="$PATH:$PWD/Nim/bin" - nimble install_pinned export NIMFLAGS="--lineDir:on --passC:-fprofile-arcs --passC:-ftest-coverage --passL:-fprofile-arcs --passL:-ftest-coverage" nimble testnative nimble testpubsub @@ -38,7 +58,8 @@ jobs: lcov --extract coverage/coverage.info `pwd`/libp2p/{*,**/*}.nim --output-file coverage/coverage.f.info genhtml coverage/coverage.f.info --output-directory coverage/output bash <(curl -s https://codecov.io/bash) -f coverage/coverage.f.info || echo "Codecov did not collect coverage reports" - - uses: actions/upload-artifact@master - with: - name: coverage - path: coverage + + #- uses: actions/upload-artifact@master + # with: + # name: coverage + # path: coverage diff --git a/.github/workflows/multi_nim.yml b/.github/workflows/multi_nim.yml index 7954fc763..1334983ec 100644 --- a/.github/workflows/multi_nim.yml +++ b/.github/workflows/multi_nim.yml @@ -5,7 +5,13 @@ on: workflow_dispatch: jobs: + delete-cache: + runs-on: ubuntu-latest + steps: + - uses: snnaplab/delete-branch-cache-action@v1 + build: + needs: delete-cache timeout-minutes: 120 strategy: fail-fast: false @@ -46,107 +52,14 @@ jobs: steps: - name: Checkout uses: actions/checkout@v2 + + - name: Setup Nim + uses: "./.github/actions/install_nim" with: - submodules: true - - - name: Install build dependencies (Linux i386) - if: runner.os == 'Linux' && matrix.target.cpu == 'i386' - run: | - sudo dpkg --add-architecture i386 - sudo apt-get update -qq - sudo DEBIAN_FRONTEND='noninteractive' apt-get install \ - --no-install-recommends -yq gcc-multilib g++-multilib \ - libssl-dev:i386 - mkdir -p external/bin - cat << EOF > external/bin/gcc - #!/bin/bash - exec $(which gcc) -m32 "\$@" - EOF - cat << EOF > external/bin/g++ - #!/bin/bash - exec $(which g++) -m32 "\$@" - EOF - chmod 755 external/bin/gcc external/bin/g++ - echo '${{ github.workspace }}/external/bin' >> $GITHUB_PATH - - - name: MSYS2 (Windows i386) - if: runner.os == 'Windows' && matrix.target.cpu == 'i386' - uses: msys2/setup-msys2@v2 - with: - path-type: inherit - msystem: MINGW32 - install: >- - base-devel - git - mingw-w64-i686-toolchain - - - name: MSYS2 (Windows amd64) - if: runner.os == 'Windows' && matrix.target.cpu == 'amd64' - uses: msys2/setup-msys2@v2 - with: - path-type: inherit - install: >- - base-devel - git - mingw-w64-x86_64-toolchain - - - name: Restore Nim DLLs dependencies (Windows) from cache - if: runner.os == 'Windows' - id: windows-dlls-cache - uses: actions/cache@v2 - with: - path: external/dlls - key: 'dlls' - - - name: Install DLL dependencies (Windows) - if: > - steps.windows-dlls-cache.outputs.cache-hit != 'true' && - runner.os == 'Windows' - run: | - mkdir external - curl -L "https://nim-lang.org/download/windeps.zip" -o external/windeps.zip - 7z x external/windeps.zip -oexternal/dlls - - - name: Path to cached dependencies (Windows) - if: > - runner.os == 'Windows' - run: | - echo '${{ github.workspace }}'"/external/dlls" >> $GITHUB_PATH - - - name: Derive environment variables - run: | - if [[ '${{ matrix.target.cpu }}' == 'amd64' ]]; then - PLATFORM=x64 - else - PLATFORM=x86 - fi - echo "PLATFORM=$PLATFORM" >> $GITHUB_ENV - - ncpu= - MAKE_CMD="make" - case '${{ runner.os }}' in - 'Linux') - ncpu=$(nproc) - ;; - 'macOS') - ncpu=$(sysctl -n hw.ncpu) - ;; - 'Windows') - ncpu=$NUMBER_OF_PROCESSORS - MAKE_CMD="mingw32-make" - ;; - esac - [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1 - echo "ncpu=$ncpu" >> $GITHUB_ENV - echo "MAKE_CMD=${MAKE_CMD}" >> $GITHUB_ENV - - - name: Build Nim and Nimble - run: | - curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_nim.sh - env MAKE="${MAKE_CMD} -j${ncpu}" ARCH_OVERRIDE=${PLATFORM} NIM_COMMIT=${{ matrix.branch }} \ - QUICK_AND_DIRTY_COMPILER=1 QUICK_AND_DIRTY_NIMBLE=1 CC=gcc \ - bash build_nim.sh nim csources dist/nimble NimBinaries - echo '${{ github.workspace }}/nim/bin' >> $GITHUB_PATH + os: ${{ matrix.target.os }} + shell: ${{ matrix.shell }} + nim_branch: ${{ matrix.branch }} + cpu: ${{ matrix.target.cpu }} - name: Setup Go uses: actions/setup-go@v2