From db6bd9af1628211cbdaab2e6ad1419f66582a9ed Mon Sep 17 00:00:00 2001 From: andri lim Date: Fri, 8 Sep 2023 22:10:16 +0700 Subject: [PATCH] Switch to llvm-mingw for faster Windows CI (#1740) --- .github/workflows/ci.yml | 54 +++++++++++++-------- .github/workflows/fluffy.yml | 39 ++++++++++----- .github/workflows/nimbus_verified_proxy.yml | 50 ++++++++++--------- .github/workflows/simulators.yml | 28 ++++++++--- Makefile | 10 ++++ vendor/nim-libbacktrace | 2 +- vendor/nim-nat-traversal | 2 +- 7 files changed, 117 insertions(+), 68 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2c7c4d9a0..b6b05ffa0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,22 +20,21 @@ jobs: - os: linux cpu: amd64 evmc: evmc - # - os: linux - # cpu: i386 - # evmc: evmc + #- os: linux + # cpu: i386 + # evmc: evmc - os: macos cpu: amd64 evmc: evmc - os: windows cpu: amd64 evmc: evmc - # vm2 - os: linux cpu: amd64 evmc: nimvm - # - os: linux - # cpu: i386 - # evmc: nimvm + #- os: linux + # cpu: i386 + # evmc: nimvm - os: windows cpu: amd64 evmc: nimvm @@ -46,19 +45,16 @@ jobs: - target: os: linux builder: ubuntu-20.04 - shell: bash - target: os: macos builder: macos-11 - shell: bash - target: os: windows builder: windows-latest - shell: msys2 {0} defaults: run: - shell: ${{ matrix.shell }} + shell: bash name: '${{ matrix.target.os }}-${{ matrix.target.cpu }}-${{ matrix.target.evmc }}' runs-on: ${{ matrix.builder }} @@ -67,7 +63,6 @@ jobs: uses: actions/checkout@v3 - name: Derive environment variables - shell: bash run: | if [[ '${{ matrix.target.cpu }}' == 'amd64' ]]; then PLATFORM=x64 @@ -166,15 +161,13 @@ jobs: curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_rocksdb.sh bash build_rocksdb.sh rocks-db-cache-${{ matrix.target.cpu }} - - name: MSYS2 (Windows amd64) - if: runner.os == 'Windows' && matrix.target.cpu == 'amd64' - uses: msys2/setup-msys2@v2 + - name: Restore llvm-mingw (Windows) from cache + if: runner.os == 'Windows' + id: windows-mingw-cache + uses: actions/cache@v3 with: - path-type: inherit - install: >- - base-devel - git - mingw-w64-x86_64-toolchain + path: external/mingw-${{ matrix.target.cpu }} + key: 'mingw-llvm-17-${{ matrix.target.cpu }}' - name: Restore Nim DLLs dependencies (Windows) from cache if: runner.os == 'Windows' @@ -186,6 +179,24 @@ jobs: # so change dlls# to force new cache contents (for some number #) key: dlls0-${{ matrix.target.cpu }} + - name: Install llvm-mingw dependency (Windows) + if: > + steps.windows-mingw-cache.outputs.cache-hit != 'true' && + runner.os == 'Windows' + run: | + mkdir -p external + MINGW_BASE="https://github.com/mstorsjo/llvm-mingw/releases/download/20230905" + if [[ '${{ matrix.target.cpu }}' == 'amd64' ]]; then + MINGW_URL="$MINGW_BASE/llvm-mingw-20230905-ucrt-x86_64.zip" + ARCH=64 + else + MINGW_URL="$MINGW_BASE/llvm-mingw-20230905-ucrt-i686.zip" + ARCH=32 + fi + curl -L "$MINGW_URL" -o "external/mingw-${{ matrix.target.cpu }}.zip" + 7z x -y "external/mingw-${{ matrix.target.cpu }}.zip" -oexternal/mingw-${{ matrix.target.cpu }}/ + mv external/mingw-${{ matrix.target.cpu }}/**/* ./external/mingw-${{ matrix.target.cpu }} + - name: Install DLLs dependencies (Windows) if: > steps.windows-dlls-cache.outputs.cache-hit != 'true' && @@ -209,7 +220,8 @@ jobs: if: > runner.os == 'Windows' run: | - echo '${{ github.workspace }}'"/external/dlls-${{ matrix.target.cpu }}" >> $GITHUB_PATH + echo '${{ github.workspace }}'"/external/mingw-${{ matrix.target.cpu }}/bin" >> $GITHUB_PATH + echo '${{ github.workspace }}'"/external/dlls-${{ matrix.target.cpu }}" >> $GITHUB_PATH - name: Get latest nimbus-build-system commit hash id: versions diff --git a/.github/workflows/fluffy.yml b/.github/workflows/fluffy.yml index 9c9a57ece..2abbfd67d 100644 --- a/.github/workflows/fluffy.yml +++ b/.github/workflows/fluffy.yml @@ -109,19 +109,16 @@ jobs: - target: os: linux builder: ubuntu-22.04 - shell: bash - target: os: macos builder: macos-11 - shell: bash - target: os: windows builder: windows-latest - shell: msys2 {0} defaults: run: - shell: ${{ matrix.shell }} + shell: bash name: '${{ matrix.target.os }}-${{ matrix.target.cpu }}' runs-on: ${{ matrix.builder }} @@ -130,7 +127,6 @@ jobs: uses: actions/checkout@v3 - name: Derive environment variables - shell: bash run: | if [[ '${{ matrix.target.cpu }}' == 'amd64' ]]; then PLATFORM=x64 @@ -186,15 +182,13 @@ jobs: brew install gnu-getopt brew link --force gnu-getopt - - name: MSYS2 (Windows amd64) - if: runner.os == 'Windows' && matrix.target.cpu == 'amd64' - uses: msys2/setup-msys2@v2 + - name: Restore llvm-mingw (Windows) from cache + if: runner.os == 'Windows' + id: windows-mingw-cache + uses: actions/cache@v3 with: - path-type: inherit - install: >- - base-devel - git - mingw-w64-x86_64-toolchain + path: external/mingw-${{ matrix.target.cpu }} + key: 'mingw-llvm-17-${{ matrix.target.cpu }}' - name: Restore Nim DLLs dependencies (Windows) from cache if: runner.os == 'Windows' @@ -204,6 +198,24 @@ jobs: path: external/dlls-${{ matrix.target.cpu }} key: 'dlls-${{ matrix.target.cpu }}-fluffy' + - name: Install llvm-mingw dependency (Windows) + if: > + steps.windows-mingw-cache.outputs.cache-hit != 'true' && + runner.os == 'Windows' + run: | + mkdir -p external + MINGW_BASE="https://github.com/mstorsjo/llvm-mingw/releases/download/20230905" + if [[ '${{ matrix.target.cpu }}' == 'amd64' ]]; then + MINGW_URL="$MINGW_BASE/llvm-mingw-20230905-ucrt-x86_64.zip" + ARCH=64 + else + MINGW_URL="$MINGW_BASE/llvm-mingw-20230905-ucrt-x86_64.zip" + ARCH=32 + fi + curl -L "$MINGW_URL" -o "external/mingw-${{ matrix.target.cpu }}.zip" + 7z x -y "external/mingw-${{ matrix.target.cpu }}.zip" -oexternal/mingw-${{ matrix.target.cpu }}/ + mv external/mingw-${{ matrix.target.cpu }}/**/* ./external/mingw-${{ matrix.target.cpu }} + - name: Install DLLs dependencies (Windows) if: > steps.windows-dlls-cache.outputs.cache-hit != 'true' && @@ -218,6 +230,7 @@ jobs: if: > runner.os == 'Windows' run: | + echo '${{ github.workspace }}'"/external/mingw-${{ matrix.target.cpu }}/bin" >> $GITHUB_PATH echo '${{ github.workspace }}'"/external/dlls-${{ matrix.target.cpu }}" >> $GITHUB_PATH - name: Get latest nimbus-build-system commit hash diff --git a/.github/workflows/nimbus_verified_proxy.yml b/.github/workflows/nimbus_verified_proxy.yml index 6bfa82616..81c9fed9a 100644 --- a/.github/workflows/nimbus_verified_proxy.yml +++ b/.github/workflows/nimbus_verified_proxy.yml @@ -39,19 +39,16 @@ jobs: - target: os: linux builder: ubuntu-20.04 - shell: bash - target: os: macos builder: macos-11 - shell: bash - target: os: windows builder: windows-latest - shell: msys2 {0} defaults: run: - shell: ${{ matrix.shell }} + shell: bash name: '${{ matrix.target.os }}-${{ matrix.target.cpu }}' runs-on: ${{ matrix.builder }} @@ -60,7 +57,6 @@ jobs: uses: actions/checkout@v3 - name: Derive environment variables - shell: bash run: | if [[ '${{ matrix.target.cpu }}' == 'amd64' ]]; then PLATFORM=x64 @@ -116,26 +112,13 @@ jobs: brew install gnu-getopt brew link --force gnu-getopt - - name: MSYS2 (Windows i386) - if: runner.os == 'Windows' && matrix.target.cpu == 'i386' - uses: msys2/setup-msys2@v2 + - name: Restore llvm-mingw (Windows) from cache + if: runner.os == 'Windows' + id: windows-mingw-cache + uses: actions/cache@v3 with: - path-type: inherit - msystem: MINGW32 - install: >- - base-devel - git - mingw-w64-i686-toolchain - - - name: MSYS2 (Windows amd64) - if: runner.os == 'Windows' && matrix.target.cpu == 'amd64' - uses: msys2/setup-msys2@v2 - with: - path-type: inherit - install: >- - base-devel - git - mingw-w64-x86_64-toolchain + path: external/mingw-${{ matrix.target.cpu }} + key: 'mingw-llvm-17-${{ matrix.target.cpu }}' - name: Restore Nim DLLs dependencies (Windows) from cache if: runner.os == 'Windows' @@ -145,6 +128,24 @@ jobs: path: external/dlls-${{ matrix.target.cpu }} key: 'dlls-${{ matrix.target.cpu }}-verified-proxy' + - name: Install llvm-mingw dependency (Windows) + if: > + steps.windows-mingw-cache.outputs.cache-hit != 'true' && + runner.os == 'Windows' + run: | + mkdir -p external + MINGW_BASE="https://github.com/mstorsjo/llvm-mingw/releases/download/20230905" + if [[ '${{ matrix.target.cpu }}' == 'amd64' ]]; then + MINGW_URL="$MINGW_BASE/llvm-mingw-20230905-ucrt-x86_64.zip" + ARCH=64 + else + MINGW_URL="$MINGW_BASE/llvm-mingw-20230905-ucrt-x86_64.zip" + ARCH=32 + fi + curl -L "$MINGW_URL" -o "external/mingw-${{ matrix.target.cpu }}.zip" + 7z x -y "external/mingw-${{ matrix.target.cpu }}.zip" -oexternal/mingw-${{ matrix.target.cpu }}/ + mv external/mingw-${{ matrix.target.cpu }}/**/* ./external/mingw-${{ matrix.target.cpu }} + - name: Install DLLs dependencies (Windows) if: > steps.windows-dlls-cache.outputs.cache-hit != 'true' && @@ -159,6 +160,7 @@ jobs: if: > runner.os == 'Windows' run: | + echo '${{ github.workspace }}'"/external/mingw-${{ matrix.target.cpu }}/bin" >> $GITHUB_PATH echo '${{ github.workspace }}'"/external/dlls-${{ matrix.target.cpu }}" >> $GITHUB_PATH - name: Get latest nimbus-build-system commit hash diff --git a/.github/workflows/simulators.yml b/.github/workflows/simulators.yml index 4b84bf03f..6c4c507de 100644 --- a/.github/workflows/simulators.yml +++ b/.github/workflows/simulators.yml @@ -95,19 +95,31 @@ jobs: runs-on: windows-latest defaults: run: - shell: msys2 {0} + shell: bash steps: - name: Checkout code uses: actions/checkout@v3 - - name: MSYS2 - uses: msys2/setup-msys2@v2 + - name: Restore llvm-mingw (Windows) from cache + if: runner.os == 'Windows' + id: windows-mingw-cache + uses: actions/cache@v3 with: - path-type: inherit - install: >- - base-devel - git - mingw-w64-x86_64-toolchain + path: external/mingw-amd64 + key: 'mingw-llvm-17-amd64' + + - name: Install llvm-mingw dependency (Windows) + if: > + steps.windows-mingw-cache.outputs.cache-hit != 'true' && + runner.os == 'Windows' + run: | + mkdir -p external + MINGW_BASE="https://github.com/mstorsjo/llvm-mingw/releases/download/20230905" + MINGW_URL="$MINGW_BASE/llvm-mingw-20230905-ucrt-x86_64.zip" + curl -L "$MINGW_URL" -o "external/mingw-amd64.zip" + 7z x -y "external/mingw-amd64.zip" -oexternal/mingw-amd64/ + mv external/mingw-amd64/**/* ./external/mingw-amd64 + echo '${{ github.workspace }}'"/external/mingw-amd64/bin" >> $GITHUB_PATH - name: Get latest nimbus-build-system commit hash id: versions diff --git a/Makefile b/Makefile index 3555bbdf0..1bafc40a0 100644 --- a/Makefile +++ b/Makefile @@ -140,6 +140,16 @@ ifeq ($(USE_LIBBACKTRACE), 0) NIM_PARAMS += -d:disable_libbacktrace endif +# TODO: port this back to nimbus-build-system +# after we can switch to nim 1.6.16 +# This rule override the one in targets.mk +libnatpmp.a: | sanity-checks +ifeq ($(OS), Windows_NT) + + "$(MAKE)" -C vendor/nim-nat-traversal/vendor/libnatpmp-upstream OS=mingw CC=$(CC) CFLAGS="-Wall -Wno-cpp -Os -DWIN32 -DNATPMP_STATICLIB -DENABLE_STRNATPMPERR -DNATPMP_MAX_RETRIES=4 $(CFLAGS)" $@ $(HANDLE_OUTPUT) +else + + "$(MAKE)" CFLAGS="-Wall -Wno-cpp -Os -DENABLE_STRNATPMPERR -DNATPMP_MAX_RETRIES=4 $(CFLAGS)" -C vendor/nim-nat-traversal/vendor/libnatpmp-upstream $@ $(HANDLE_OUTPUT) +endif + deps: | deps-common nat-libs nimbus.nims ifneq ($(USE_LIBBACKTRACE), 0) deps: | libbacktrace diff --git a/vendor/nim-libbacktrace b/vendor/nim-libbacktrace index b29c22ba0..aad5aa227 160000 --- a/vendor/nim-libbacktrace +++ b/vendor/nim-libbacktrace @@ -1 +1 @@ -Subproject commit b29c22ba0ef13de50b779c776830dbea1d50cd33 +Subproject commit aad5aa227c43a83984245b180c6c295ce9ff561b diff --git a/vendor/nim-nat-traversal b/vendor/nim-nat-traversal index 27d314d65..14e016503 160000 --- a/vendor/nim-nat-traversal +++ b/vendor/nim-nat-traversal @@ -1 +1 @@ -Subproject commit 27d314d65c9078924b3239fe4e2f5af0c512b28c +Subproject commit 14e016503fe6488230e6d0e59396efde96bd9252