Switch to llvm-mingw for faster Windows CI (#1740)

2023-09-08 22:10:16 +07:00 · 2023-09-08 22:10:16 +07:00 · db6bd9af16
parent 348a9aea6f
commit db6bd9af16
7 changed files with 117 additions and 68 deletions
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@ -20,22 +20,21 @@ jobs:
          - os: linux
            cpu: amd64
            evmc: evmc
-          # - os: linux
-          #   cpu: i386
-          #   evmc: evmc
+          #- os: linux
+          #  cpu: i386
+          #  evmc: evmc
          - os: macos
            cpu: amd64
            evmc: evmc
          - os: windows
            cpu: amd64
            evmc: evmc
-          # vm2
          - os: linux
            cpu: amd64
            evmc: nimvm
-          # - os: linux
-          #   cpu: i386
-          #   evmc: nimvm
+          #- os: linux
+          #  cpu: i386
+          #  evmc: nimvm
          - os: windows
            cpu: amd64
            evmc: nimvm
@ -46,19 +45,16 @@ jobs:
          - target:
              os: linux
            builder: ubuntu-20.04
-            shell: bash
          - target:
              os: macos
            builder: macos-11
-            shell: bash
          - target:
              os: windows
            builder: windows-latest
-            shell: msys2 {0}

    defaults:
      run:
-        shell: ${{ matrix.shell }}
+        shell: bash

    name: '${{ matrix.target.os }}-${{ matrix.target.cpu }}-${{ matrix.target.evmc }}'
    runs-on: ${{ matrix.builder }}
@ -67,7 +63,6 @@ jobs:
        uses: actions/checkout@v3

      - name: Derive environment variables
-        shell: bash
        run: |
          if [[ '${{ matrix.target.cpu }}' == 'amd64' ]]; then
            PLATFORM=x64
@ -166,15 +161,13 @@ jobs:
          curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_rocksdb.sh
          bash build_rocksdb.sh rocks-db-cache-${{ matrix.target.cpu }}

-      - name: MSYS2 (Windows amd64)
-        if: runner.os == 'Windows' && matrix.target.cpu == 'amd64'
-        uses: msys2/setup-msys2@v2
+      - name: Restore llvm-mingw (Windows) from cache
+        if: runner.os == 'Windows'
+        id: windows-mingw-cache
+        uses: actions/cache@v3
        with:
-          path-type: inherit
-          install: >-
-            base-devel
-            git
-            mingw-w64-x86_64-toolchain
+          path: external/mingw-${{ matrix.target.cpu }}
+          key: 'mingw-llvm-17-${{ matrix.target.cpu }}'

      - name: Restore Nim DLLs dependencies (Windows) from cache
        if: runner.os == 'Windows'
@ -186,6 +179,24 @@ jobs:
          # so change dlls# to force new cache contents (for some number #)
          key: dlls0-${{ matrix.target.cpu }}

+      - name: Install llvm-mingw dependency (Windows)
+        if: >
+          steps.windows-mingw-cache.outputs.cache-hit != 'true' &&
+          runner.os == 'Windows'
+        run: |
+          mkdir -p external
+          MINGW_BASE="https://github.com/mstorsjo/llvm-mingw/releases/download/20230905"
+          if [[ '${{ matrix.target.cpu }}' == 'amd64' ]]; then
+            MINGW_URL="$MINGW_BASE/llvm-mingw-20230905-ucrt-x86_64.zip"
+            ARCH=64
+          else
+            MINGW_URL="$MINGW_BASE/llvm-mingw-20230905-ucrt-i686.zip"
+            ARCH=32
+          fi
+          curl -L "$MINGW_URL" -o "external/mingw-${{ matrix.target.cpu }}.zip"
+          7z x -y "external/mingw-${{ matrix.target.cpu }}.zip" -oexternal/mingw-${{ matrix.target.cpu }}/
+          mv external/mingw-${{ matrix.target.cpu }}/**/* ./external/mingw-${{ matrix.target.cpu }}
+
      - name: Install DLLs dependencies (Windows)
        if: >
          steps.windows-dlls-cache.outputs.cache-hit != 'true' &&
@ -209,6 +220,7 @@ jobs:
        if: >
          runner.os == 'Windows'
        run: |
+          echo '${{ github.workspace }}'"/external/mingw-${{ matrix.target.cpu }}/bin" >> $GITHUB_PATH
          echo '${{ github.workspace }}'"/external/dlls-${{ matrix.target.cpu }}" >> $GITHUB_PATH          

      - name: Get latest nimbus-build-system commit hash
--- a/.github/workflows/fluffy.yml
+++ b/.github/workflows/fluffy.yml
@ -109,19 +109,16 @@ jobs:
          - target:
              os: linux
            builder: ubuntu-22.04
-            shell: bash
          - target:
              os: macos
            builder: macos-11
-            shell: bash
          - target:
              os: windows
            builder: windows-latest
-            shell: msys2 {0}

    defaults:
      run:
-        shell: ${{ matrix.shell }}
+        shell: bash

    name: '${{ matrix.target.os }}-${{ matrix.target.cpu }}'
    runs-on: ${{ matrix.builder }}
@ -130,7 +127,6 @@ jobs:
        uses: actions/checkout@v3

      - name: Derive environment variables
-        shell: bash
        run: |
          if [[ '${{ matrix.target.cpu }}' == 'amd64' ]]; then
            PLATFORM=x64
@ -186,15 +182,13 @@ jobs:
          brew install gnu-getopt
          brew link --force gnu-getopt

-      - name: MSYS2 (Windows amd64)
-        if: runner.os == 'Windows' && matrix.target.cpu == 'amd64'
-        uses: msys2/setup-msys2@v2
+      - name: Restore llvm-mingw (Windows) from cache
+        if: runner.os == 'Windows'
+        id: windows-mingw-cache
+        uses: actions/cache@v3
        with:
-          path-type: inherit
-          install: >-
-            base-devel
-            git
-            mingw-w64-x86_64-toolchain
+          path: external/mingw-${{ matrix.target.cpu }}
+          key: 'mingw-llvm-17-${{ matrix.target.cpu }}'

      - name: Restore Nim DLLs dependencies (Windows) from cache
        if: runner.os == 'Windows'
@ -204,6 +198,24 @@ jobs:
          path: external/dlls-${{ matrix.target.cpu }}
          key: 'dlls-${{ matrix.target.cpu }}-fluffy'

+      - name: Install llvm-mingw dependency (Windows)
+        if: >
+          steps.windows-mingw-cache.outputs.cache-hit != 'true' &&
+          runner.os == 'Windows'
+        run: |
+          mkdir -p external
+          MINGW_BASE="https://github.com/mstorsjo/llvm-mingw/releases/download/20230905"
+          if [[ '${{ matrix.target.cpu }}' == 'amd64' ]]; then
+            MINGW_URL="$MINGW_BASE/llvm-mingw-20230905-ucrt-x86_64.zip"
+            ARCH=64
+          else
+            MINGW_URL="$MINGW_BASE/llvm-mingw-20230905-ucrt-x86_64.zip"
+            ARCH=32
+          fi
+          curl -L "$MINGW_URL" -o "external/mingw-${{ matrix.target.cpu }}.zip"
+          7z x -y "external/mingw-${{ matrix.target.cpu }}.zip" -oexternal/mingw-${{ matrix.target.cpu }}/
+          mv external/mingw-${{ matrix.target.cpu }}/**/* ./external/mingw-${{ matrix.target.cpu }}
+
      - name: Install DLLs dependencies (Windows)
        if: >
          steps.windows-dlls-cache.outputs.cache-hit != 'true' &&
@ -218,6 +230,7 @@ jobs:
        if: >
          runner.os == 'Windows'
        run: |
+          echo '${{ github.workspace }}'"/external/mingw-${{ matrix.target.cpu }}/bin" >> $GITHUB_PATH
          echo '${{ github.workspace }}'"/external/dlls-${{ matrix.target.cpu }}" >> $GITHUB_PATH

      - name: Get latest nimbus-build-system commit hash
--- a/.github/workflows/nimbus_verified_proxy.yml
+++ b/.github/workflows/nimbus_verified_proxy.yml
@ -39,19 +39,16 @@ jobs:
          - target:
              os: linux
            builder: ubuntu-20.04
-            shell: bash
          - target:
              os: macos
            builder: macos-11
-            shell: bash
          - target:
              os: windows
            builder: windows-latest
-            shell: msys2 {0}

    defaults:
      run:
-        shell: ${{ matrix.shell }}
+        shell: bash

    name: '${{ matrix.target.os }}-${{ matrix.target.cpu }}'
    runs-on: ${{ matrix.builder }}
@ -60,7 +57,6 @@ jobs:
        uses: actions/checkout@v3

      - name: Derive environment variables
-        shell: bash
        run: |
          if [[ '${{ matrix.target.cpu }}' == 'amd64' ]]; then
            PLATFORM=x64
@ -116,26 +112,13 @@ jobs:
          brew install gnu-getopt
          brew link --force gnu-getopt

-      - name: MSYS2 (Windows i386)
-        if: runner.os == 'Windows' && matrix.target.cpu == 'i386'
-        uses: msys2/setup-msys2@v2
+      - name: Restore llvm-mingw (Windows) from cache
+        if: runner.os == 'Windows'
+        id: windows-mingw-cache
+        uses: actions/cache@v3
        with:
-          path-type: inherit
-          msystem: MINGW32
-          install: >-
-            base-devel
-            git
-            mingw-w64-i686-toolchain
-
-      - name: MSYS2 (Windows amd64)
-        if: runner.os == 'Windows' && matrix.target.cpu == 'amd64'
-        uses: msys2/setup-msys2@v2
-        with:
-          path-type: inherit
-          install: >-
-            base-devel
-            git
-            mingw-w64-x86_64-toolchain
+          path: external/mingw-${{ matrix.target.cpu }}
+          key: 'mingw-llvm-17-${{ matrix.target.cpu }}'

      - name: Restore Nim DLLs dependencies (Windows) from cache
        if: runner.os == 'Windows'
@ -145,6 +128,24 @@ jobs:
          path: external/dlls-${{ matrix.target.cpu }}
          key: 'dlls-${{ matrix.target.cpu }}-verified-proxy'

+      - name: Install llvm-mingw dependency (Windows)
+        if: >
+          steps.windows-mingw-cache.outputs.cache-hit != 'true' &&
+          runner.os == 'Windows'
+        run: |
+          mkdir -p external
+          MINGW_BASE="https://github.com/mstorsjo/llvm-mingw/releases/download/20230905"
+          if [[ '${{ matrix.target.cpu }}' == 'amd64' ]]; then
+            MINGW_URL="$MINGW_BASE/llvm-mingw-20230905-ucrt-x86_64.zip"
+            ARCH=64
+          else
+            MINGW_URL="$MINGW_BASE/llvm-mingw-20230905-ucrt-x86_64.zip"
+            ARCH=32
+          fi
+          curl -L "$MINGW_URL" -o "external/mingw-${{ matrix.target.cpu }}.zip"
+          7z x -y "external/mingw-${{ matrix.target.cpu }}.zip" -oexternal/mingw-${{ matrix.target.cpu }}/
+          mv external/mingw-${{ matrix.target.cpu }}/**/* ./external/mingw-${{ matrix.target.cpu }}
+
      - name: Install DLLs dependencies (Windows)
        if: >
          steps.windows-dlls-cache.outputs.cache-hit != 'true' &&
@ -159,6 +160,7 @@ jobs:
        if: >
          runner.os == 'Windows'
        run: |
+          echo '${{ github.workspace }}'"/external/mingw-${{ matrix.target.cpu }}/bin" >> $GITHUB_PATH
          echo '${{ github.workspace }}'"/external/dlls-${{ matrix.target.cpu }}" >> $GITHUB_PATH

      - name: Get latest nimbus-build-system commit hash
--- a/.github/workflows/simulators.yml
+++ b/.github/workflows/simulators.yml
@ -95,19 +95,31 @@ jobs:
    runs-on: windows-latest
    defaults:
      run:
-        shell: msys2 {0}
+        shell: bash
    steps:
      - name: Checkout code
        uses: actions/checkout@v3

-      - name: MSYS2
-        uses: msys2/setup-msys2@v2
+      - name: Restore llvm-mingw (Windows) from cache
+        if: runner.os == 'Windows'
+        id: windows-mingw-cache
+        uses: actions/cache@v3
        with:
-          path-type: inherit
-          install: >-
-            base-devel
-            git
-            mingw-w64-x86_64-toolchain
+          path: external/mingw-amd64
+          key: 'mingw-llvm-17-amd64'
+
+      - name: Install llvm-mingw dependency (Windows)
+        if: >
+          steps.windows-mingw-cache.outputs.cache-hit != 'true' &&
+          runner.os == 'Windows'
+        run: |
+          mkdir -p external
+          MINGW_BASE="https://github.com/mstorsjo/llvm-mingw/releases/download/20230905"
+          MINGW_URL="$MINGW_BASE/llvm-mingw-20230905-ucrt-x86_64.zip"
+          curl -L "$MINGW_URL" -o "external/mingw-amd64.zip"
+          7z x -y "external/mingw-amd64.zip" -oexternal/mingw-amd64/
+          mv external/mingw-amd64/**/* ./external/mingw-amd64
+          echo '${{ github.workspace }}'"/external/mingw-amd64/bin" >> $GITHUB_PATH

      - name: Get latest nimbus-build-system commit hash
        id: versions
--- a/10
+++ b/10
@ -140,6 +140,16 @@ ifeq ($(USE_LIBBACKTRACE), 0)
  NIM_PARAMS += -d:disable_libbacktrace
 endif

+# TODO: port this back to nimbus-build-system
+# after we can switch to nim 1.6.16
+# This rule override the one in targets.mk
+libnatpmp.a: | sanity-checks
+ifeq ($(OS), Windows_NT)
+	+ "$(MAKE)" -C vendor/nim-nat-traversal/vendor/libnatpmp-upstream OS=mingw CC=$(CC) CFLAGS="-Wall -Wno-cpp -Os -DWIN32 -DNATPMP_STATICLIB -DENABLE_STRNATPMPERR -DNATPMP_MAX_RETRIES=4 $(CFLAGS)" $@ $(HANDLE_OUTPUT)
+else
+	+ "$(MAKE)" CFLAGS="-Wall -Wno-cpp -Os -DENABLE_STRNATPMPERR -DNATPMP_MAX_RETRIES=4 $(CFLAGS)" -C vendor/nim-nat-traversal/vendor/libnatpmp-upstream $@ $(HANDLE_OUTPUT)
+endif
+
 deps: | deps-common nat-libs nimbus.nims
 ifneq ($(USE_LIBBACKTRACE), 0)
 deps: | libbacktrace
--- a/vendor/nim-libbacktrace
+++ b/vendor/nim-libbacktrace
@ -1 +1 @@
-Subproject commit b29c22ba0ef13de50b779c776830dbea1d50cd33
+Subproject commit aad5aa227c43a83984245b180c6c295ce9ff561b
--- a/vendor/nim-nat-traversal
+++ b/vendor/nim-nat-traversal
@ -1 +1 @@
-Subproject commit 27d314d65c9078924b3239fe4e2f5af0c512b28c
+Subproject commit 14e016503fe6488230e6d0e59396efde96bd9252