refactor(ci): cache all of submodules/deps to speed up build time (#1731)

.github/workflows/ci-experimental.yml

@@ -61,27 +61,19 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v3
-        with:
-          submodules: recursive
 
-      - name: Get submodule versions
+      - name: Get submodules hash
         id: submodules
-        shell: bash
         run: |
-          echo "nim-hash=$(git rev-parse HEAD:vendor/nimbus-build-system)" >> $GITHUB_OUTPUT
-          echo "zerokit-hash=$(git rev-parse HEAD:vendor/zerokit)" >> $GITHUB_OUTPUT
+          echo "hash=$(git submodule status | sha256sum | sed 's/[ -]*//g')" >> $GITHUB_OUTPUT
 
-      - name: Cache nim compiler
+      - name: Cache submodules
         uses: actions/cache@v3
         with:
-          path: vendor/nimbus-build-system/vendor/Nim/bin
-          key: ${{ runner.os }}-nim-${{ steps.submodules.outputs.nim-hash }}
-
-      - name: Cache zerokit artifacts
-        uses: actions/cache@v3
-        with:
-          path: vendor/zerokit/target/release
-          key: ${{ runner.os }}-zerokit-${{ steps.submodules.outputs.zerokit-hash }}
+          path: |
+            vendor/
+            .git/modules
+          key: ${{ runner.os }}-vendor-modules-${{ steps.submodules.outputs.hash }}
 
       - name: Build binaries
         run: make V=1 LOG_LEVEL=DEBUG QUICK_AND_DIRTY_COMPILER=1 v2
@@ -99,27 +91,19 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v3
-        with:
-          submodules: recursive
 
-      - name: Get submodule versions
+      - name: Get submodules hash
         id: submodules
-        shell: bash
         run: |
-          echo "nim-hash=$(git rev-parse HEAD:vendor/nimbus-build-system)" >> $GITHUB_OUTPUT
-          echo "zerokit-hash=$(git rev-parse HEAD:vendor/zerokit)" >> $GITHUB_OUTPUT
+          echo "hash=$(git submodule status | sha256sum | sed 's/[ -]*//g')" >> $GITHUB_OUTPUT
 
-      - name: Cache nim compiler
+      - name: Cache submodules
         uses: actions/cache@v3
         with:
-          path: vendor/nimbus-build-system/vendor/Nim/bin
-          key: ${{ runner.os }}-nim-${{ steps.submodules.outputs.nim-hash }}
-
-      - name: Cache zerokit artifacts
-        uses: actions/cache@v3
-        with:
-          path: vendor/zerokit/target/release
-          key: ${{ runner.os }}-zerokit-${{ steps.submodules.outputs.zerokit-hash }}
+          path: |
+            vendor/
+            .git/modules
+          key: ${{ runner.os }}-vendor-modules-${{ steps.submodules.outputs.hash }}
 
       - name: Run tests
         run: make V=1 LOG_LEVEL=DEBUG QUICK_AND_DIRTY_COMPILER=1 test2 testwakunode2

.github/workflows/ci.yml

@@ -133,20 +133,19 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v3
-        with:
-          submodules: recursive
 
-      - name: Get submodule versions
+      - name: Get submodules hash
         id: submodules
-        shell: bash
         run: |
-          echo "nim-hash=$(git rev-parse HEAD:vendor/nimbus-build-system)" >> $GITHUB_OUTPUT
+          echo "hash=$(git submodule status | sha256sum | sed 's/[ -]*//g')" >> $GITHUB_OUTPUT
 
-      - name: Cache nim
+      - name: Cache submodules
         uses: actions/cache@v3
         with:
-          path: vendor/nimbus-build-system/vendor/Nim/bin
-          key: ${{ runner.os }}-nim-${{ steps.submodules.outputs.nim-hash }}
+          path: |
+            vendor/
+            .git/modules
+          key: ${{ runner.os }}-vendor-modules-${{ steps.submodules.outputs.hash }}
 
       - name: Build binaries
         run: make V=1 LOG_LEVEL=DEBUG QUICK_AND_DIRTY_COMPILER=1 v1
@@ -164,20 +163,19 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v3
-        with:
-          submodules: recursive
 
-      - name: Get submodule vesions
+      - name: Get submodules hash
         id: submodules
-        shell: bash
         run: |
-          echo "nim-hash=$(git rev-parse HEAD:vendor/nimbus-build-system)" >> $GITHUB_OUTPUT
+          echo "hash=$(git submodule status | sha256sum | sed 's/[ -]*//g')" >> $GITHUB_OUTPUT
 
-      - name: Cache nim
+      - name: Cache submodules
         uses: actions/cache@v3
         with:
-          path: vendor/nimbus-build-system/vendor/Nim/bin
-          key: ${{ runner.os }}-nim-${{ steps.submodules.outputs.nim-hash }}
+          path: |
+            vendor/
+            .git/modules
+          key: ${{ runner.os }}-vendor-modules-${{ steps.submodules.outputs.hash }}
 
       - name: Run tests
         run: make V=1 LOG_LEVEL=DEBUG QUICK_AND_DIRTY_COMPILER=1 test1

.github/workflows/release-assets.yml

@@ -24,19 +24,18 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v2
 
-        # We need to do this because of how github cache works
-        # I am not sure we can move the cache file, so if we do not do this
-        # make update breaks because the cached compiler is there where the submodules
-        # are meant to go.
-      - name: Submodules
+      - name: Get submodules hash
+        id: submodules
         run: |
-          git submodule update --init --recursive
-          
-      - name: Cache nim
-        uses: actions/cache@v1
+          echo "hash=$(git submodule status | sha256sum | sed 's/[ -]*//g')" >> $GITHUB_OUTPUT
+
+      - name: Cache submodules
+        uses: actions/cache@v3
         with:
-          path: vendor/nimbus-build-system/vendor/Nim/bin
-          key: ${{ runner.os }}-${{ matrix.env.NPROC }}-nim-${{ hashFiles('.gitmodules') }}
+          path: |
+            vendor/
+            .git/modules
+          key: ${{ runner.os }}-vendor-modules-${{ steps.submodules.outputs.hash }}
 
       - name: Install dependencies
         run: |