Revert "ci: remove test splitting for compatibility tests (#17166)" (#17262)

This reverts commit 861a8151d5.

.github/workflows/test-integrations.yml

@@ -237,11 +237,47 @@ jobs:
           name: ${{ env.TEST_RESULTS_ARTIFACT_NAME }}
           path: ${{ env.TEST_RESULTS_DIR }}
 
+  generate-compatibility-job-matrices:
+    needs: [setup]
+    runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }}
+    name: Generate Compatibility Job Matrices
+    outputs:
+      compatibility-matrix: ${{ steps.set-matrix.outputs.compatibility-matrix }}
+    steps:
+      - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0
+      - name: Generate Compatibility Job Matrix
+        id: set-matrix
+        env:
+          TOTAL_RUNNERS: 6
+          JQ_SLICER: '[ inputs ] | [_nwise(length / $runnercount | floor)]'
+        run: |
+          cd ./test/integration/consul-container
+          NUM_RUNNERS=$TOTAL_RUNNERS
+          NUM_DIRS=$(find ./test -mindepth 1 -maxdepth 2 -type d | wc -l)
+
+          if [ "$NUM_DIRS" -lt "$NUM_RUNNERS" ]; then
+            echo "TOTAL_RUNNERS is larger than the number of tests/packages to split."
+            NUM_RUNNERS=$((NUM_DIRS-1))
+          fi
+          # fix issue where test splitting calculation generates 1 more split than TOTAL_RUNNERS.
+          NUM_RUNNERS=$((NUM_RUNNERS-1))
+          {
+            echo -n "compatibility-matrix="
+            find ./test -maxdepth 2 -type d -print0 | xargs -0 -n 1 \
+              | grep -v util | grep -v upgrade \
+              | jq --raw-input --argjson runnercount "$NUM_RUNNERS" "$JQ_SLICER" \
+              | jq --compact-output 'map(join(" "))'
+          } >> "$GITHUB_OUTPUT"
   compatibility-integration-test:
     runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }}
     needs:
       - setup
       - dev-build
+      - generate-compatibility-job-matrices
+    strategy:
+      fail-fast: false
+      matrix:
+        test-cases: ${{ fromJSON(needs.generate-compatibility-job-matrices.outputs.compatibility-matrix) }}
     steps:
       - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0
       - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0
@@ -271,9 +307,12 @@ jobs:
             mkdir -p "/tmp/test-results"
             cd ./test/integration/consul-container
             docker run --rm ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local consul version
+            echo "Running $(sed 's,|, ,g' <<< "${{ matrix.test-cases }}" |wc -w) subtests"
+            # shellcheck disable=SC2001
+            sed 's, ,\n,g' <<< "${{ matrix.test-cases }}"
             go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \
               --raw-command \
-              --format=standard-verbose \
+              --format=short-verbose \
               --debug \
               --rerun-fails=3 \
               -- \
@@ -282,7 +321,7 @@ jobs:
               -tags "${{ env.GOTAGS }}" \
               -timeout=30m \
               -json \
-              `go list ./... | grep -v upgrade` \
+              ${{ matrix.test-cases }} \
               --target-image ${{ env.CONSUL_LATEST_IMAGE_NAME }} \
               --target-version local \
               --latest-image docker.mirror.hashicorp.services/${{ env.CONSUL_LATEST_IMAGE_NAME }} \
@@ -383,6 +422,7 @@ jobs:
             --raw-command \
             --format=short-verbose \
             --debug \
+            --rerun-fails=3 \
             --packages="./..." \
             -- \
             go test \
@@ -416,6 +456,7 @@ jobs:
     - vault-integration-test
     - generate-envoy-job-matrices
     - envoy-integration-test
+    - generate-compatibility-job-matrices
     - compatibility-integration-test
     - generate-upgrade-job-matrices
     - upgrade-integration-test