PLAT-1192 - Run CI on smaller instances (#18624)

Use smaller runners

.github/scripts/get_runner_classes.sh

@@ -14,7 +14,7 @@ case "$GITHUB_REPOSITORY" in
 	echo "compute-medium=['self-hosted', 'linux', 'medium']" >>"$GITHUB_OUTPUT"
 	echo "compute-large=['self-hosted', 'linux', 'large']" >>"$GITHUB_OUTPUT"
 	# m5d.8xlarge is equivalent to our xl custom runner in CE
-        echo "compute-xl=['self-hosted', 'ondemand', 'linux', 'type=m5d.8xlarge']" >> "$GITHUB_OUTPUT"
+	echo "compute-xl=['self-hosted', 'ondemand', 'linux', 'type=m6a.2xlarge']" >>"$GITHUB_OUTPUT"
 	;;
 *)
 	# shellcheck disable=SC2129

.github/scripts/get_runner_classes_windows.sh

@@ -14,7 +14,7 @@ case "$GITHUB_REPOSITORY" in
 	echo "compute-medium=['self-hosted', 'windows', 'medium']" >>"$GITHUB_OUTPUT"
 	echo "compute-large=['self-hosted', 'windows', 'large']" >>"$GITHUB_OUTPUT"
 	# m5d.8xlarge is equivalent to our xl custom runner in CE
-        echo "compute-xl=['self-hosted', 'ondemand', 'windows', 'type=m5d.8xlarge']" >> "$GITHUB_OUTPUT"
+	echo "compute-xl=['self-hosted', 'ondemand', 'windows', 'type=m6a.2xlarge']" >>"$GITHUB_OUTPUT"
 	;;
 *)
 	# shellcheck disable=SC2129

.github/workflows/frontend.yml

@@ -74,7 +74,7 @@ jobs:
 
   ember-build-test:
     needs: setup
-    runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }}
+    runs-on: ${{ fromJSON(needs.setup.outputs.compute-large ) }}
     strategy:
       matrix:
         partition: [1, 2, 3, 4]

.github/workflows/go-tests.yml

@@ -178,7 +178,7 @@ jobs:
     - setup
     uses: ./.github/workflows/reusable-lint.yml
     with:
-      runs-on: ${{ needs.setup.outputs.compute-xl }}
+      runs-on: ${{ needs.setup.outputs.compute-large }}
       repository-name: ${{ github.repository }}
     secrets:
       elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
@@ -189,7 +189,7 @@ jobs:
     uses: ./.github/workflows/reusable-lint.yml
     with:
       go-arch: "386"
-      runs-on: ${{ needs.setup.outputs.compute-xl }}
+      runs-on: ${{ needs.setup.outputs.compute-large }}
       repository-name: ${{ github.repository }}
     secrets:
       elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
@@ -200,7 +200,7 @@ jobs:
     - setup
     uses: ./.github/workflows/reusable-dev-build.yml
     with:
-      runs-on: ${{ needs.setup.outputs.compute-xl }}
+      runs-on: ${{ needs.setup.outputs.compute-large }}
       repository-name: ${{ github.repository }}
     secrets:
       elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
@@ -212,7 +212,7 @@ jobs:
   #   uses: ./.github/workflows/reusable-dev-build.yml
   #   with:
   #     uploaded-binary-name: 'consul-bin-s390x'
-  #     runs-on: ${{ needs.setup.outputs.compute-xl }}
+  #     runs-on: ${{ needs.setup.outputs.compute-large }}
   #     go-arch: "s390x"
   #     repository-name: ${{ github.repository }}
   #   secrets:
@@ -226,7 +226,7 @@ jobs:
   #   uses: ./.github/workflows/reusable-dev-build.yml
   #   with:
   #     uploaded-binary-name: 'consul-bin-arm64'
-  #     runs-on: ${{ needs.setup.outputs.compute-xl }}
+  #     runs-on: ${{ needs.setup.outputs.compute-large }}
   #     go-arch: "arm64"
   #     repository-name: ${{ github.repository }}
   #   secrets:
@@ -259,7 +259,7 @@ jobs:
     with:
       directory: .
       runner-count: 12
-      runs-on: ${{ needs.setup.outputs.compute-xl }}
+      runs-on: ${{ needs.setup.outputs.compute-large }}
       repository-name: ${{ github.repository }}
       go-tags: ""
     permissions:
@@ -279,7 +279,7 @@ jobs:
     with:
       directory: .
       runner-count: 12
-      runs-on: ${{ needs.setup.outputs.compute-xl }}
+      runs-on: ${{ needs.setup.outputs.compute-large }}
       repository-name: ${{ github.repository }}
       go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}"
     permissions:
@@ -299,7 +299,7 @@ jobs:
       directory: .
       go-test-flags: 'GO_TEST_FLAGS="-race -gcflags=all=-d=checkptr=0"'
       package-names-command: "go list ./... | grep -E -v '^github.com/hashicorp/consul/agent(/consul|/local|/routine-leak-checker)?$' | grep -E -v '^github.com/hashicorp/consul(/command|/connect|/snapshot)'"
-      runs-on: ${{ needs.setup.outputs.compute-xl }}
+      runs-on: ${{ needs.setup.outputs.compute-large }}
       repository-name: ${{ github.repository }}
       go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}"
     permissions:
@@ -319,7 +319,7 @@ jobs:
       directory: .
       go-arch: "386"
       go-test-flags: 'export GO_TEST_FLAGS="-short"'
-      runs-on: ${{ needs.setup.outputs.compute-xl }}
+      runs-on: ${{ needs.setup.outputs.compute-large }}
       repository-name: ${{ github.repository }}
       go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}"
     permissions:
@@ -340,7 +340,7 @@ jobs:
   #     uploaded-binary-name: 'consul-bin-s390x'
   #     directory: .
   #     go-test-flags: 'export GO_TEST_FLAGS="-short"'
-  #     runs-on: ${{ needs.setup.outputs.compute-xl }}
+  #     runs-on: ${{ needs.setup.outputs.compute-large }}
   #     repository-name: ${{ github.repository }}
   #     go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}"
   #   permissions:
@@ -358,7 +358,7 @@ jobs:
     uses: ./.github/workflows/reusable-unit.yml
     with:
       directory: envoyextensions
-      runs-on: ${{ needs.setup.outputs.compute-xl }}
+      runs-on: ${{ needs.setup.outputs.compute-large }}
       repository-name: ${{ github.repository }}
       go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}"
     permissions:
@@ -376,7 +376,7 @@ jobs:
     uses: ./.github/workflows/reusable-unit.yml
     with:
       directory: troubleshoot
-      runs-on: ${{ needs.setup.outputs.compute-xl }}
+      runs-on: ${{ needs.setup.outputs.compute-large }}
       repository-name: ${{ github.repository }}
       go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}"
     permissions:
@@ -394,7 +394,7 @@ jobs:
     uses: ./.github/workflows/reusable-unit.yml
     with:
       directory: api
-      runs-on: ${{ needs.setup.outputs.compute-xl }}
+      runs-on: ${{ needs.setup.outputs.compute-large }}
       repository-name: ${{ github.repository }}
       go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}"
       go-version: "1.19"
@@ -413,7 +413,7 @@ jobs:
     uses: ./.github/workflows/reusable-unit.yml
     with:
       directory: api
-      runs-on: ${{ needs.setup.outputs.compute-xl }}
+      runs-on: ${{ needs.setup.outputs.compute-large }}
       repository-name: ${{ github.repository }}
       go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}"
       go-version: "1.20"
@@ -432,7 +432,7 @@ jobs:
     uses: ./.github/workflows/reusable-unit.yml
     with:
       directory: sdk
-      runs-on: ${{ needs.setup.outputs.compute-xl }}
+      runs-on: ${{ needs.setup.outputs.compute-large }}
       repository-name: ${{ github.repository }}
       go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}"
       go-version: "1.19"
@@ -451,7 +451,7 @@ jobs:
     uses: ./.github/workflows/reusable-unit.yml
     with:
       directory: sdk
-      runs-on: ${{ needs.setup.outputs.compute-xl }}
+      runs-on: ${{ needs.setup.outputs.compute-large }}
       repository-name: ${{ github.repository }}
       go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}"
       go-version: "1.20"

.github/workflows/nightly-test-integrations-1.15.x.yml

@@ -91,7 +91,7 @@ jobs:
           } >> "$GITHUB_OUTPUT"
   
   envoy-integration-test:
-    runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }}
+    runs-on: ${{ fromJSON(needs.setup.outputs.compute-large) }}
     needs:
       - setup
       - generate-envoy-job-matrices
@@ -183,7 +183,7 @@ jobs:
         run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" $TEST_RESULTS_DIR/results.xml
   
   upgrade-integration-test:
-    runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }}
+    runs-on: ${{ fromJSON(needs.setup.outputs.compute-large) }}
     needs:
       - setup
       - dev-build

.github/workflows/nightly-test-integrations-1.16.x.yml

@@ -91,7 +91,7 @@ jobs:
           } >> "$GITHUB_OUTPUT"
   
   envoy-integration-test:
-    runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }}
+    runs-on: ${{ fromJSON(needs.setup.outputs.compute-large) }}
     needs:
       - setup
       - generate-envoy-job-matrices
@@ -186,7 +186,7 @@ jobs:
         run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" $TEST_RESULTS_DIR/results.xml
   
   upgrade-integration-test:
-    runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }}
+    runs-on: ${{ fromJSON(needs.setup.outputs.compute-large) }}
     needs:
       - setup
       - dev-build

.github/workflows/nightly-test-integrations.yml

@@ -42,7 +42,7 @@ jobs:
     needs: [setup]
     uses: ./.github/workflows/reusable-dev-build.yml
     with:
-      runs-on: ${{ needs.setup.outputs.compute-xl }}
+      runs-on: ${{ needs.setup.outputs.compute-large }}
       repository-name: ${{ github.repository }}
       uploaded-binary-name: 'consul-bin'
     secrets:
@@ -88,7 +88,7 @@ jobs:
           } >> "$GITHUB_OUTPUT"
   
   envoy-integration-test:
-    runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }}
+    runs-on: ${{ fromJSON(needs.setup.outputs.compute-large ) }}
     needs:
       - setup
       - generate-envoy-job-matrices
@@ -183,7 +183,7 @@ jobs:
         run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" $TEST_RESULTS_DIR/results.xml
   
   upgrade-integration-test:
-    runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }}
+    runs-on: ${{ fromJSON(needs.setup.outputs.compute-large ) }}
     needs:
       - setup
       - dev-build

.github/workflows/test-integrations-windows.yml

@@ -8,6 +8,7 @@ on:
     # * is a special character in YAML so you have to quote this string
     # Run nightly at 12AM UTC/8PM EST/5PM PST.
     - cron: '0 0 * * *'
+  workflow_dispatch:
 
 env:
   TEST_RESULTS_DIR: /tmp/test-results
@@ -38,7 +39,7 @@ jobs:
   dev-build:
     uses: ./.github/workflows/reusable-dev-build-windows.yml
     with:
-      runs-on: ${{ needs.setup.outputs.compute-xl }}
+      runs-on: ${{ needs.setup.outputs.compute-large }}
       repository-name: ${{ github.repository }}
       uploaded-binary-name: 'consul.exe'
     secrets:

.github/workflows/test-integrations.yml

@@ -63,7 +63,7 @@ jobs:
     needs: [setup]
     uses: ./.github/workflows/reusable-dev-build.yml
     with:
-      runs-on: ${{ needs.setup.outputs.compute-xl }}
+      runs-on: ${{ needs.setup.outputs.compute-large }}
       repository-name: ${{ github.repository }}
       uploaded-binary-name: 'consul-bin'
     secrets:
@@ -282,7 +282,7 @@ jobs:
           } >> "$GITHUB_OUTPUT"
   
   envoy-integration-test:
-    runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }}
+    runs-on: ${{ fromJSON(needs.setup.outputs.compute-large) }}
     needs:
       - setup
       - generate-envoy-job-matrices
@@ -374,7 +374,7 @@ jobs:
         run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" $TEST_RESULTS_DIR/results.xml
 
   compatibility-integration-test:
-    runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }}
+    runs-on: ${{ fromJSON(needs.setup.outputs.compute-large) }}
     needs:
       - setup
       - dev-build
@@ -483,97 +483,6 @@ jobs:
           DD_ENV: ci
         run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" $TEST_RESULTS_DIR/results.xml
 
-  peering_commontopo-integration-test:
-    runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }}
-    needs:
-      - setup
-      - dev-build
-    permissions:
-      id-token: write # NOTE: this permission is explicitly required for Vault auth.
-      contents: read
-    strategy:
-      fail-fast: false
-    env:
-      ENVOY_VERSION: "1.24.6"
-    steps:
-      - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
-      # NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos.
-      - name: Setup Git
-        if: ${{ endsWith(github.repository, '-enterprise') }}
-        run: git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN }}:@github.com".insteadOf "https://github.com"
-      - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1
-        with:
-          go-version-file: 'go.mod'
-      - run: go env
-
-      # Get go binary from workspace
-      - name: fetch binary
-        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
-        with:
-          name: '${{ env.CONSUL_BINARY_UPLOAD_NAME }}'
-          path: .
-      - name: restore mode+x
-        run: chmod +x consul
-      - name: Build consul:local image
-        run: docker build -t ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local -f ./build-support/docker/Consul-Dev.dockerfile .
-      - name: Peering commonTopo Integration Tests
-        run: |
-          mkdir -p "${{ env.TEST_RESULTS_DIR }}"
-          cd ./test-integ/peering_commontopo
-          docker run --rm ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local consul version
-          go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \
-            --raw-command \
-            --format=short-verbose \
-            --debug \
-            --packages="./..." \
-            -- \
-            go test \
-            -tags "${{ env.GOTAGS }}" \
-            -timeout=30m \
-            -json . \
-            --target-image ${{ env.CONSUL_LATEST_IMAGE_NAME }} \
-            --target-version local \
-            --latest-image docker.mirror.hashicorp.services/${{ env.CONSUL_LATEST_IMAGE_NAME }} \
-            --latest-version latest
-          ls -lrt
-        env:
-          # this is needed because of incompatibility between RYUK container and GHA
-          GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml
-          GOTESTSUM_FORMAT: standard-verbose
-          COMPOSE_INTERACTIVE_NO_CLI: 1
-          # tput complains if this isn't set to something.
-          TERM: ansi
-      # NOTE: ENT specific step as we store secrets in Vault.
-      - name: Authenticate to Vault
-        if: ${{ endsWith(github.repository, '-enterprise') }}
-        id: vault-auth
-        run: vault-auth
-
-      # NOTE: ENT specific step as we store secrets in Vault.
-      - name: Fetch Secrets
-        if: ${{ endsWith(github.repository, '-enterprise') }}
-        id: secrets
-        uses: hashicorp/vault-action@v2.5.0
-        with:
-          url: ${{ steps.vault-auth.outputs.addr }}
-          caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }}
-          token: ${{ steps.vault-auth.outputs.token }}
-          secrets: |
-              kv/data/github/${{ github.repository }}/datadog apikey | DATADOG_API_KEY;
-
-      - name: prepare datadog-ci
-        if: ${{ !endsWith(github.repository, '-enterprise') }}
-        run: |
-          curl -L --fail "https://github.com/DataDog/datadog-ci/releases/latest/download/datadog-ci_linux-x64" --output "/usr/local/bin/datadog-ci"
-          chmod +x /usr/local/bin/datadog-ci
-
-      - name: upload coverage
-        # do not run on forks
-        if: github.event.pull_request.head.repo.full_name == github.repository
-        env:
-          DATADOG_API_KEY: "${{ endsWith(github.repository, '-enterprise') && env.DATADOG_API_KEY || secrets.DATADOG_API_KEY }}"
-          DD_ENV: ci
-        run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" $TEST_RESULTS_DIR/results.xml
 
   test-integrations-success:
     needs: 
@@ -585,7 +494,6 @@ jobs:
     - generate-envoy-job-matrices
     - envoy-integration-test
     - compatibility-integration-test
-    - peering_commontopo-integration-test
     runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }}
     if: always() && needs.conditional-skip.outputs.trigger-ci == 'true'
     steps: