status-im
/
consul
mirror of https://github.com/status-im/consul.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

PLAT-1192 - Run CI on smaller instances (#18624) 

Use smaller runners
This commit is contained in:
Curt Bushko 2023-08-31 10:56:59 -04:00 committed by GitHub
parent 7b9e243297
commit f2ce472ae1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 60 additions and 151 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
.github/scripts/get_runner_classes.sh vendored
View File

@ -8,19 +8,19 @@
set -euo pipefail
case "$GITHUB_REPOSITORY" in
*-enterprise)
# shellcheck disable=SC2129
echo "compute-small=['self-hosted', 'linux', 'small']" >> "$GITHUB_OUTPUT"
echo "compute-medium=['self-hosted', 'linux', 'medium']" >> "$GITHUB_OUTPUT"
echo "compute-large=['self-hosted', 'linux', 'large']" >> "$GITHUB_OUTPUT"
# m5d.8xlarge is equivalent to our xl custom runner in CE
echo "compute-xl=['self-hosted', 'ondemand', 'linux', 'type=m5d.8xlarge']" >> "$GITHUB_OUTPUT"
;;
*)
# shellcheck disable=SC2129
echo "compute-small=['custom-linux-s-consul-latest']" >> "$GITHUB_OUTPUT"
echo "compute-medium=['custom-linux-m-consul-latest']" >> "$GITHUB_OUTPUT"
echo "compute-large=['custom-linux-l-consul-latest']" >> "$GITHUB_OUTPUT"
echo "compute-xl=['custom-linux-xl-consul-latest']" >> "$GITHUB_OUTPUT"
;;
*-enterprise)
# shellcheck disable=SC2129
echo "compute-small=['self-hosted', 'linux', 'small']" >>"$GITHUB_OUTPUT"
echo "compute-medium=['self-hosted', 'linux', 'medium']" >>"$GITHUB_OUTPUT"
echo "compute-large=['self-hosted', 'linux', 'large']" >>"$GITHUB_OUTPUT"
# m5d.8xlarge is equivalent to our xl custom runner in CE
echo "compute-xl=['self-hosted', 'ondemand', 'linux', 'type=m6a.2xlarge']" >>"$GITHUB_OUTPUT"
;;
*)
# shellcheck disable=SC2129
echo "compute-small=['custom-linux-s-consul-latest']" >>"$GITHUB_OUTPUT"
echo "compute-medium=['custom-linux-m-consul-latest']" >>"$GITHUB_OUTPUT"
echo "compute-large=['custom-linux-l-consul-latest']" >>"$GITHUB_OUTPUT"
echo "compute-xl=['custom-linux-xl-consul-latest']" >>"$GITHUB_OUTPUT"
;;
esac

30
.github/scripts/get_runner_classes_windows.sh vendored
View File

@ -8,19 +8,19 @@
set -euo pipefail
case "$GITHUB_REPOSITORY" in
*-enterprise)
# shellcheck disable=SC2129
echo "compute-small=['self-hosted', 'windows', 'small']" >> "$GITHUB_OUTPUT"
echo "compute-medium=['self-hosted', 'windows', 'medium']" >> "$GITHUB_OUTPUT"
echo "compute-large=['self-hosted', 'windows', 'large']" >> "$GITHUB_OUTPUT"
# m5d.8xlarge is equivalent to our xl custom runner in CE
echo "compute-xl=['self-hosted', 'ondemand', 'windows', 'type=m5d.8xlarge']" >> "$GITHUB_OUTPUT"
;;
*)
# shellcheck disable=SC2129
echo "compute-small=['windows-2019']" >> "$GITHUB_OUTPUT"
echo "compute-medium=['windows-2019']" >> "$GITHUB_OUTPUT"
echo "compute-large=['windows-2019']" >> "$GITHUB_OUTPUT"
echo "compute-xl=['windows-2019']" >> "$GITHUB_OUTPUT"
;;
*-enterprise)
# shellcheck disable=SC2129
echo "compute-small=['self-hosted', 'windows', 'small']" >>"$GITHUB_OUTPUT"
echo "compute-medium=['self-hosted', 'windows', 'medium']" >>"$GITHUB_OUTPUT"
echo "compute-large=['self-hosted', 'windows', 'large']" >>"$GITHUB_OUTPUT"
# m5d.8xlarge is equivalent to our xl custom runner in CE
echo "compute-xl=['self-hosted', 'ondemand', 'windows', 'type=m6a.2xlarge']" >>"$GITHUB_OUTPUT"
;;
*)
# shellcheck disable=SC2129
echo "compute-small=['windows-2019']" >>"$GITHUB_OUTPUT"
echo "compute-medium=['windows-2019']" >>"$GITHUB_OUTPUT"
echo "compute-large=['windows-2019']" >>"$GITHUB_OUTPUT"
echo "compute-xl=['windows-2019']" >>"$GITHUB_OUTPUT"
;;
esac

2
.github/workflows/frontend.yml vendored
View File

@ -74,7 +74,7 @@ jobs:
ember-build-test:
needs: setup
runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }}
runs-on: ${{ fromJSON(needs.setup.outputs.compute-large ) }}
strategy:
matrix:
partition: [1, 2, 3, 4]

32
.github/workflows/go-tests.yml vendored
View File

@ -178,7 +178,7 @@ jobs:
- setup
uses: ./.github/workflows/reusable-lint.yml
with:
runs-on: ${{ needs.setup.outputs.compute-xl }}
runs-on: ${{ needs.setup.outputs.compute-large }}
repository-name: ${{ github.repository }}
secrets:
elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
@ -189,7 +189,7 @@ jobs:
uses: ./.github/workflows/reusable-lint.yml
with:
go-arch: "386"
runs-on: ${{ needs.setup.outputs.compute-xl }}
runs-on: ${{ needs.setup.outputs.compute-large }}
repository-name: ${{ github.repository }}
secrets:
elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
@ -200,7 +200,7 @@ jobs:
- setup
uses: ./.github/workflows/reusable-dev-build.yml
with:
runs-on: ${{ needs.setup.outputs.compute-xl }}
runs-on: ${{ needs.setup.outputs.compute-large }}
repository-name: ${{ github.repository }}
secrets:
elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
@ -212,7 +212,7 @@ jobs:
# uses: ./.github/workflows/reusable-dev-build.yml
# with:
# uploaded-binary-name: 'consul-bin-s390x'
# runs-on: ${{ needs.setup.outputs.compute-xl }}
# runs-on: ${{ needs.setup.outputs.compute-large }}
# go-arch: "s390x"
# repository-name: ${{ github.repository }}
# secrets:
@ -226,7 +226,7 @@ jobs:
# uses: ./.github/workflows/reusable-dev-build.yml
# with:
# uploaded-binary-name: 'consul-bin-arm64'
# runs-on: ${{ needs.setup.outputs.compute-xl }}
# runs-on: ${{ needs.setup.outputs.compute-large }}
# go-arch: "arm64"
# repository-name: ${{ github.repository }}
# secrets:
@ -259,7 +259,7 @@ jobs:
with:
directory: .
runner-count: 12
runs-on: ${{ needs.setup.outputs.compute-xl }}
runs-on: ${{ needs.setup.outputs.compute-large }}
repository-name: ${{ github.repository }}
go-tags: ""
permissions:
@ -279,7 +279,7 @@ jobs:
with:
directory: .
runner-count: 12
runs-on: ${{ needs.setup.outputs.compute-xl }}
runs-on: ${{ needs.setup.outputs.compute-large }}
repository-name: ${{ github.repository }}
go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}"
permissions:
@ -299,7 +299,7 @@ jobs:
directory: .
go-test-flags: 'GO_TEST_FLAGS="-race -gcflags=all=-d=checkptr=0"'
package-names-command: "go list ./... | grep -E -v '^github.com/hashicorp/consul/agent(/consul|/local|/routine-leak-checker)?$' | grep -E -v '^github.com/hashicorp/consul(/command|/connect|/snapshot)'"
runs-on: ${{ needs.setup.outputs.compute-xl }}
runs-on: ${{ needs.setup.outputs.compute-large }}
repository-name: ${{ github.repository }}
go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}"
permissions:
@ -319,7 +319,7 @@ jobs:
directory: .
go-arch: "386"
go-test-flags: 'export GO_TEST_FLAGS="-short"'
runs-on: ${{ needs.setup.outputs.compute-xl }}
runs-on: ${{ needs.setup.outputs.compute-large }}
repository-name: ${{ github.repository }}
go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}"
permissions:
@ -340,7 +340,7 @@ jobs:
# uploaded-binary-name: 'consul-bin-s390x'
# directory: .
# go-test-flags: 'export GO_TEST_FLAGS="-short"'
# runs-on: ${{ needs.setup.outputs.compute-xl }}
# runs-on: ${{ needs.setup.outputs.compute-large }}
# repository-name: ${{ github.repository }}
# go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}"
# permissions:
@ -358,7 +358,7 @@ jobs:
uses: ./.github/workflows/reusable-unit.yml
with:
directory: envoyextensions
runs-on: ${{ needs.setup.outputs.compute-xl }}
runs-on: ${{ needs.setup.outputs.compute-large }}
repository-name: ${{ github.repository }}
go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}"
permissions:
@ -376,7 +376,7 @@ jobs:
uses: ./.github/workflows/reusable-unit.yml
with:
directory: troubleshoot
runs-on: ${{ needs.setup.outputs.compute-xl }}
runs-on: ${{ needs.setup.outputs.compute-large }}
repository-name: ${{ github.repository }}
go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}"
permissions:
@ -394,7 +394,7 @@ jobs:
uses: ./.github/workflows/reusable-unit.yml
with:
directory: api
runs-on: ${{ needs.setup.outputs.compute-xl }}
runs-on: ${{ needs.setup.outputs.compute-large }}
repository-name: ${{ github.repository }}
go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}"
go-version: "1.19"
@ -413,7 +413,7 @@ jobs:
uses: ./.github/workflows/reusable-unit.yml
with:
directory: api
runs-on: ${{ needs.setup.outputs.compute-xl }}
runs-on: ${{ needs.setup.outputs.compute-large }}
repository-name: ${{ github.repository }}
go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}"
go-version: "1.20"
@ -432,7 +432,7 @@ jobs:
uses: ./.github/workflows/reusable-unit.yml
with:
directory: sdk
runs-on: ${{ needs.setup.outputs.compute-xl }}
runs-on: ${{ needs.setup.outputs.compute-large }}
repository-name: ${{ github.repository }}
go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}"
go-version: "1.19"
@ -451,7 +451,7 @@ jobs:
uses: ./.github/workflows/reusable-unit.yml
with:
directory: sdk
runs-on: ${{ needs.setup.outputs.compute-xl }}
runs-on: ${{ needs.setup.outputs.compute-large }}
repository-name: ${{ github.repository }}
go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}"
go-version: "1.20"

4
.github/workflows/nightly-test-integrations-1.15.x.yml vendored
View File

@ -91,7 +91,7 @@ jobs:
} >> "$GITHUB_OUTPUT"
envoy-integration-test:
runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }}
runs-on: ${{ fromJSON(needs.setup.outputs.compute-large) }}
needs:
- setup
- generate-envoy-job-matrices
@ -183,7 +183,7 @@ jobs:
run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" $TEST_RESULTS_DIR/results.xml
upgrade-integration-test:
runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }}
runs-on: ${{ fromJSON(needs.setup.outputs.compute-large) }}
needs:
- setup
- dev-build

4
.github/workflows/nightly-test-integrations-1.16.x.yml vendored
View File

@ -91,7 +91,7 @@ jobs:
} >> "$GITHUB_OUTPUT"
envoy-integration-test:
runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }}
runs-on: ${{ fromJSON(needs.setup.outputs.compute-large) }}
needs:
- setup
- generate-envoy-job-matrices
@ -186,7 +186,7 @@ jobs:
run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" $TEST_RESULTS_DIR/results.xml
upgrade-integration-test:
runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }}
runs-on: ${{ fromJSON(needs.setup.outputs.compute-large) }}
needs:
- setup
- dev-build

6
.github/workflows/nightly-test-integrations.yml vendored
View File

@ -42,7 +42,7 @@ jobs:
needs: [setup]
uses: ./.github/workflows/reusable-dev-build.yml
with:
runs-on: ${{ needs.setup.outputs.compute-xl }}
runs-on: ${{ needs.setup.outputs.compute-large }}
repository-name: ${{ github.repository }}
uploaded-binary-name: 'consul-bin'
secrets:
@ -88,7 +88,7 @@ jobs:
} >> "$GITHUB_OUTPUT"
envoy-integration-test:
runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }}
runs-on: ${{ fromJSON(needs.setup.outputs.compute-large ) }}
needs:
- setup
- generate-envoy-job-matrices
@ -183,7 +183,7 @@ jobs:
run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" $TEST_RESULTS_DIR/results.xml
upgrade-integration-test:
runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }}
runs-on: ${{ fromJSON(needs.setup.outputs.compute-large ) }}
needs:
- setup
- dev-build

5
.github/workflows/test-integrations-windows.yml vendored
View File

@ -8,6 +8,7 @@ on:
# * is a special character in YAML so you have to quote this string
# Run nightly at 12AM UTC/8PM EST/5PM PST.
- cron: '0 0 * * *'
workflow_dispatch:
env:
TEST_RESULTS_DIR: /tmp/test-results
@ -38,7 +39,7 @@ jobs:
dev-build:
uses: ./.github/workflows/reusable-dev-build-windows.yml
with:
runs-on: ${{ needs.setup.outputs.compute-xl }}
runs-on: ${{ needs.setup.outputs.compute-large }}
repository-name: ${{ github.repository }}
uploaded-binary-name: 'consul.exe'
secrets:
@ -1206,4 +1207,4 @@ jobs:
if printf '${{ toJSON(needs) }}' | grep -E -i '\"result\": \"(failure|cancelled)\"'; then
printf "Tests failed or workflow cancelled:\n\n${{ toJSON(needs) }}"
exit 1
fi
fi

98
.github/workflows/test-integrations.yml vendored
View File

@ -63,7 +63,7 @@ jobs:
needs: [setup]
uses: ./.github/workflows/reusable-dev-build.yml
with:
runs-on: ${{ needs.setup.outputs.compute-xl }}
runs-on: ${{ needs.setup.outputs.compute-large }}
repository-name: ${{ github.repository }}
uploaded-binary-name: 'consul-bin'
secrets:
@ -282,7 +282,7 @@ jobs:
} >> "$GITHUB_OUTPUT"
envoy-integration-test:
runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }}
runs-on: ${{ fromJSON(needs.setup.outputs.compute-large) }}
needs:
- setup
- generate-envoy-job-matrices
@ -374,7 +374,7 @@ jobs:
run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" $TEST_RESULTS_DIR/results.xml
compatibility-integration-test:
runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }}
runs-on: ${{ fromJSON(needs.setup.outputs.compute-large) }}
needs:
- setup
- dev-build
@ -483,97 +483,6 @@ jobs:
DD_ENV: ci
run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" $TEST_RESULTS_DIR/results.xml
peering_commontopo-integration-test:
runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }}
needs:
- setup
- dev-build
permissions:
id-token: write # NOTE: this permission is explicitly required for Vault auth.
contents: read
strategy:
fail-fast: false
env:
ENVOY_VERSION: "1.24.6"
steps:
- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
# NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos.
- name: Setup Git
if: ${{ endsWith(github.repository, '-enterprise') }}
run: git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN }}:@github.com".insteadOf "https://github.com"
- uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1
with:
go-version-file: 'go.mod'
- run: go env
# Get go binary from workspace
- name: fetch binary
uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
with:
name: '${{ env.CONSUL_BINARY_UPLOAD_NAME }}'
path: .
- name: restore mode+x
run: chmod +x consul
- name: Build consul:local image
run: docker build -t ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local -f ./build-support/docker/Consul-Dev.dockerfile .
- name: Peering commonTopo Integration Tests
run: |
mkdir -p "${{ env.TEST_RESULTS_DIR }}"
cd ./test-integ/peering_commontopo
docker run --rm ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local consul version
go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \
--raw-command \
--format=short-verbose \
--debug \
--packages="./..." \
-- \
go test \
-tags "${{ env.GOTAGS }}" \
-timeout=30m \
-json . \
--target-image ${{ env.CONSUL_LATEST_IMAGE_NAME }} \
--target-version local \
--latest-image docker.mirror.hashicorp.services/${{ env.CONSUL_LATEST_IMAGE_NAME }} \
--latest-version latest
ls -lrt
env:
# this is needed because of incompatibility between RYUK container and GHA
GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml
GOTESTSUM_FORMAT: standard-verbose
COMPOSE_INTERACTIVE_NO_CLI: 1
# tput complains if this isn't set to something.
TERM: ansi
# NOTE: ENT specific step as we store secrets in Vault.
- name: Authenticate to Vault
if: ${{ endsWith(github.repository, '-enterprise') }}
id: vault-auth
run: vault-auth
# NOTE: ENT specific step as we store secrets in Vault.
- name: Fetch Secrets
if: ${{ endsWith(github.repository, '-enterprise') }}
id: secrets
uses: hashicorp/vault-action@v2.5.0
with:
url: ${{ steps.vault-auth.outputs.addr }}
caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }}
token: ${{ steps.vault-auth.outputs.token }}
secrets: |
kv/data/github/${{ github.repository }}/datadog apikey | DATADOG_API_KEY;
- name: prepare datadog-ci
if: ${{ !endsWith(github.repository, '-enterprise') }}
run: |
curl -L --fail "https://github.com/DataDog/datadog-ci/releases/latest/download/datadog-ci_linux-x64" --output "/usr/local/bin/datadog-ci"
chmod +x /usr/local/bin/datadog-ci
- name: upload coverage
# do not run on forks
if: github.event.pull_request.head.repo.full_name == github.repository
env:
DATADOG_API_KEY: "${{ endsWith(github.repository, '-enterprise') && env.DATADOG_API_KEY || secrets.DATADOG_API_KEY }}"
DD_ENV: ci
run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" $TEST_RESULTS_DIR/results.xml
test-integrations-success:
needs:
@ -585,7 +494,6 @@ jobs:
- generate-envoy-job-matrices
- envoy-integration-test
- compatibility-integration-test
- peering_commontopo-integration-test
runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }}
if: always() && needs.conditional-skip.outputs.trigger-ci == 'true'
steps: