consul/.github/workflows/nightly-test-integrations.yml
Michael Zalimeni 40ca4ad6d0
[NET-5622] build: consolidate Envoy version management (#21245)
* build: consolidate Envoy version management

Simplify Envoy version management by consolidating all runtime, build,
and CI sources of Envoy versions into a single plaintext file.

The goal of this change is to avoid common mistakes missing an update of
some Envoy versions (both in general and due to release branch
inconsistency), and enable automated Envoy version updates in the
future.

* ci: add missing ref argument for get-go-version

Supports nightly tests.
2024-07-05 14:19:23 -05:00

449 lines
20 KiB
YAML

# Copyright (c) HashiCorp, Inc.
# SPDX-License-Identifier: MPL-2.0
name: Nightly test-integrations
on:
schedule:
# Run nightly at 12AM UTC/8PM EST/5PM PST
- cron: '0 0 * * *'
workflow_dispatch: {}
env:
TEST_RESULTS_DIR: /tmp/test-results
TEST_RESULTS_ARTIFACT_NAME: test-results
CONSUL_LICENSE: ${{ secrets.CONSUL_LICENSE }}
GOTAGS: ${{ endsWith(github.repository, '-enterprise') && 'consulent' || '' }}
GOTESTSUM_VERSION: "1.11.0"
CONSUL_BINARY_UPLOAD_NAME: consul-bin
# strip the hashicorp/ off the front of github.repository for consul
CONSUL_LATEST_IMAGE_NAME: ${{ endsWith(github.repository, '-enterprise') && github.repository || 'hashicorp/consul' }}
GOPRIVATE: github.com/hashicorp # Required for enterprise deps
jobs:
setup:
runs-on: ubuntu-latest
name: Setup
outputs:
compute-small: ${{ steps.runners.outputs.compute-small }}
compute-medium: ${{ steps.runners.outputs.compute-medium }}
compute-large: ${{ steps.runners.outputs.compute-large }}
compute-xl: ${{ steps.runners.outputs.compute-xl }}
enterprise: ${{ steps.runners.outputs.enterprise }}
steps:
- name: Checkout code
uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
with:
ref: ${{ inputs.branch }}
- id: runners
run: .github/scripts/get_runner_classes.sh
get-go-version:
uses: ./.github/workflows/reusable-get-go-version.yml
get-envoy-versions:
uses: ./.github/workflows/reusable-get-envoy-versions.yml
dev-build:
needs:
- setup
- get-go-version
uses: ./.github/workflows/reusable-dev-build.yml
with:
runs-on: ${{ needs.setup.outputs.compute-large }}
repository-name: ${{ github.repository }}
uploaded-binary-name: 'consul-bin'
go-version: ${{ needs.get-go-version.outputs.go-version }}
secrets:
elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
generate-envoy-job-matrices:
needs:
- setup
- get-envoy-versions
runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }}
name: Generate Envoy Job Matrices
outputs:
envoy-matrix: ${{ steps.set-matrix.outputs.envoy-matrix }}
steps:
- name: Checkout code
uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
with:
ref: ${{ inputs.branch }}
- name: Generate Envoy Job Matrix
id: set-matrix
env:
# TEST_SPLITS sets the number of test case splits to use in the matrix. This will be
# further multiplied in envoy-integration tests by the other dimensions in the matrix
# to determine the total number of runners used.
TEST_SPLITS: 4
JQ_SLICER: '[ inputs ] | [_nwise(length / $runnercount | floor)]'
run: |
NUM_DIRS=$(find ./test/integration/connect/envoy -mindepth 1 -maxdepth 1 -type d | wc -l)
if [ "$NUM_DIRS" -lt "$TEST_SPLITS" ]; then
echo "TEST_SPLITS is larger than the number of tests/packages to split."
TEST_SPLITS=$((NUM_DIRS-1))
fi
# fix issue where test splitting calculation generates 1 more split than TEST_SPLITS.
TEST_SPLITS=$((TEST_SPLITS-1))
{
echo -n "envoy-matrix="
find ./test/integration/connect/envoy -maxdepth 1 -type d -print0 \
| xargs -0 -n 1 basename \
| jq --raw-input --argjson runnercount "$TEST_SPLITS" "$JQ_SLICER" \
| jq --compact-output 'map(join("|"))'
} >> "$GITHUB_OUTPUT"
envoy-integration-test:
runs-on: ${{ fromJSON(needs.setup.outputs.compute-large ) }}
needs:
- setup
- get-go-version
- get-envoy-versions
- generate-envoy-job-matrices
- dev-build
permissions:
id-token: write # NOTE: this permission is explicitly required for Vault auth.
contents: read
strategy:
fail-fast: false
matrix:
envoy-version: ${{ fromJSON(needs.get-envoy-versions.outputs.envoy-versions-json) }}
xds-target: ["server", "client"]
test-cases: ${{ fromJSON(needs.generate-envoy-job-matrices.outputs.envoy-matrix) }}
env:
ENVOY_VERSION: ${{ matrix.envoy-version }}
XDS_TARGET: ${{ matrix.xds-target }}
AWS_LAMBDA_REGION: us-west-2
steps:
- name: Checkout code
uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
with:
ref: ${{ inputs.branch }}
- uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
with:
go-version: ${{ needs.get-go-version.outputs.go-version }}
- name: fetch binary
uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
with:
name: '${{ env.CONSUL_BINARY_UPLOAD_NAME }}'
path: ./bin
- name: restore mode+x
run: chmod +x ./bin/consul
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0
- name: Docker build
run: docker build -t consul:local -f ./build-support/docker/Consul-Dev.dockerfile ./bin
- name: Envoy Integration Tests
env:
GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml
GOTESTSUM_FORMAT: standard-verbose
COMPOSE_INTERACTIVE_NO_CLI: 1
LAMBDA_TESTS_ENABLED: "true"
# tput complains if this isn't set to something.
TERM: ansi
run: |
# shellcheck disable=SC2001
echo "Running $(sed 's,|, ,g' <<< "${{ matrix.test-cases }}" |wc -w) subtests"
# shellcheck disable=SC2001
sed 's,|,\n,g' <<< "${{ matrix.test-cases }}"
go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \
--debug \
--rerun-fails \
--rerun-fails-report=/tmp/gotestsum-rerun-fails \
--jsonfile /tmp/jsonfile/go-test.log \
--packages=./test/integration/connect/envoy \
-- -timeout=30m -tags integration -run="TestEnvoy/(${{ matrix.test-cases }})"
# NOTE: ENT specific step as we store secrets in Vault.
- name: Authenticate to Vault
if: ${{ !cancelled() && endsWith(github.repository, '-enterprise') }}
id: vault-auth
run: vault-auth
# NOTE: ENT specific step as we store secrets in Vault.
- name: Fetch Secrets
if: ${{ !cancelled() && endsWith(github.repository, '-enterprise') }}
id: secrets
uses: hashicorp/vault-action@v3
with:
url: ${{ steps.vault-auth.outputs.addr }}
caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }}
token: ${{ steps.vault-auth.outputs.token }}
secrets: |
kv/data/github/${{ github.repository }}/datadog apikey | DATADOG_API_KEY;
- name: prepare datadog-ci
if: ${{ !cancelled() && !endsWith(github.repository, '-enterprise') }}
run: |
curl -L --fail "https://github.com/DataDog/datadog-ci/releases/latest/download/datadog-ci_linux-x64" --output "/usr/local/bin/datadog-ci"
chmod +x /usr/local/bin/datadog-ci
- name: upload coverage
# do not run on forks
if: ${{ !cancelled() && github.event.pull_request.head.repo.full_name == github.repository }}
env:
DATADOG_API_KEY: "${{ endsWith(github.repository, '-enterprise') && env.DATADOG_API_KEY || secrets.DATADOG_API_KEY }}"
DD_ENV: ci
run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" $TEST_RESULTS_DIR/results.xml
upgrade-integration-test:
runs-on: ${{ fromJSON(needs.setup.outputs.compute-large ) }}
needs:
- setup
- get-go-version
- dev-build
permissions:
id-token: write # NOTE: this permission is explicitly required for Vault auth.
contents: read
strategy:
fail-fast: false
matrix:
consul-version: ["1.17", "1.18", "1.19"]
env:
CONSUL_LATEST_VERSION: ${{ matrix.consul-version }}
# ENVOY_VERSION should be the latest version supported by _all_ Consul versions in the
# matrix.consul-version, since we are testing upgrade from an older Consul version.
# In practice, this should be the highest Envoy version supported by the lowest non-LTS
# Consul version in the matrix (LTS versions receive additional Envoy version support).
#
# This value should be kept current in new nightly test workflows, and updated any time
# a new major Envoy release is added to the set supported by Consul versions in
# matrix.consul-version (i.e. whenever the highest common Envoy version across active
# Consul versions changes). The minor Envoy version does not necessarily need to be
# kept current for the purpose of these tests, but the major (1.N) version should be.
ENVOY_VERSION: 1.27.6
steps:
- name: Checkout code
uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
with:
ref: ${{ inputs.branch }}
# NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos.
- name: Setup Git
if: ${{ endsWith(github.repository, '-enterprise') }}
run: git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN }}:@github.com".insteadOf "https://github.com"
- uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
with:
go-version: ${{ needs.get-go-version.outputs.go-version }}
- run: go env
# Get go binary from workspace
- name: fetch binary
uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
with:
name: '${{ env.CONSUL_BINARY_UPLOAD_NAME }}'
path: .
- name: restore mode+x
run: chmod +x consul
- name: Build consul:local image
run: docker build -t ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local -f ./build-support/docker/Consul-Dev.dockerfile .
- name: Build consul-envoy:latest-version image
id: buildConsulEnvoyLatestImage
run: |
if ${{ endsWith(github.repository, '-enterprise') }} == 'true'
then
docker build -t consul-envoy:latest-version --build-arg CONSUL_IMAGE=docker.mirror.hashicorp.services/${{ env.CONSUL_LATEST_IMAGE_NAME }}:${{ env.CONSUL_LATEST_VERSION }}-ent --build-arg ENVOY_VERSION=${{ env.ENVOY_VERSION }} -f ./test/integration/consul-container/assets/Dockerfile-consul-envoy ./test/integration/consul-container/assets
else
docker build -t consul-envoy:latest-version --build-arg CONSUL_IMAGE=docker.mirror.hashicorp.services/${{ env.CONSUL_LATEST_IMAGE_NAME }}:${{ env.CONSUL_LATEST_VERSION }} --build-arg ENVOY_VERSION=${{ env.ENVOY_VERSION }} -f ./test/integration/consul-container/assets/Dockerfile-consul-envoy ./test/integration/consul-container/assets
fi
- name: Build consul-envoy:target-version image
id: buildConsulEnvoyTargetImage
continue-on-error: true
run: docker build -t consul-envoy:target-version --build-arg CONSUL_IMAGE=${{ env.CONSUL_LATEST_IMAGE_NAME }}:local --build-arg ENVOY_VERSION=${{ env.ENVOY_VERSION }} -f ./test/integration/consul-container/assets/Dockerfile-consul-envoy ./test/integration/consul-container/assets
- name: Retry Build consul-envoy:target-version image
if: steps.buildConsulEnvoyTargetImage.outcome == 'failure'
run: docker build -t consul-envoy:target-version --build-arg CONSUL_IMAGE=${{ env.CONSUL_LATEST_IMAGE_NAME }}:local --build-arg ENVOY_VERSION=${{ env.ENVOY_VERSION }} -f ./test/integration/consul-container/assets/Dockerfile-consul-envoy ./test/integration/consul-container/assets
- name: Build sds image
run: docker build -t consul-sds-server ./test/integration/connect/envoy/test-sds-server/
- name: Configure GH workaround for ipv6 loopback
if: ${{ !endsWith(github.repository, '-enterprise') }}
run: |
cat /etc/hosts && echo "-----------"
sudo sed -i 's/::1 *localhost ip6-localhost ip6-loopback/::1 ip6-localhost ip6-loopback/g' /etc/hosts
cat /etc/hosts
- name: Upgrade Integration Tests
run: |
mkdir -p "${{ env.TEST_RESULTS_DIR }}"
cd ./test/integration/consul-container/test/upgrade
docker run --rm ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local consul version
go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \
--raw-command \
--format=github-actions \
--rerun-fails \
--packages="./..." \
-- \
go test \
-p=4 \
-tags "${{ env.GOTAGS }}" \
-timeout=30m \
-json \
./... \
--follow-log=false \
--target-image ${{ env.CONSUL_LATEST_IMAGE_NAME }} \
--target-version local \
--latest-image docker.mirror.hashicorp.services/${{ env.CONSUL_LATEST_IMAGE_NAME }} \
--latest-version "${{ env.CONSUL_LATEST_VERSION }}"
ls -lrt
env:
# this is needed because of incompatibility between RYUK container and GHA
GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml
GOTESTSUM_FORMAT: standard-verbose
COMPOSE_INTERACTIVE_NO_CLI: 1
# tput complains if this isn't set to something.
TERM: ansi
# NOTE: ENT specific step as we store secrets in Vault.
- name: Authenticate to Vault
if: ${{ !cancelled() && endsWith(github.repository, '-enterprise') }}
id: vault-auth
run: vault-auth
# NOTE: ENT specific step as we store secrets in Vault.
- name: Fetch Secrets
if: ${{ !cancelled() && endsWith(github.repository, '-enterprise') }}
id: secrets
uses: hashicorp/vault-action@v3
with:
url: ${{ steps.vault-auth.outputs.addr }}
caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }}
token: ${{ steps.vault-auth.outputs.token }}
secrets: |
kv/data/github/${{ github.repository }}/datadog apikey | DATADOG_API_KEY;
- name: prepare datadog-ci
if: ${{ !cancelled() && !endsWith(github.repository, '-enterprise') }}
run: |
curl -L --fail "https://github.com/DataDog/datadog-ci/releases/latest/download/datadog-ci_linux-x64" --output "/usr/local/bin/datadog-ci"
chmod +x /usr/local/bin/datadog-ci
- name: upload coverage
# do not run on forks
if: ${{ !cancelled() && github.event.pull_request.head.repo.full_name == github.repository }}
env:
DATADOG_API_KEY: "${{ endsWith(github.repository, '-enterprise') && env.DATADOG_API_KEY || secrets.DATADOG_API_KEY }}"
DD_ENV: ci
run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" $TEST_RESULTS_DIR/results.xml
upgrade-integration-test-deployer:
runs-on: ${{ fromJSON(needs.setup.outputs.compute-large ) }}
needs:
- setup
- get-go-version
permissions:
id-token: write # NOTE: this permission is explicitly required for Vault auth.
contents: read
strategy:
fail-fast: false
matrix:
consul-version: [ "1.17", "1.18"]
env:
CONSUL_LATEST_VERSION: ${{ matrix.consul-version }}
steps:
- name: Checkout code
uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
with:
ref: ${{ inputs.branch }}
# NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos.
- name: Setup Git
if: ${{ endsWith(github.repository, '-enterprise') }}
run: git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN }}:@github.com".insteadOf "https://github.com"
- uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
with:
go-version: ${{ needs.get-go-version.outputs.go-version }}
- run: go env
- name: Build image
run: make test-deployer-setup
- name: Upgrade Integration Tests
run: |
mkdir -p "${{ env.TEST_RESULTS_DIR }}"
#export NOLOGBUFFER=1
cd ./test-integ/upgrade
docker run --rm ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local consul version
go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \
--raw-command \
--format=github-actions \
--packages="./..." \
-- \
go test \
-tags "${{ env.GOTAGS }}" \
-timeout=20m \
-parallel=2 \
-json \
./... \
--target-image ${{ env.CONSUL_LATEST_IMAGE_NAME }} \
--target-version local \
--latest-image docker.mirror.hashicorp.services/${{ env.CONSUL_LATEST_IMAGE_NAME }} \
--latest-version "${{ env.CONSUL_LATEST_VERSION }}"
env:
# this is needed because of incompatibility between RYUK container and GHA
GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml
GOTESTSUM_FORMAT: standard-verbose
COMPOSE_INTERACTIVE_NO_CLI: 1
# tput complains if this isn't set to something.
TERM: ansi
# NOTE: ENT specific step as we store secrets in Vault.
- name: Authenticate to Vault
if: ${{ !cancelled() && endsWith(github.repository, '-enterprise') }}
id: vault-auth
run: vault-auth
# NOTE: ENT specific step as we store secrets in Vault.
- name: Fetch Secrets
if: ${{ !cancelled() && endsWith(github.repository, '-enterprise') }}
id: secrets
uses: hashicorp/vault-action@v3
with:
url: ${{ steps.vault-auth.outputs.addr }}
caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }}
token: ${{ steps.vault-auth.outputs.token }}
secrets: |
kv/data/github/${{ github.repository }}/datadog apikey | DATADOG_API_KEY;
- name: prepare datadog-ci
if: ${{ !cancelled() && !endsWith(github.repository, '-enterprise') }}
run: |
curl -L --fail "https://github.com/DataDog/datadog-ci/releases/latest/download/datadog-ci_linux-x64" --output "/usr/local/bin/datadog-ci"
chmod +x /usr/local/bin/datadog-ci
- name: upload coverage
# do not run on forks
if: ${{ !cancelled() && github.event.pull_request.head.repo.full_name == github.repository }}
env:
DATADOG_API_KEY: "${{ endsWith(github.repository, '-enterprise') && env.DATADOG_API_KEY || secrets.DATADOG_API_KEY }}"
DD_ENV: ci
run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" $TEST_RESULTS_DIR/results.xml
test-integrations-success:
needs:
- setup
- dev-build
- generate-envoy-job-matrices
- envoy-integration-test
- upgrade-integration-test
- upgrade-integration-test-deployer
runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }}
if: ${{ always() }}
steps:
- name: evaluate upstream job results
run: |
# exit 1 if failure or cancelled result for any upstream job
if printf '${{ toJSON(needs) }}' | grep -E -i '\"result\": \"(failure|cancelled)\"'; then
printf "Tests failed or workflow cancelled:\n\n${{ toJSON(needs) }}"
exit 1
fi
- name: Notify Slack
if: ${{ failure() }}
id: slack
uses: slackapi/slack-github-action@70cd7be8e40a46e8b0eced40b0de447bdb42f68e # v1.26.0
with:
payload: |
{
"message": "One or more nightly integration tests have failed. ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
}
env:
SLACK_WEBHOOK_URL: ${{ secrets.CONSUL_NIGHTLY_INTEG_TEST_SLACK_WEBHOOK }}