# Copyright (c) HashiCorp, Inc. # SPDX-License-Identifier: MPL-2.0 name: Nightly test-integrations on: schedule: # Run nightly at 12AM UTC/8PM EST/5PM PST - cron: '0 0 * * *' workflow_dispatch: {} env: TEST_RESULTS_DIR: /tmp/test-results TEST_RESULTS_ARTIFACT_NAME: test-results CONSUL_LICENSE: ${{ secrets.CONSUL_LICENSE }} GOTAGS: ${{ endsWith(github.repository, '-enterprise') && 'consulent' || '' }} GOTESTSUM_VERSION: "1.11.0" CONSUL_BINARY_UPLOAD_NAME: consul-bin # strip the hashicorp/ off the front of github.repository for consul CONSUL_LATEST_IMAGE_NAME: ${{ endsWith(github.repository, '-enterprise') && github.repository || 'hashicorp/consul' }} GOPRIVATE: github.com/hashicorp # Required for enterprise deps jobs: setup: runs-on: ubuntu-latest name: Setup outputs: compute-small: ${{ steps.runners.outputs.compute-small }} compute-medium: ${{ steps.runners.outputs.compute-medium }} compute-large: ${{ steps.runners.outputs.compute-large }} compute-xl: ${{ steps.runners.outputs.compute-xl }} enterprise: ${{ steps.runners.outputs.enterprise }} steps: - name: Checkout code uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 with: ref: ${{ inputs.branch }} - id: runners run: .github/scripts/get_runner_classes.sh get-go-version: uses: ./.github/workflows/reusable-get-go-version.yml dev-build: needs: - setup - get-go-version uses: ./.github/workflows/reusable-dev-build.yml with: runs-on: ${{ needs.setup.outputs.compute-large }} repository-name: ${{ github.repository }} uploaded-binary-name: 'consul-bin' go-version: ${{ needs.get-go-version.outputs.go-version }} secrets: elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} generate-envoy-job-matrices: needs: [setup] runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} name: Generate Envoy Job Matrices outputs: envoy-matrix: ${{ steps.set-matrix.outputs.envoy-matrix }} steps: - name: Checkout code uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 with: ref: ${{ inputs.branch }} - name: Generate Envoy Job Matrix id: set-matrix env: # this is further going to multiplied in envoy-integration tests by the # other dimensions in the matrix. Currently TOTAL_RUNNERS would be # multiplied by 8 based on these values: # envoy-version: ["1.26.8", "1.27.5", "1.28.3", "1.29.4"] # xds-target: ["server", "client"] TOTAL_RUNNERS: 8 JQ_SLICER: '[ inputs ] | [_nwise(length / $runnercount | floor)]' run: | NUM_RUNNERS=$TOTAL_RUNNERS NUM_DIRS=$(find ./test/integration/connect/envoy -mindepth 1 -maxdepth 1 -type d | wc -l) if [ "$NUM_DIRS" -lt "$NUM_RUNNERS" ]; then echo "TOTAL_RUNNERS is larger than the number of tests/packages to split." NUM_RUNNERS=$((NUM_DIRS-1)) fi # fix issue where test splitting calculation generates 1 more split than TOTAL_RUNNERS. NUM_RUNNERS=$((NUM_RUNNERS-1)) { echo -n "envoy-matrix=" find ./test/integration/connect/envoy -maxdepth 1 -type d -print0 \ | xargs -0 -n 1 basename \ | jq --raw-input --argjson runnercount "$NUM_RUNNERS" "$JQ_SLICER" \ | jq --compact-output 'map(join("|"))' } >> "$GITHUB_OUTPUT" envoy-integration-test: runs-on: ${{ fromJSON(needs.setup.outputs.compute-large ) }} needs: - setup - get-go-version - generate-envoy-job-matrices - dev-build permissions: id-token: write # NOTE: this permission is explicitly required for Vault auth. contents: read strategy: fail-fast: false matrix: envoy-version: ["1.26.8", "1.27.6", "1.28.4", "1.29.5"] xds-target: ["server", "client"] test-cases: ${{ fromJSON(needs.generate-envoy-job-matrices.outputs.envoy-matrix) }} env: ENVOY_VERSION: ${{ matrix.envoy-version }} XDS_TARGET: ${{ matrix.xds-target }} AWS_LAMBDA_REGION: us-west-2 steps: - name: Checkout code uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 with: ref: ${{ inputs.branch }} - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 with: go-version: ${{ needs.get-go-version.outputs.go-version }} - name: fetch binary uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7 with: name: '${{ env.CONSUL_BINARY_UPLOAD_NAME }}' path: ./bin - name: restore mode+x run: chmod +x ./bin/consul - name: Set up Docker Buildx uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0 - name: Docker build run: docker build -t consul:local -f ./build-support/docker/Consul-Dev.dockerfile ./bin - name: Envoy Integration Tests env: GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml GOTESTSUM_FORMAT: standard-verbose COMPOSE_INTERACTIVE_NO_CLI: 1 LAMBDA_TESTS_ENABLED: "true" # tput complains if this isn't set to something. TERM: ansi run: | # shellcheck disable=SC2001 echo "Running $(sed 's,|, ,g' <<< "${{ matrix.test-cases }}" |wc -w) subtests" # shellcheck disable=SC2001 sed 's,|,\n,g' <<< "${{ matrix.test-cases }}" go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \ --debug \ --rerun-fails \ --rerun-fails-report=/tmp/gotestsum-rerun-fails \ --jsonfile /tmp/jsonfile/go-test.log \ --packages=./test/integration/connect/envoy \ -- -timeout=30m -tags integration -run="TestEnvoy/(${{ matrix.test-cases }})" # NOTE: ENT specific step as we store secrets in Vault. - name: Authenticate to Vault if: ${{ !cancelled() && endsWith(github.repository, '-enterprise') }} id: vault-auth run: vault-auth # NOTE: ENT specific step as we store secrets in Vault. - name: Fetch Secrets if: ${{ !cancelled() && endsWith(github.repository, '-enterprise') }} id: secrets uses: hashicorp/vault-action@v3 with: url: ${{ steps.vault-auth.outputs.addr }} caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }} token: ${{ steps.vault-auth.outputs.token }} secrets: | kv/data/github/${{ github.repository }}/datadog apikey | DATADOG_API_KEY; - name: prepare datadog-ci if: ${{ !cancelled() && !endsWith(github.repository, '-enterprise') }} run: | curl -L --fail "https://github.com/DataDog/datadog-ci/releases/latest/download/datadog-ci_linux-x64" --output "/usr/local/bin/datadog-ci" chmod +x /usr/local/bin/datadog-ci - name: upload coverage # do not run on forks if: ${{ !cancelled() && github.event.pull_request.head.repo.full_name == github.repository }} env: DATADOG_API_KEY: "${{ endsWith(github.repository, '-enterprise') && env.DATADOG_API_KEY || secrets.DATADOG_API_KEY }}" DD_ENV: ci run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" $TEST_RESULTS_DIR/results.xml upgrade-integration-test: runs-on: ${{ fromJSON(needs.setup.outputs.compute-large ) }} needs: - setup - get-go-version - dev-build permissions: id-token: write # NOTE: this permission is explicitly required for Vault auth. contents: read strategy: fail-fast: false matrix: consul-version: [ "1.17", "1.18"] env: CONSUL_LATEST_VERSION: ${{ matrix.consul-version }} # ENVOY_VERSION should be the latest version upported by all # consul versions in the matrix.consul-version, since we are testing upgrade from # an older consul version, e.g., 1.27.x is supported by both 1.17 and 1.18. ENVOY_VERSION: "1.27.6" steps: - name: Checkout code uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 with: ref: ${{ inputs.branch }} # NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos. - name: Setup Git if: ${{ endsWith(github.repository, '-enterprise') }} run: git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN }}:@github.com".insteadOf "https://github.com" - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 with: go-version: ${{ needs.get-go-version.outputs.go-version }} - run: go env # Get go binary from workspace - name: fetch binary uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7 with: name: '${{ env.CONSUL_BINARY_UPLOAD_NAME }}' path: . - name: restore mode+x run: chmod +x consul - name: Build consul:local image run: docker build -t ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local -f ./build-support/docker/Consul-Dev.dockerfile . - name: Build consul-envoy:latest-version image id: buildConsulEnvoyLatestImage run: | if ${{ endsWith(github.repository, '-enterprise') }} == 'true' then docker build -t consul-envoy:latest-version --build-arg CONSUL_IMAGE=docker.mirror.hashicorp.services/${{ env.CONSUL_LATEST_IMAGE_NAME }}:${{ env.CONSUL_LATEST_VERSION }}-ent --build-arg ENVOY_VERSION=${{ env.ENVOY_VERSION }} -f ./test/integration/consul-container/assets/Dockerfile-consul-envoy ./test/integration/consul-container/assets else docker build -t consul-envoy:latest-version --build-arg CONSUL_IMAGE=docker.mirror.hashicorp.services/${{ env.CONSUL_LATEST_IMAGE_NAME }}:${{ env.CONSUL_LATEST_VERSION }} --build-arg ENVOY_VERSION=${{ env.ENVOY_VERSION }} -f ./test/integration/consul-container/assets/Dockerfile-consul-envoy ./test/integration/consul-container/assets fi - name: Build consul-envoy:target-version image id: buildConsulEnvoyTargetImage continue-on-error: true run: docker build -t consul-envoy:target-version --build-arg CONSUL_IMAGE=${{ env.CONSUL_LATEST_IMAGE_NAME }}:local --build-arg ENVOY_VERSION=${{ env.ENVOY_VERSION }} -f ./test/integration/consul-container/assets/Dockerfile-consul-envoy ./test/integration/consul-container/assets - name: Retry Build consul-envoy:target-version image if: steps.buildConsulEnvoyTargetImage.outcome == 'failure' run: docker build -t consul-envoy:target-version --build-arg CONSUL_IMAGE=${{ env.CONSUL_LATEST_IMAGE_NAME }}:local --build-arg ENVOY_VERSION=${{ env.ENVOY_VERSION }} -f ./test/integration/consul-container/assets/Dockerfile-consul-envoy ./test/integration/consul-container/assets - name: Build sds image run: docker build -t consul-sds-server ./test/integration/connect/envoy/test-sds-server/ - name: Configure GH workaround for ipv6 loopback if: ${{ !endsWith(github.repository, '-enterprise') }} run: | cat /etc/hosts && echo "-----------" sudo sed -i 's/::1 *localhost ip6-localhost ip6-loopback/::1 ip6-localhost ip6-loopback/g' /etc/hosts cat /etc/hosts - name: Upgrade Integration Tests run: | mkdir -p "${{ env.TEST_RESULTS_DIR }}" cd ./test/integration/consul-container/test/upgrade docker run --rm ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local consul version go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \ --raw-command \ --format=github-actions \ --rerun-fails \ --packages="./..." \ -- \ go test \ -p=4 \ -tags "${{ env.GOTAGS }}" \ -timeout=30m \ -json \ ./... \ --follow-log=false \ --target-image ${{ env.CONSUL_LATEST_IMAGE_NAME }} \ --target-version local \ --latest-image docker.mirror.hashicorp.services/${{ env.CONSUL_LATEST_IMAGE_NAME }} \ --latest-version "${{ env.CONSUL_LATEST_VERSION }}" ls -lrt env: # this is needed because of incompatibility between RYUK container and GHA GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml GOTESTSUM_FORMAT: standard-verbose COMPOSE_INTERACTIVE_NO_CLI: 1 # tput complains if this isn't set to something. TERM: ansi # NOTE: ENT specific step as we store secrets in Vault. - name: Authenticate to Vault if: ${{ !cancelled() && endsWith(github.repository, '-enterprise') }} id: vault-auth run: vault-auth # NOTE: ENT specific step as we store secrets in Vault. - name: Fetch Secrets if: ${{ !cancelled() && endsWith(github.repository, '-enterprise') }} id: secrets uses: hashicorp/vault-action@v3 with: url: ${{ steps.vault-auth.outputs.addr }} caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }} token: ${{ steps.vault-auth.outputs.token }} secrets: | kv/data/github/${{ github.repository }}/datadog apikey | DATADOG_API_KEY; - name: prepare datadog-ci if: ${{ !cancelled() && !endsWith(github.repository, '-enterprise') }} run: | curl -L --fail "https://github.com/DataDog/datadog-ci/releases/latest/download/datadog-ci_linux-x64" --output "/usr/local/bin/datadog-ci" chmod +x /usr/local/bin/datadog-ci - name: upload coverage # do not run on forks if: ${{ !cancelled() && github.event.pull_request.head.repo.full_name == github.repository }} env: DATADOG_API_KEY: "${{ endsWith(github.repository, '-enterprise') && env.DATADOG_API_KEY || secrets.DATADOG_API_KEY }}" DD_ENV: ci run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" $TEST_RESULTS_DIR/results.xml upgrade-integration-test-deployer: runs-on: ${{ fromJSON(needs.setup.outputs.compute-large ) }} needs: - setup - get-go-version permissions: id-token: write # NOTE: this permission is explicitly required for Vault auth. contents: read strategy: fail-fast: false matrix: consul-version: [ "1.17", "1.18"] env: CONSUL_LATEST_VERSION: ${{ matrix.consul-version }} steps: - name: Checkout code uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 with: ref: ${{ inputs.branch }} # NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos. - name: Setup Git if: ${{ endsWith(github.repository, '-enterprise') }} run: git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN }}:@github.com".insteadOf "https://github.com" - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 with: go-version: ${{ needs.get-go-version.outputs.go-version }} - run: go env - name: Build image run: make test-deployer-setup - name: Upgrade Integration Tests run: | mkdir -p "${{ env.TEST_RESULTS_DIR }}" #export NOLOGBUFFER=1 cd ./test-integ/upgrade docker run --rm ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local consul version go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \ --raw-command \ --format=github-actions \ --packages="./..." \ -- \ go test \ -tags "${{ env.GOTAGS }}" \ -timeout=20m \ -parallel=2 \ -json \ ./... \ --target-image ${{ env.CONSUL_LATEST_IMAGE_NAME }} \ --target-version local \ --latest-image docker.mirror.hashicorp.services/${{ env.CONSUL_LATEST_IMAGE_NAME }} \ --latest-version "${{ env.CONSUL_LATEST_VERSION }}" env: # this is needed because of incompatibility between RYUK container and GHA GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml GOTESTSUM_FORMAT: standard-verbose COMPOSE_INTERACTIVE_NO_CLI: 1 # tput complains if this isn't set to something. TERM: ansi # NOTE: ENT specific step as we store secrets in Vault. - name: Authenticate to Vault if: ${{ !cancelled() && endsWith(github.repository, '-enterprise') }} id: vault-auth run: vault-auth # NOTE: ENT specific step as we store secrets in Vault. - name: Fetch Secrets if: ${{ !cancelled() && endsWith(github.repository, '-enterprise') }} id: secrets uses: hashicorp/vault-action@v3 with: url: ${{ steps.vault-auth.outputs.addr }} caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }} token: ${{ steps.vault-auth.outputs.token }} secrets: | kv/data/github/${{ github.repository }}/datadog apikey | DATADOG_API_KEY; - name: prepare datadog-ci if: ${{ !cancelled() && !endsWith(github.repository, '-enterprise') }} run: | curl -L --fail "https://github.com/DataDog/datadog-ci/releases/latest/download/datadog-ci_linux-x64" --output "/usr/local/bin/datadog-ci" chmod +x /usr/local/bin/datadog-ci - name: upload coverage # do not run on forks if: ${{ !cancelled() && github.event.pull_request.head.repo.full_name == github.repository }} env: DATADOG_API_KEY: "${{ endsWith(github.repository, '-enterprise') && env.DATADOG_API_KEY || secrets.DATADOG_API_KEY }}" DD_ENV: ci run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" $TEST_RESULTS_DIR/results.xml test-integrations-success: needs: - setup - dev-build - generate-envoy-job-matrices - envoy-integration-test - upgrade-integration-test - upgrade-integration-test-deployer runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} if: ${{ always() }} steps: - name: evaluate upstream job results run: | # exit 1 if failure or cancelled result for any upstream job if printf '${{ toJSON(needs) }}' | grep -E -i '\"result\": \"(failure|cancelled)\"'; then printf "Tests failed or workflow cancelled:\n\n${{ toJSON(needs) }}" exit 1 fi - name: Notify Slack if: ${{ failure() }} id: slack uses: slackapi/slack-github-action@70cd7be8e40a46e8b0eced40b0de447bdb42f68e # v1.26.0 with: payload: | { "message": "One or more nightly integration tests have failed. ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" } env: SLACK_WEBHOOK_URL: ${{ secrets.CONSUL_NIGHTLY_INTEG_TEST_SLACK_WEBHOOK }}