From d3837e389cde0e59bcf51da61de4273eae14330f Mon Sep 17 00:00:00 2001 From: cskh Date: Fri, 18 Aug 2023 10:56:33 -0400 Subject: [PATCH] CI Split integration tests to run nightly and every PR (#18518) * CI Split integration tests to run nightly and every PR * Checkout release branch for nightly test --- .../workflows/nightly-test-integrations.yml | 326 ++++++++++++++++++ .github/workflows/test-integrations.yml | 124 +------ 2 files changed, 327 insertions(+), 123 deletions(-) create mode 100644 .github/workflows/nightly-test-integrations.yml diff --git a/.github/workflows/nightly-test-integrations.yml b/.github/workflows/nightly-test-integrations.yml new file mode 100644 index 0000000000..4e15221944 --- /dev/null +++ b/.github/workflows/nightly-test-integrations.yml @@ -0,0 +1,326 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +name: Nightly test-integrations + +on: + schedule: + # Run nightly at 12AM UTC/8PM EST/5PM PST + - cron: '* 0 * * *' + workflow_dispatch: {} + +env: + TEST_RESULTS_DIR: /tmp/test-results + TEST_RESULTS_ARTIFACT_NAME: test-results + CONSUL_LICENSE: ${{ secrets.CONSUL_LICENSE }} + GOTAGS: ${{ endsWith(github.repository, '-enterprise') && 'consulent' || '' }} + GOTESTSUM_VERSION: "1.10.1" + CONSUL_BINARY_UPLOAD_NAME: consul-bin + # strip the hashicorp/ off the front of github.repository for consul + CONSUL_LATEST_IMAGE_NAME: ${{ endsWith(github.repository, '-enterprise') && github.repository || 'hashicorp/consul' }} + GOPRIVATE: github.com/hashicorp # Required for enterprise deps + +jobs: + setup: + runs-on: ubuntu-latest + name: Setup + outputs: + compute-small: ${{ steps.runners.outputs.compute-small }} + compute-medium: ${{ steps.runners.outputs.compute-medium }} + compute-large: ${{ steps.runners.outputs.compute-large }} + compute-xl: ${{ steps.runners.outputs.compute-xl }} + enterprise: ${{ steps.runners.outputs.enterprise }} + steps: + - name: Checkout code + uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + with: + ref: ${{ inputs.branch }} + - id: runners + run: .github/scripts/get_runner_classes.sh + + dev-build: + needs: [setup] + uses: ./.github/workflows/reusable-dev-build.yml + with: + runs-on: ${{ needs.setup.outputs.compute-xl }} + repository-name: ${{ github.repository }} + uploaded-binary-name: 'consul-bin' + secrets: + elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} + + generate-envoy-job-matrices: + needs: [setup] + runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} + name: Generate Envoy Job Matrices + outputs: + envoy-matrix: ${{ steps.set-matrix.outputs.envoy-matrix }} + steps: + - name: Checkout code + uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + with: + ref: ${{ inputs.branch }} + - name: Generate Envoy Job Matrix + id: set-matrix + env: + # this is further going to multiplied in envoy-integration tests by the + # other dimensions in the matrix. Currently TOTAL_RUNNERS would be + # multiplied by 8 based on these values: + # envoy-version: ["1.24.10", "1.25.9", "1.26.4", "1.27.0"] + # xds-target: ["server", "client"] + TOTAL_RUNNERS: 4 + JQ_SLICER: '[ inputs ] | [_nwise(length / $runnercount | floor)]' + run: | + NUM_RUNNERS=$TOTAL_RUNNERS + NUM_DIRS=$(find ./test/integration/connect/envoy -mindepth 1 -maxdepth 1 -type d | wc -l) + + if [ "$NUM_DIRS" -lt "$NUM_RUNNERS" ]; then + echo "TOTAL_RUNNERS is larger than the number of tests/packages to split." + NUM_RUNNERS=$((NUM_DIRS-1)) + fi + # fix issue where test splitting calculation generates 1 more split than TOTAL_RUNNERS. + NUM_RUNNERS=$((NUM_RUNNERS-1)) + { + echo -n "envoy-matrix=" + find ./test/integration/connect/envoy -maxdepth 1 -type d -print0 \ + | xargs -0 -n 1 basename \ + | jq --raw-input --argjson runnercount "$NUM_RUNNERS" "$JQ_SLICER" \ + | jq --compact-output 'map(join("|"))' + } >> "$GITHUB_OUTPUT" + + envoy-integration-test: + runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }} + needs: + - setup + - generate-envoy-job-matrices + - dev-build + permissions: + id-token: write # NOTE: this permission is explicitly required for Vault auth. + contents: read + strategy: + fail-fast: false + matrix: + envoy-version: ["1.24.10", "1.25.9", "1.26.4", "1.27.0"] + xds-target: ["server", "client"] + test-cases: ${{ fromJSON(needs.generate-envoy-job-matrices.outputs.envoy-matrix) }} + env: + ENVOY_VERSION: ${{ matrix.envoy-version }} + XDS_TARGET: ${{ matrix.xds-target }} + AWS_LAMBDA_REGION: us-west-2 + steps: + - name: Checkout code + uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + with: + ref: ${{ inputs.branch }} + - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 + with: + go-version-file: 'go.mod' + + - name: fetch binary + uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + with: + name: '${{ env.CONSUL_BINARY_UPLOAD_NAME }}' + path: ./bin + - name: restore mode+x + run: chmod +x ./bin/consul + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@2a1a44ac4aa01993040736bd95bb470da1a38365 # v2.9.0 + + - name: Docker build + run: docker build -t consul:local -f ./build-support/docker/Consul-Dev.dockerfile ./bin + + - name: Envoy Integration Tests + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running $(sed 's,|, ,g' <<< "${{ matrix.test-cases }}" |wc -w) subtests" + # shellcheck disable=SC2001 + sed 's,|,\n,g' <<< "${{ matrix.test-cases }}" + go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \ + --debug \ + --rerun-fails \ + --rerun-fails-report=/tmp/gotestsum-rerun-fails \ + --jsonfile /tmp/jsonfile/go-test.log \ + --packages=./test/integration/connect/envoy \ + -- -timeout=30m -tags integration -run="TestEnvoy/(${{ matrix.test-cases }})" + + # NOTE: ENT specific step as we store secrets in Vault. + - name: Authenticate to Vault + if: ${{ endsWith(github.repository, '-enterprise') }} + id: vault-auth + run: vault-auth + + # NOTE: ENT specific step as we store secrets in Vault. + - name: Fetch Secrets + if: ${{ endsWith(github.repository, '-enterprise') }} + id: secrets + uses: hashicorp/vault-action@v2.5.0 + with: + url: ${{ steps.vault-auth.outputs.addr }} + caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }} + token: ${{ steps.vault-auth.outputs.token }} + secrets: | + kv/data/github/${{ github.repository }}/datadog apikey | DATADOG_API_KEY; + + - name: prepare datadog-ci + if: ${{ !endsWith(github.repository, '-enterprise') }} + run: | + curl -L --fail "https://github.com/DataDog/datadog-ci/releases/latest/download/datadog-ci_linux-x64" --output "/usr/local/bin/datadog-ci" + chmod +x /usr/local/bin/datadog-ci + + - name: upload coverage + # do not run on forks + if: github.event.pull_request.head.repo.full_name == github.repository + env: + DATADOG_API_KEY: "${{ endsWith(github.repository, '-enterprise') && env.DATADOG_API_KEY || secrets.DATADOG_API_KEY }}" + DD_ENV: ci + run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" $TEST_RESULTS_DIR/results.xml + + upgrade-integration-test: + runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }} + needs: + - setup + - dev-build + permissions: + id-token: write # NOTE: this permission is explicitly required for Vault auth. + contents: read + strategy: + fail-fast: false + matrix: + consul-version: [ "1.15", "1.16"] + env: + CONSUL_LATEST_VERSION: ${{ matrix.consul-version }} + ENVOY_VERSION: "1.24.6" + steps: + - name: Checkout code + uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + with: + ref: ${{ inputs.branch }} + # NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos. + - name: Setup Git + if: ${{ endsWith(github.repository, '-enterprise') }} + run: git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN }}:@github.com".insteadOf "https://github.com" + - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 + with: + go-version-file: 'go.mod' + - run: go env + + # Get go binary from workspace + - name: fetch binary + uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + with: + name: '${{ env.CONSUL_BINARY_UPLOAD_NAME }}' + path: . + - name: restore mode+x + run: chmod +x consul + - name: Build consul:local image + run: docker build -t ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local -f ./build-support/docker/Consul-Dev.dockerfile . + - name: Build consul-envoy:latest-version image + id: buildConsulEnvoyLatestImage + continue-on-error: true + run: docker build -t consul-envoy:latest-version --build-arg CONSUL_IMAGE=docker.mirror.hashicorp.services/${{ env.CONSUL_LATEST_IMAGE_NAME }}:${{ env.CONSUL_LATEST_VERSION }} --build-arg ENVOY_VERSION=${{ env.ENVOY_VERSION }} -f ./test/integration/consul-container/assets/Dockerfile-consul-envoy ./test/integration/consul-container/assets + - name: Retry Build consul-envoy:latest-version image + if: steps.buildConsulEnvoyLatestImage.outcome == 'failure' + run: docker build -t consul-envoy:latest-version --build-arg CONSUL_IMAGE=docker.mirror.hashicorp.services/${{ env.CONSUL_LATEST_IMAGE_NAME }}:${{ env.CONSUL_LATEST_VERSION }} --build-arg ENVOY_VERSION=${{ env.ENVOY_VERSION }} -f ./test/integration/consul-container/assets/Dockerfile-consul-envoy ./test/integration/consul-container/assets + - name: Build consul-envoy:target-version image + id: buildConsulEnvoyTargetImage + continue-on-error: true + run: docker build -t consul-envoy:target-version --build-arg CONSUL_IMAGE=${{ env.CONSUL_LATEST_IMAGE_NAME }}:local --build-arg ENVOY_VERSION=${{ env.ENVOY_VERSION }} -f ./test/integration/consul-container/assets/Dockerfile-consul-envoy ./test/integration/consul-container/assets + - name: Retry Build consul-envoy:target-version image + if: steps.buildConsulEnvoyTargetImage.outcome == 'failure' + run: docker build -t consul-envoy:target-version --build-arg CONSUL_IMAGE=${{ env.CONSUL_LATEST_IMAGE_NAME }}:local --build-arg ENVOY_VERSION=${{ env.ENVOY_VERSION }} -f ./test/integration/consul-container/assets/Dockerfile-consul-envoy ./test/integration/consul-container/assets + - name: Build sds image + run: docker build -t consul-sds-server ./test/integration/connect/envoy/test-sds-server/ + - name: Configure GH workaround for ipv6 loopback + if: ${{ !endsWith(github.repository, '-enterprise') }} + run: | + cat /etc/hosts && echo "-----------" + sudo sed -i 's/::1 *localhost ip6-localhost ip6-loopback/::1 ip6-localhost ip6-loopback/g' /etc/hosts + cat /etc/hosts + - name: Upgrade Integration Tests + run: | + mkdir -p "${{ env.TEST_RESULTS_DIR }}" + cd ./test/integration/consul-container/test/upgrade + docker run --rm ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local consul version + go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \ + --raw-command \ + --format=short-verbose \ + --debug \ + --rerun-fails=2 \ + --packages="./..." \ + -- \ + go test \ + -p=4 \ + -tags "${{ env.GOTAGS }}" \ + -timeout=30m \ + -json \ + ./... \ + --follow-log=false \ + --target-image ${{ env.CONSUL_LATEST_IMAGE_NAME }} \ + --target-version local \ + --latest-image docker.mirror.hashicorp.services/${{ env.CONSUL_LATEST_IMAGE_NAME }} \ + --latest-version "${{ env.CONSUL_LATEST_VERSION }}" + ls -lrt + env: + # this is needed because of incompatibility between RYUK container and GHA + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + # tput complains if this isn't set to something. + TERM: ansi + # NOTE: ENT specific step as we store secrets in Vault. + - name: Authenticate to Vault + if: ${{ endsWith(github.repository, '-enterprise') }} + id: vault-auth + run: vault-auth + + # NOTE: ENT specific step as we store secrets in Vault. + - name: Fetch Secrets + if: ${{ endsWith(github.repository, '-enterprise') }} + id: secrets + uses: hashicorp/vault-action@v2.5.0 + with: + url: ${{ steps.vault-auth.outputs.addr }} + caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }} + token: ${{ steps.vault-auth.outputs.token }} + secrets: | + kv/data/github/${{ github.repository }}/datadog apikey | DATADOG_API_KEY; + + - name: prepare datadog-ci + if: ${{ !endsWith(github.repository, '-enterprise') }} + run: | + curl -L --fail "https://github.com/DataDog/datadog-ci/releases/latest/download/datadog-ci_linux-x64" --output "/usr/local/bin/datadog-ci" + chmod +x /usr/local/bin/datadog-ci + + - name: upload coverage + # do not run on forks + if: github.event.pull_request.head.repo.full_name == github.repository + env: + DATADOG_API_KEY: "${{ endsWith(github.repository, '-enterprise') && env.DATADOG_API_KEY || secrets.DATADOG_API_KEY }}" + DD_ENV: ci + run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" $TEST_RESULTS_DIR/results.xml + + + test-integrations-success: + needs: + - setup + - dev-build + - generate-envoy-job-matrices + - envoy-integration-test + - upgrade-integration-test + runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} + if: ${{ always() }} + steps: + - name: evaluate upstream job results + run: | + # exit 1 if failure or cancelled result for any upstream job + if printf '${{ toJSON(needs) }}' | grep -E -i '\"result\": \"(failure|cancelled)\"'; then + printf "Tests failed or workflow cancelled:\n\n${{ toJSON(needs) }}" + exit 1 + fi diff --git a/.github/workflows/test-integrations.yml b/.github/workflows/test-integrations.yml index 4947662d56..9fc3a59f90 100644 --- a/.github/workflows/test-integrations.yml +++ b/.github/workflows/test-integrations.yml @@ -274,7 +274,7 @@ jobs: strategy: fail-fast: false matrix: - envoy-version: ["1.24.10", "1.25.9", "1.26.4", "1.27.0"] + envoy-version: ["1.27.0"] xds-target: ["server", "client"] test-cases: ${{ fromJSON(needs.generate-envoy-job-matrices.outputs.envoy-matrix) }} env: @@ -463,127 +463,6 @@ jobs: DATADOG_API_KEY: "${{ endsWith(github.repository, '-enterprise') && env.DATADOG_API_KEY || secrets.DATADOG_API_KEY }}" DD_ENV: ci run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" $TEST_RESULTS_DIR/results.xml - - upgrade-integration-test: - runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }} - needs: - - setup - - dev-build - permissions: - id-token: write # NOTE: this permission is explicitly required for Vault auth. - contents: read - strategy: - fail-fast: false - matrix: - consul-version: [ "1.15", "1.16"] - env: - CONSUL_LATEST_VERSION: ${{ matrix.consul-version }} - ENVOY_VERSION: "1.24.6" - steps: - - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 - # NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos. - - name: Setup Git - if: ${{ endsWith(github.repository, '-enterprise') }} - run: git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN }}:@github.com".insteadOf "https://github.com" - - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 - with: - go-version-file: 'go.mod' - - run: go env - - # Get go binary from workspace - - name: fetch binary - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 - with: - name: '${{ env.CONSUL_BINARY_UPLOAD_NAME }}' - path: . - - name: restore mode+x - run: chmod +x consul - - name: Build consul:local image - run: docker build -t ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local -f ./build-support/docker/Consul-Dev.dockerfile . - - name: Build consul-envoy:latest-version image - id: buildConsulEnvoyLatestImage - continue-on-error: true - run: docker build -t consul-envoy:latest-version --build-arg CONSUL_IMAGE=docker.mirror.hashicorp.services/${{ env.CONSUL_LATEST_IMAGE_NAME }}:${{ env.CONSUL_LATEST_VERSION }} --build-arg ENVOY_VERSION=${{ env.ENVOY_VERSION }} -f ./test/integration/consul-container/assets/Dockerfile-consul-envoy ./test/integration/consul-container/assets - - name: Retry Build consul-envoy:latest-version image - if: steps.buildConsulEnvoyLatestImage.outcome == 'failure' - run: docker build -t consul-envoy:latest-version --build-arg CONSUL_IMAGE=docker.mirror.hashicorp.services/${{ env.CONSUL_LATEST_IMAGE_NAME }}:${{ env.CONSUL_LATEST_VERSION }} --build-arg ENVOY_VERSION=${{ env.ENVOY_VERSION }} -f ./test/integration/consul-container/assets/Dockerfile-consul-envoy ./test/integration/consul-container/assets - - name: Build consul-envoy:target-version image - id: buildConsulEnvoyTargetImage - continue-on-error: true - run: docker build -t consul-envoy:target-version --build-arg CONSUL_IMAGE=${{ env.CONSUL_LATEST_IMAGE_NAME }}:local --build-arg ENVOY_VERSION=${{ env.ENVOY_VERSION }} -f ./test/integration/consul-container/assets/Dockerfile-consul-envoy ./test/integration/consul-container/assets - - name: Retry Build consul-envoy:target-version image - if: steps.buildConsulEnvoyTargetImage.outcome == 'failure' - run: docker build -t consul-envoy:target-version --build-arg CONSUL_IMAGE=${{ env.CONSUL_LATEST_IMAGE_NAME }}:local --build-arg ENVOY_VERSION=${{ env.ENVOY_VERSION }} -f ./test/integration/consul-container/assets/Dockerfile-consul-envoy ./test/integration/consul-container/assets - - name: Build sds image - run: docker build -t consul-sds-server ./test/integration/connect/envoy/test-sds-server/ - - name: Configure GH workaround for ipv6 loopback - if: ${{ !endsWith(github.repository, '-enterprise') }} - run: | - cat /etc/hosts && echo "-----------" - sudo sed -i 's/::1 *localhost ip6-localhost ip6-loopback/::1 ip6-localhost ip6-loopback/g' /etc/hosts - cat /etc/hosts - - name: Upgrade Integration Tests - run: | - mkdir -p "${{ env.TEST_RESULTS_DIR }}" - cd ./test/integration/consul-container/test/upgrade - docker run --rm ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local consul version - go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \ - --raw-command \ - --format=short-verbose \ - --debug \ - --rerun-fails=2 \ - --packages="./..." \ - -- \ - go test \ - -p=4 \ - -tags "${{ env.GOTAGS }}" \ - -timeout=30m \ - -json \ - ./... \ - --follow-log=false \ - --target-image ${{ env.CONSUL_LATEST_IMAGE_NAME }} \ - --target-version local \ - --latest-image docker.mirror.hashicorp.services/${{ env.CONSUL_LATEST_IMAGE_NAME }} \ - --latest-version "${{ env.CONSUL_LATEST_VERSION }}" - ls -lrt - env: - # this is needed because of incompatibility between RYUK container and GHA - GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml - GOTESTSUM_FORMAT: standard-verbose - COMPOSE_INTERACTIVE_NO_CLI: 1 - # tput complains if this isn't set to something. - TERM: ansi - # NOTE: ENT specific step as we store secrets in Vault. - - name: Authenticate to Vault - if: ${{ endsWith(github.repository, '-enterprise') }} - id: vault-auth - run: vault-auth - - # NOTE: ENT specific step as we store secrets in Vault. - - name: Fetch Secrets - if: ${{ endsWith(github.repository, '-enterprise') }} - id: secrets - uses: hashicorp/vault-action@v2.5.0 - with: - url: ${{ steps.vault-auth.outputs.addr }} - caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }} - token: ${{ steps.vault-auth.outputs.token }} - secrets: | - kv/data/github/${{ github.repository }}/datadog apikey | DATADOG_API_KEY; - - - name: prepare datadog-ci - if: ${{ !endsWith(github.repository, '-enterprise') }} - run: | - curl -L --fail "https://github.com/DataDog/datadog-ci/releases/latest/download/datadog-ci_linux-x64" --output "/usr/local/bin/datadog-ci" - chmod +x /usr/local/bin/datadog-ci - - - name: upload coverage - # do not run on forks - if: github.event.pull_request.head.repo.full_name == github.repository - env: - DATADOG_API_KEY: "${{ endsWith(github.repository, '-enterprise') && env.DATADOG_API_KEY || secrets.DATADOG_API_KEY }}" - DD_ENV: ci - run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" $TEST_RESULTS_DIR/results.xml peering_commontopo-integration-test: runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }} @@ -687,7 +566,6 @@ jobs: - envoy-integration-test - compatibility-integration-test - peering_commontopo-integration-test - - upgrade-integration-test runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} if: ${{ always() }} steps: