consul/.github/workflows/nightly-test-integrations.yml

# Copyright (c) HashiCorp, Inc.
# SPDX-License-Identifier: MPL-2.0

name: Nightly test-integrations

on:
  schedule:
    # Run nightly at 12AM UTC/8PM EST/5PM PST
    - cron: '* 0 * * *'
  workflow_dispatch: {}

env:
  TEST_RESULTS_DIR: /tmp/test-results
  TEST_RESULTS_ARTIFACT_NAME: test-results
  CONSUL_LICENSE: ${{ secrets.CONSUL_LICENSE }}
  GOTAGS: ${{ endsWith(github.repository, '-enterprise') && 'consulent' || '' }}
  GOTESTSUM_VERSION: "1.11.0"
  CONSUL_BINARY_UPLOAD_NAME: consul-bin
  # strip the hashicorp/ off the front of github.repository for consul
  CONSUL_LATEST_IMAGE_NAME: ${{ endsWith(github.repository, '-enterprise') && github.repository || 'hashicorp/consul' }}
  GOPRIVATE: github.com/hashicorp # Required for enterprise deps

jobs:
  setup:
    runs-on: ubuntu-latest
    name: Setup
    outputs:
      compute-small: ${{ steps.runners.outputs.compute-small }}
      compute-medium: ${{ steps.runners.outputs.compute-medium }}
      compute-large: ${{ steps.runners.outputs.compute-large }}
      compute-xl: ${{ steps.runners.outputs.compute-xl }}
      enterprise: ${{ steps.runners.outputs.enterprise }}
    steps:
      - name: Checkout code
        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
        with:
          ref: ${{ inputs.branch }}
      - id: runners
        run: .github/scripts/get_runner_classes.sh

  get-go-version:
    uses: ./.github/workflows/reusable-get-go-version.yml

  dev-build:
    needs:
    - setup
    - get-go-version
    uses: ./.github/workflows/reusable-dev-build.yml
    with:
      runs-on: ${{ needs.setup.outputs.compute-large }}
      repository-name: ${{ github.repository }}
      uploaded-binary-name: 'consul-bin'
      go-version: ${{ needs.get-go-version.outputs.go-version }}
    secrets:
      elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}

  generate-envoy-job-matrices:
    needs: [setup]
    runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }}
    name: Generate Envoy Job Matrices
    outputs:
      envoy-matrix: ${{ steps.set-matrix.outputs.envoy-matrix }}
    steps:
      - name: Checkout code
        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
        with:
          ref: ${{ inputs.branch }}
      - name: Generate Envoy Job Matrix
        id: set-matrix
        env:
          # this is further going to multiplied in envoy-integration tests by the 
          # other dimensions in the matrix.  Currently TOTAL_RUNNERS would be
          # multiplied by 8 based on these values:
          # envoy-version: ["1.25.11", "1.26.8", "1.27.5", "1.28.3"]
          # xds-target: ["server", "client"]
          TOTAL_RUNNERS: 8
          JQ_SLICER: '[ inputs ] | [_nwise(length / $runnercount | floor)]'
        run: |
          NUM_RUNNERS=$TOTAL_RUNNERS
          NUM_DIRS=$(find ./test/integration/connect/envoy -mindepth 1 -maxdepth 1 -type d | wc -l)

          if [ "$NUM_DIRS" -lt "$NUM_RUNNERS" ]; then
            echo "TOTAL_RUNNERS is larger than the number of tests/packages to split."
            NUM_RUNNERS=$((NUM_DIRS-1))
          fi
          # fix issue where test splitting calculation generates 1 more split than TOTAL_RUNNERS.
          NUM_RUNNERS=$((NUM_RUNNERS-1))
          {
            echo -n "envoy-matrix="
            find ./test/integration/connect/envoy -maxdepth 1 -type d -print0 \
              | xargs -0 -n 1 basename \
              | jq --raw-input --argjson runnercount "$NUM_RUNNERS" "$JQ_SLICER" \
              | jq --compact-output 'map(join("|"))'
          } >> "$GITHUB_OUTPUT"
  
  envoy-integration-test:
    runs-on: ${{ fromJSON(needs.setup.outputs.compute-large ) }}
    needs:
      - setup
      - get-go-version
      - generate-envoy-job-matrices
      - dev-build
    permissions:
      id-token: write # NOTE: this permission is explicitly required for Vault auth.
      contents: read
    strategy:
      fail-fast: false
      matrix:
        envoy-version: ["1.25.11", "1.26.8", "1.27.5", "1.28.3"]
        xds-target: ["server", "client"]
        test-cases: ${{ fromJSON(needs.generate-envoy-job-matrices.outputs.envoy-matrix) }}
    env:
      ENVOY_VERSION: ${{ matrix.envoy-version }}
      XDS_TARGET: ${{ matrix.xds-target }}
      AWS_LAMBDA_REGION: us-west-2
    steps:
      - name: Checkout code
        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
        with:
          ref: ${{ inputs.branch }}
      - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
        with:
          go-version: ${{ needs.get-go-version.outputs.go-version }}

      - name: fetch binary
        uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
        with:
          name: '${{ env.CONSUL_BINARY_UPLOAD_NAME }}'
          path: ./bin
      - name: restore mode+x
        run: chmod +x ./bin/consul

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0

      - name: Docker build
        run: docker build -t consul:local -f ./build-support/docker/Consul-Dev.dockerfile ./bin

      - name: Envoy Integration Tests
        env:
          GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml
          GOTESTSUM_FORMAT: standard-verbose
          COMPOSE_INTERACTIVE_NO_CLI: 1
          LAMBDA_TESTS_ENABLED: "true"
          # tput complains if this isn't set to something.
          TERM: ansi
        run: |
          # shellcheck disable=SC2001
          echo "Running $(sed 's,|, ,g' <<< "${{ matrix.test-cases }}" |wc -w) subtests"
          # shellcheck disable=SC2001
          sed 's,|,\n,g' <<< "${{ matrix.test-cases }}"
          go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \
              --debug \
              --rerun-fails \
              --rerun-fails-report=/tmp/gotestsum-rerun-fails \
              --jsonfile /tmp/jsonfile/go-test.log \
              --packages=./test/integration/connect/envoy \
              -- -timeout=30m -tags integration -run="TestEnvoy/(${{ matrix.test-cases }})"

      # NOTE: ENT specific step as we store secrets in Vault.
      - name: Authenticate to Vault
        if: ${{ !cancelled() && endsWith(github.repository, '-enterprise') }}
        id: vault-auth
        run: vault-auth

      # NOTE: ENT specific step as we store secrets in Vault.
      - name: Fetch Secrets
        if: ${{ !cancelled() && endsWith(github.repository, '-enterprise') }}
        id: secrets
        uses: hashicorp/vault-action@v3
        with:
          url: ${{ steps.vault-auth.outputs.addr }}
          caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }}
          token: ${{ steps.vault-auth.outputs.token }}
          secrets: |
              kv/data/github/${{ github.repository }}/datadog apikey | DATADOG_API_KEY;

      - name: prepare datadog-ci
        if: ${{ !cancelled() && !endsWith(github.repository, '-enterprise') }}
        run: |
          curl -L --fail "https://github.com/DataDog/datadog-ci/releases/latest/download/datadog-ci_linux-x64" --output "/usr/local/bin/datadog-ci"
          chmod +x /usr/local/bin/datadog-ci

      - name: upload coverage
        # do not run on forks
        if: ${{ !cancelled() && github.event.pull_request.head.repo.full_name == github.repository }}
        env:
          DATADOG_API_KEY: "${{ endsWith(github.repository, '-enterprise') && env.DATADOG_API_KEY || secrets.DATADOG_API_KEY }}"
          DD_ENV: ci
        run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" $TEST_RESULTS_DIR/results.xml
  
  upgrade-integration-test:
    runs-on: ${{ fromJSON(needs.setup.outputs.compute-large ) }}
    needs:
      - setup
      - get-go-version
      - dev-build
    permissions:
      id-token: write # NOTE: this permission is explicitly required for Vault auth.
      contents: read
    strategy:
      fail-fast: false
      matrix:
        consul-version: [ "1.16", "1.17"]
    env:
      CONSUL_LATEST_VERSION: ${{ matrix.consul-version }}
      # ENVOY_VERSION should be the latest version upported by all
      # consul versions in the matrix.consul-version, since we are testing upgrade from
      # an older consul version, e.g., 1.26.6 is supported by both 1.16 and 1.17.
      ENVOY_VERSION: "1.26.6"
    steps:
      - name: Checkout code
        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
        with:
          ref: ${{ inputs.branch }}
      # NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos.
      - name: Setup Git
        if: ${{ endsWith(github.repository, '-enterprise') }}
        run: git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN }}:@github.com".insteadOf "https://github.com"
      - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
        with:
          go-version: ${{ needs.get-go-version.outputs.go-version }}
      - run: go env

      # Get go binary from workspace
      - name: fetch binary
        uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
        with:
          name: '${{ env.CONSUL_BINARY_UPLOAD_NAME }}'
          path: .
      - name: restore mode+x
        run: chmod +x consul
      - name: Build consul:local image
        run: docker build -t ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local -f ./build-support/docker/Consul-Dev.dockerfile .
      - name: Build consul-envoy:latest-version image
        id: buildConsulEnvoyLatestImage
        run: |
          if ${{ endsWith(github.repository, '-enterprise') }} == 'true'
          then
            docker build -t consul-envoy:latest-version --build-arg CONSUL_IMAGE=docker.mirror.hashicorp.services/${{ env.CONSUL_LATEST_IMAGE_NAME }}:${{ env.CONSUL_LATEST_VERSION }}-ent --build-arg ENVOY_VERSION=${{ env.ENVOY_VERSION }} -f ./test/integration/consul-container/assets/Dockerfile-consul-envoy ./test/integration/consul-container/assets
          else
            docker build -t consul-envoy:latest-version --build-arg CONSUL_IMAGE=docker.mirror.hashicorp.services/${{ env.CONSUL_LATEST_IMAGE_NAME }}:${{ env.CONSUL_LATEST_VERSION }}     --build-arg ENVOY_VERSION=${{ env.ENVOY_VERSION }} -f ./test/integration/consul-container/assets/Dockerfile-consul-envoy ./test/integration/consul-container/assets
          fi
      - name: Build consul-envoy:target-version image
        id: buildConsulEnvoyTargetImage
        continue-on-error: true
        run: docker build -t consul-envoy:target-version --build-arg CONSUL_IMAGE=${{ env.CONSUL_LATEST_IMAGE_NAME }}:local --build-arg ENVOY_VERSION=${{ env.ENVOY_VERSION }} -f ./test/integration/consul-container/assets/Dockerfile-consul-envoy ./test/integration/consul-container/assets
      - name: Retry Build consul-envoy:target-version image
        if: steps.buildConsulEnvoyTargetImage.outcome == 'failure'
        run: docker build -t consul-envoy:target-version --build-arg CONSUL_IMAGE=${{ env.CONSUL_LATEST_IMAGE_NAME }}:local --build-arg ENVOY_VERSION=${{ env.ENVOY_VERSION }} -f ./test/integration/consul-container/assets/Dockerfile-consul-envoy ./test/integration/consul-container/assets
      - name: Build sds image
        run: docker build -t consul-sds-server ./test/integration/connect/envoy/test-sds-server/
      - name: Configure GH workaround for ipv6 loopback
        if: ${{ !endsWith(github.repository, '-enterprise') }}
        run: |
          cat /etc/hosts && echo "-----------"
          sudo sed -i 's/::1 *localhost ip6-localhost ip6-loopback/::1 ip6-localhost ip6-loopback/g' /etc/hosts
          cat /etc/hosts
      - name: Upgrade Integration Tests
        run: |
          mkdir -p "${{ env.TEST_RESULTS_DIR }}"
          cd ./test/integration/consul-container/test/upgrade
          docker run --rm ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local consul version
          go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \
            --raw-command \
            --format=github-actions \
            --rerun-fails \
            --packages="./..." \
            -- \
            go test \
            -p=4 \
            -tags "${{ env.GOTAGS }}" \
            -timeout=30m \
            -json \
            ./... \
            --follow-log=false \
            --target-image ${{ env.CONSUL_LATEST_IMAGE_NAME }} \
            --target-version local \
            --latest-image docker.mirror.hashicorp.services/${{ env.CONSUL_LATEST_IMAGE_NAME }} \
            --latest-version "${{ env.CONSUL_LATEST_VERSION }}"
          ls -lrt
        env:
          # this is needed because of incompatibility between RYUK container and GHA
          GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml
          GOTESTSUM_FORMAT: standard-verbose
          COMPOSE_INTERACTIVE_NO_CLI: 1
          # tput complains if this isn't set to something.
          TERM: ansi
      # NOTE: ENT specific step as we store secrets in Vault.
      - name: Authenticate to Vault
        if: ${{ !cancelled() && endsWith(github.repository, '-enterprise') }}
        id: vault-auth
        run: vault-auth

      # NOTE: ENT specific step as we store secrets in Vault.
      - name: Fetch Secrets
        if: ${{ !cancelled() && endsWith(github.repository, '-enterprise') }}
        id: secrets
        uses: hashicorp/vault-action@v3
        with:
          url: ${{ steps.vault-auth.outputs.addr }}
          caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }}
          token: ${{ steps.vault-auth.outputs.token }}
          secrets: |
              kv/data/github/${{ github.repository }}/datadog apikey | DATADOG_API_KEY;

      - name: prepare datadog-ci
        if: ${{ !cancelled() && !endsWith(github.repository, '-enterprise') }}
        run: |
          curl -L --fail "https://github.com/DataDog/datadog-ci/releases/latest/download/datadog-ci_linux-x64" --output "/usr/local/bin/datadog-ci"
          chmod +x /usr/local/bin/datadog-ci

      - name: upload coverage
        # do not run on forks
        if: ${{ !cancelled() && github.event.pull_request.head.repo.full_name == github.repository }}
        env:
          DATADOG_API_KEY: "${{ endsWith(github.repository, '-enterprise') && env.DATADOG_API_KEY || secrets.DATADOG_API_KEY }}"
          DD_ENV: ci
        run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" $TEST_RESULTS_DIR/results.xml

  upgrade-integration-test-deployer:
    runs-on: ${{ fromJSON(needs.setup.outputs.compute-large ) }}
    needs:
      - setup
      - get-go-version
    permissions:
      id-token: write # NOTE: this permission is explicitly required for Vault auth.
      contents: read
    strategy:
      fail-fast: false
      matrix:
        consul-version: [ "1.16", "1.17"]
    env:
      CONSUL_LATEST_VERSION: ${{ matrix.consul-version }}
    steps:
      - name: Checkout code
        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
        with:
          ref: ${{ inputs.branch }}
      # NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos.
      - name: Setup Git
        if: ${{ endsWith(github.repository, '-enterprise') }}
        run: git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN }}:@github.com".insteadOf "https://github.com"
      - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
        with:
          go-version: ${{ needs.get-go-version.outputs.go-version }}
      - run: go env
      - name: Build image
        run: make test-deployer-setup
      - name: Upgrade Integration Tests
        run: |
          mkdir -p "${{ env.TEST_RESULTS_DIR }}"
          #export NOLOGBUFFER=1
          cd ./test-integ/upgrade
          docker run --rm ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local consul version
          go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \
            --raw-command \
            --format=github-actions \
            --packages="./..." \
            -- \
            go test \
            -tags "${{ env.GOTAGS }}" \
            -timeout=20m \
            -parallel=2 \
            -json \
            ./... \
            --target-image ${{ env.CONSUL_LATEST_IMAGE_NAME }} \
            --target-version local \
            --latest-image docker.mirror.hashicorp.services/${{ env.CONSUL_LATEST_IMAGE_NAME }} \
            --latest-version "${{ env.CONSUL_LATEST_VERSION }}"
        env:
          # this is needed because of incompatibility between RYUK container and GHA
          GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml
          GOTESTSUM_FORMAT: standard-verbose
          COMPOSE_INTERACTIVE_NO_CLI: 1
          # tput complains if this isn't set to something.
          TERM: ansi
      # NOTE: ENT specific step as we store secrets in Vault.
      - name: Authenticate to Vault
        if: ${{ !cancelled() && endsWith(github.repository, '-enterprise') }}
        id: vault-auth
        run: vault-auth

      # NOTE: ENT specific step as we store secrets in Vault.
      - name: Fetch Secrets
        if: ${{ !cancelled() && endsWith(github.repository, '-enterprise') }}
        id: secrets
        uses: hashicorp/vault-action@v3
        with:
          url: ${{ steps.vault-auth.outputs.addr }}
          caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }}
          token: ${{ steps.vault-auth.outputs.token }}
          secrets: |
              kv/data/github/${{ github.repository }}/datadog apikey | DATADOG_API_KEY;

      - name: prepare datadog-ci
        if: ${{ !cancelled() && !endsWith(github.repository, '-enterprise') }}
        run: |
          curl -L --fail "https://github.com/DataDog/datadog-ci/releases/latest/download/datadog-ci_linux-x64" --output "/usr/local/bin/datadog-ci"
          chmod +x /usr/local/bin/datadog-ci

      - name: upload coverage
        # do not run on forks
        if: ${{ !cancelled() && github.event.pull_request.head.repo.full_name == github.repository }}
        env:
          DATADOG_API_KEY: "${{ endsWith(github.repository, '-enterprise') && env.DATADOG_API_KEY || secrets.DATADOG_API_KEY }}"
          DD_ENV: ci
        run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" $TEST_RESULTS_DIR/results.xml

  test-integrations-success:
    needs: 
    - setup
    - dev-build
    - generate-envoy-job-matrices
    - envoy-integration-test
    - upgrade-integration-test
    - upgrade-integration-test-deployer
    runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }}
    if: ${{ always() }}
    steps:
      - name: evaluate upstream job results
        run: |
          # exit 1 if failure or cancelled result for any upstream job
          if printf '${{ toJSON(needs) }}' | grep -E -i '\"result\": \"(failure|cancelled)\"'; then
            printf "Tests failed or workflow cancelled:\n\n${{ toJSON(needs) }}"
            exit 1
          fi
      - name: Notify Slack
        if: ${{ failure() }}
        id: slack
        uses: slackapi/slack-github-action@70cd7be8e40a46e8b0eced40b0de447bdb42f68e # v1.26.0
        with:
          payload: |
            {
              "message": "One or more nightly integration tests have failed. ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
            }
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.CONSUL_NIGHTLY_INTEG_TEST_SLACK_WEBHOOK }}