diff --git a/.changelog/18007.txt b/.changelog/18007.txt new file mode 100644 index 0000000000..b963d2f77f --- /dev/null +++ b/.changelog/18007.txt @@ -0,0 +1,3 @@ +```release-note:improvement +Windows: Integration tests for Consul Windows VMs +``` diff --git a/.github/scripts/get_runner_classes_windows.sh b/.github/scripts/get_runner_classes_windows.sh new file mode 100755 index 0000000000..c2e424d170 --- /dev/null +++ b/.github/scripts/get_runner_classes_windows.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +# +# This script generates tag-sets that can be used as runs-on: values to select runners. + +set -euo pipefail + +case "$GITHUB_REPOSITORY" in + *-enterprise) + # shellcheck disable=SC2129 + echo "compute-small=['self-hosted', 'windows', 'small']" >> "$GITHUB_OUTPUT" + echo "compute-medium=['self-hosted', 'windows', 'medium']" >> "$GITHUB_OUTPUT" + echo "compute-large=['self-hosted', 'windows', 'large']" >> "$GITHUB_OUTPUT" + # m5d.8xlarge is equivalent to our xl custom runner in OSS + echo "compute-xl=['self-hosted', 'ondemand', 'windows', 'type=m5d.8xlarge']" >> "$GITHUB_OUTPUT" + ;; + *) + # shellcheck disable=SC2129 + echo "compute-small=['windows-2019']" >> "$GITHUB_OUTPUT" + echo "compute-medium=['windows-2019']" >> "$GITHUB_OUTPUT" + echo "compute-large=['windows-2019']" >> "$GITHUB_OUTPUT" + echo "compute-xl=['windows-2019']" >> "$GITHUB_OUTPUT" + ;; +esac diff --git a/.github/workflows/reusable-dev-build-windows.yml b/.github/workflows/reusable-dev-build-windows.yml new file mode 100644 index 0000000000..446fbb7136 --- /dev/null +++ b/.github/workflows/reusable-dev-build-windows.yml @@ -0,0 +1,47 @@ +name: reusable-dev-build-windows + +on: + workflow_call: + inputs: + uploaded-binary-name: + required: false + type: string + default: "consul.exe" + runs-on: + description: An expression indicating which kind of runners to use. + required: true + type: string + repository-name: + required: true + type: string + go-arch: + required: false + type: string + default: "" + secrets: + elevated-github-token: + required: true +jobs: + build: + runs-on: 'windows-2019' + steps: + - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 + # NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos. + - name: Setup Git + if: ${{ endsWith(inputs.repository-name, '-enterprise') }} + run: git config --global url."https://${{ secrets.elevated-github-token }}:@github.com".insteadOf "https://github.com" + - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 + with: + go-version-file: 'go.mod' + - name: Build + env: + GOARCH: ${{ inputs.goarch }} + run: go build . + # save dev build to pass to downstream jobs + - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 + with: + name: ${{inputs.uploaded-binary-name}} + path: consul.exe + - name: Notify Slack + if: ${{ failure() }} + run: .github/scripts/notify_slack.sh diff --git a/.github/workflows/test-integrations-windows.yml b/.github/workflows/test-integrations-windows.yml new file mode 100644 index 0000000000..d71892c784 --- /dev/null +++ b/.github/workflows/test-integrations-windows.yml @@ -0,0 +1,1210 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +name: test-integrations-windows + +on: + schedule: + # * is a special character in YAML so you have to quote this string + # Run nightly at 12AM UTC/8PM EST/5PM PST. + - cron: '0 0 * * *' + +env: + TEST_RESULTS_DIR: /tmp/test-results + TEST_RESULTS_ARTIFACT_NAME: test-results + CONSUL_LICENSE: ${{ secrets.CONSUL_LICENSE }} + GOTAGS: ${{ endsWith(github.repository, '-enterprise') && 'consulent' || '' }} + GOTESTSUM_VERSION: "1.9.0" + CONSUL_BINARY_UPLOAD_NAME: consul.exe + # strip the hashicorp/ off the front of github.repository for consul + CONSUL_LATEST_IMAGE_NAME: ${{ endsWith(github.repository, '-enterprise') && github.repository || 'consul' }} + GOPRIVATE: github.com/hashicorp # Required for enterprise deps + +jobs: + setup: + runs-on: ubuntu-latest + name: Setup + outputs: + compute-small: ${{ steps.runners.outputs.compute-small }} + compute-medium: ${{ steps.runners.outputs.compute-medium }} + compute-large: ${{ steps.runners.outputs.compute-large }} + compute-xl: ${{ steps.runners.outputs.compute-xl }} + enterprise: ${{ steps.runners.outputs.enterprise }} + steps: + - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 + - id: runners + run: .github/scripts/get_runner_classes_windows.sh + + dev-build: + uses: ./.github/workflows/reusable-dev-build-windows.yml + with: + runs-on: ${{ needs.setup.outputs.compute-xl }} + repository-name: ${{ github.repository }} + uploaded-binary-name: 'consul.exe' + secrets: + elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} + + # NOTE: Jobs needs to be added here manually. Jobs when run together on windows fails intermittently. + # So they are run independently of each other. + envoy-integration-test: + needs: + - setup + - dev-build + runs-on: ${{ fromJSON(needs.setup.outputs.compute-large) }} + permissions: + id-token: write # NOTE: this permission is explicitly required for Vault auth. + contents: read + strategy: + fail-fast: false + matrix: + envoy-version: [ "1.23.10", "1.24.8", "1.25.7", "1.26.2" ] + xds-target: [ "server", "client" ] + env: + ENVOY_VERSION: ${{ matrix.envoy-version }} + XDS_TARGET: ${{ matrix.xds-target }} + AWS_LAMBDA_REGION: us-west-2 + steps: + - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 + - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 + with: + go-version-file: 'go.mod' + + - name: Fetch binary + uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + with: + name: '${{ env.CONSUL_BINARY_UPLOAD_NAME }}' + path: ${{ github.workspace }} + + - name: Restore mode+x + run: chmod +x ${{ github.workspace }}\consul.exe + + - name: Setup TcpDump Docker Image + shell: bash + run: | + cd test/integration/connect/envoy + curl -sSL "https://asheshvidyut-bucket.s3.ap-southeast-2.amazonaws.com/tcpdump.exe" -o tcpdump.exe + docker build -t envoy-tcpdump -f Dockerfile-tcpdump-windows . + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c # v2.5.0 + + - name: Docker build consul + run: docker build -t windows/consul -f Dockerfile-windows . + + - name: Docker build consul local + shell: bash + run: cd build-support/windows && ./build-consul-local-images.sh + + - name: Docker build consul dev + shell: bash + run: cd build-support/windows && ./build-consul-dev-image.sh + + # https://hashicorp.atlassian.net/browse/NET-4973 + # ^ Ticket to figure out why grouping test case is failing on Windows Machine + + - name: Envoy Integration Tests for windows case-api-gateway-http-hostnames + shell: bash + if: always() + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running Integration Test case-api-gateway-http-hostnames" + # shellcheck disable=SC2001 + go test -v -timeout=30m -tags integration \ + ./test/integration/connect/envoy -run="TestEnvoy/case-api-gateway-http-hostnames" -win=true + + - name: Envoy Integration Tests for windows case-api-gateway-http-simple + shell: bash + if: always() + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running Integration Test case-api-gateway-http-simple" + # shellcheck disable=SC2001 + go test -v -timeout=30m -tags integration \ + ./test/integration/connect/envoy -run="TestEnvoy/case-api-gateway-http-simple" -win=true + + - name: Envoy Integration Tests for windows case-api-gateway-http-splitter-targets + shell: bash + if: always() + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running Integration Test case-api-gateway-http-splitter-targets" + # shellcheck disable=SC2001 + go test -v -timeout=30m -tags integration \ + ./test/integration/connect/envoy -run="TestEnvoy/case-api-gateway-http-splitter-targets" -win=true + + - name: Envoy Integration Tests for windows case-api-gateway-http-tls-overlapping-hosts + shell: bash + if: always() + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running Integration Test case-api-gateway-http-tls-overlapping-hosts" + # shellcheck disable=SC2001 + go test -v -timeout=30m -tags integration \ + ./test/integration/connect/envoy -run="TestEnvoy/case-api-gateway-http-tls-overlapping-hosts" -win=true + + - name: Envoy Integration Tests for windows case-api-gateway-tcp-conflicted + shell: bash + if: always() + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running Integration Test case-api-gateway-tcp-conflicted" + # shellcheck disable=SC2001 + go test -v -timeout=30m -tags integration \ + ./test/integration/connect/envoy -run="TestEnvoy/case-api-gateway-tcp-conflicted" -win=true + + - name: Envoy Integration Tests for windows case-api-gateway-tcp-simple + shell: bash + if: always() + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running Integration Test case-api-gateway-tcp-simple" + # shellcheck disable=SC2001 + go test -v -timeout=30m -tags integration \ + ./test/integration/connect/envoy -run="TestEnvoy/case-api-gateway-tcp-simple" -win=true + + - name: Envoy Integration Tests for windows case-api-gateway-tcp-tls-overlapping-hosts + shell: bash + if: always() + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running Integration Test case-api-gateway-tcp-tls-overlapping-hosts" + # shellcheck disable=SC2001 + go test -v -timeout=30m -tags integration \ + ./test/integration/connect/envoy -run="TestEnvoy/case-api-gateway-tcp-tls-overlapping-hosts" -win=true + + - name: Envoy Integration Tests for windows case-badauthz + shell: bash + if: always() + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running Integration Test case-badauthz" + # shellcheck disable=SC2001 + go test -v -timeout=30m -tags integration \ + ./test/integration/connect/envoy -run="TestEnvoy/case-badauthz" -win=true + + - name: Envoy Integration Tests for windows case-basic + shell: bash + if: always() + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running Integration Test case-basic" + # shellcheck disable=SC2001 + go test -v -timeout=30m -tags integration \ + ./test/integration/connect/envoy -run="TestEnvoy/case-basic" -win=true + + - name: Envoy Integration Tests for windows case-centralconf + shell: bash + if: always() + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running Integration Test case-centralconf" + # shellcheck disable=SC2001 + go test -v -timeout=30m -tags integration \ + ./test/integration/connect/envoy -run="TestEnvoy/case-centralconf" -win=true + + - name: Envoy Integration Tests for windows case-cfg-resolver-cluster-peering-failover + shell: bash + if: always() + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running Integration Test case-cfg-resolver-cluster-peering-failover" + # shellcheck disable=SC2001 + go test -v -timeout=30m -tags integration \ + ./test/integration/connect/envoy -run="TestEnvoy/case-cfg-resolver-cluster-peering-failover" -win=true + + - name: Envoy Integration Tests for windows case-cfg-resolver-dc-failover-gateways-none + shell: bash + if: always() + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running Integration Test case-cfg-resolver-dc-failover-gateways-none" + # shellcheck disable=SC2001 + go test -v -timeout=30m -tags integration \ + ./test/integration/connect/envoy -run="TestEnvoy/case-cfg-resolver-dc-failover-gateways-none" -win=true + + - name: Envoy Integration Tests for windows case-cfg-resolver-dc-failover-gateways-remote + shell: bash + if: always() + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running Integration Test case-cfg-resolver-dc-failover-gateways-remote" + # shellcheck disable=SC2001 + go test -v -timeout=30m -tags integration \ + ./test/integration/connect/envoy -run="TestEnvoy/case-cfg-resolver-dc-failover-gateways-remote" -win=true + + - name: Envoy Integration Tests for windows case-cfg-resolver-defaultsubset + shell: bash + if: always() + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running Integration Test case-cfg-resolver-defaultsubset" + # shellcheck disable=SC2001 + go test -v -timeout=30m -tags integration \ + ./test/integration/connect/envoy -run="TestEnvoy/case-cfg-resolver-defaultsubset" -win=true + + - name: Envoy Integration Tests for windows case-cfg-resolver-features + shell: bash + if: always() + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running Integration Test case-cfg-resolver-features" + # shellcheck disable=SC2001 + go test -v -timeout=30m -tags integration \ + ./test/integration/connect/envoy -run="TestEnvoy/case-cfg-resolver-features" -win=true + + - name: Envoy Integration Tests for windows case-cfg-resolver-subset-onlypassing + shell: bash + if: always() + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running Integration Test case-cfg-resolver-subset-onlypassing" + # shellcheck disable=SC2001 + go test -v -timeout=30m -tags integration \ + ./test/integration/connect/envoy -run="TestEnvoy/case-cfg-resolver-subset-onlypassing" -win=true + + - name: Envoy Integration Tests for windows case-cfg-resolver-subset-redirect + shell: bash + if: always() + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running Integration Test case-cfg-resolver-subset-redirect" + # shellcheck disable=SC2001 + go test -v -timeout=30m -tags integration \ + ./test/integration/connect/envoy -run="TestEnvoy/case-cfg-resolver-subset-redirect" -win=true + + - name: Envoy Integration Tests for windows case-cfg-resolver-svc-failover + shell: bash + if: always() + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running Integration Test case-cfg-resolver-svc-failover" + # shellcheck disable=SC2001 + go test -v -timeout=30m -tags integration \ + ./test/integration/connect/envoy -run="TestEnvoy/case-cfg-resolver-svc-failover" -win=true + + - name: Envoy Integration Tests for windows case-cfg-resolver-svc-redirect-http + shell: bash + if: always() + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running Integration Test case-cfg-resolver-svc-redirect-http" + # shellcheck disable=SC2001 + go test -v -timeout=30m -tags integration \ + ./test/integration/connect/envoy -run="TestEnvoy/case-cfg-resolver-svc-redirect-http" -win=true + + - name: Envoy Integration Tests for windows case-cfg-resolver-svc-redirect-tcp + shell: bash + if: always() + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running Integration Test case-cfg-resolver-svc-redirect-tcp" + # shellcheck disable=SC2001 + go test -v -timeout=30m -tags integration \ + ./test/integration/connect/envoy -run="TestEnvoy/case-cfg-resolver-svc-redirect-tcp" -win=true + + - name: Envoy Integration Tests for windows case-cfg-router-features + shell: bash + if: always() + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running Integration Test case-cfg-router-features" + # shellcheck disable=SC2001 + go test -v -timeout=30m -tags integration \ + ./test/integration/connect/envoy -run="TestEnvoy/case-cfg-router-features" -win=true + + - name: Envoy Integration Tests for windows case-cfg-splitter-cluster-peering + shell: bash + if: always() + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running Integration Test case-cfg-splitter-cluster-peering" + # shellcheck disable=SC2001 + go test -v -timeout=30m -tags integration \ + ./test/integration/connect/envoy -run="TestEnvoy/case-cfg-splitter-cluster-peering" -win=true + + - name: Envoy Integration Tests for windows case-cfg-splitter-features + shell: bash + if: always() + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running Integration Test case-cfg-splitter-features" + # shellcheck disable=SC2001 + go test -v -timeout=30m -tags integration \ + ./test/integration/connect/envoy -run="TestEnvoy/case-cfg-splitter-features" -win=true + + - name: Envoy Integration Tests for windows case-cfg-splitter-peering-ingress-gateways + shell: bash + if: always() + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running Integration Test case-cfg-splitter-peering-ingress-gateways" + # shellcheck disable=SC2001 + go test -v -timeout=30m -tags integration \ + ./test/integration/connect/envoy -run="TestEnvoy/case-cfg-splitter-peering-ingress-gateways" -win=true + + # This test runs fine on windows machine but fails on CI + # Task to be picked later on - https://hashicorp.atlassian.net/browse/NET-4972 + # - name: Envoy Integration Tests for windows case-consul-exec + # if: always() + # shell: bash + # env: + # GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + # GOTESTSUM_FORMAT: standard-verbose + # COMPOSE_INTERACTIVE_NO_CLI: 1 + # LAMBDA_TESTS_ENABLED: "true" + # # tput complains if this isn't set to something. + # TERM: ansi + # run: | + # #shellcheck disable=SC2001 + # echo "Running Integration Test case-consul-exec" + # # shellcheck disable=SC2001 + # go test -v -timeout=30m -tags integration \ + # ./test/integration/connect/envoy -run="TestEnvoy/case-consul-exec" -win=true + + - name: Envoy Integration Tests for windows case-cross-peer-control-plane-mgw + if: always() + shell: bash + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running Integration Test case-cross-peer-control-plane-mgw" + # shellcheck disable=SC2001 + go test -v -timeout=30m -tags integration \ + ./test/integration/connect/envoy -run="TestEnvoy/case-cross-peer-control-plane-mgw" -win=true + + - name: Envoy Integration Tests for windows case-cross-peers + shell: bash + if: always() + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running Integration Test case-cross-peers" + # shellcheck disable=SC2001 + go test -v -timeout=30m -tags integration \ + ./test/integration/connect/envoy -run="TestEnvoy/case-cross-peers" -win=true + + - name: Envoy Integration Tests for windows case-cross-peers-http + shell: bash + if: always() + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running Integration Test case-cross-peers-http" + # shellcheck disable=SC2001 + go test -v -timeout=30m -tags integration \ + ./test/integration/connect/envoy -run="TestEnvoy/case-cross-peers-http" -win=true + + - name: Envoy Integration Tests for windows case-cross-peers-http-router + shell: bash + if: always() + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running Integration Test case-cross-peers-http-router" + # shellcheck disable=SC2001 + go test -v -timeout=30m -tags integration \ + ./test/integration/connect/envoy -run="TestEnvoy/case-cross-peers-http-router" -win=true + + - name: Envoy Integration Tests for windows case-cross-peers-resolver-redirect-tcp + shell: bash + if: always() + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running Integration Test case-cross-peers-resolver-redirect-tcp" + # shellcheck disable=SC2001 + go test -v -timeout=30m -tags integration \ + ./test/integration/connect/envoy -run="TestEnvoy/case-cross-peers-resolver-redirect-tcp" -win=true + + - name: Envoy Integration Tests for windows case-dogstatsd-udp + shell: bash + if: always() + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running Integration Test case-dogstatsd-udp" + # shellcheck disable=SC2001 + go test -v -timeout=30m -tags integration \ + ./test/integration/connect/envoy -run="TestEnvoy/case-dogstatsd-udp" -win=true + + - name: Envoy Integration Tests for windows case-envoyext-ratelimit + shell: bash + if: always() + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running Integration Test case-envoyext-ratelimit" + # shellcheck disable=SC2001 + go test -v -timeout=30m -tags integration \ + ./test/integration/connect/envoy -run="TestEnvoy/case-envoyext-ratelimit" -win=true + + - name: Envoy Integration Tests for windows case-expose-checks + shell: bash + if: always() + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running Integration Test case-expose-checks" + # shellcheck disable=SC2001 + go test -v -timeout=30m -tags integration \ + ./test/integration/connect/envoy -run="TestEnvoy/case-expose-checks" -win=true + + - name: Envoy Integration Tests for windows case-gateway-without-services + shell: bash + if: always() + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running Integration Test case-gateway-without-services" + # shellcheck disable=SC2001 + go test -v -timeout=30m -tags integration \ + ./test/integration/connect/envoy -run="TestEnvoy/case-gateway-without-services" -win=true + + - name: Envoy Integration Tests for windows case-gateways-local + shell: bash + if: always() + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running Integration Test case-gateways-local" + # shellcheck disable=SC2001 + go test -v -timeout=30m -tags integration \ + ./test/integration/connect/envoy -run="TestEnvoy/case-gateways-local" -win=true + + - name: Envoy Integration Tests for windows case-gateways-remote + shell: bash + if: always() + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running Integration Test case-gateways-remote" + # shellcheck disable=SC2001 + go test -v -timeout=30m -tags integration \ + ./test/integration/connect/envoy -run="TestEnvoy/case-gateways-remote" -win=true + + - name: Envoy Integration Tests for windows case-grpc + shell: bash + if: always() + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running Integration Test case-grpc" + # shellcheck disable=SC2001 + go test -v -timeout=30m -tags integration \ + ./test/integration/connect/envoy -run="TestEnvoy/case-grpc" -win=true + + - name: Envoy Integration Tests for windows case-http + shell: bash + if: always() + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running Integration Test case-http" + # shellcheck disable=SC2001 + go test -v -timeout=30m -tags integration \ + ./test/integration/connect/envoy -run="TestEnvoy/case-http" -win=true + + - name: Envoy Integration Tests for windows case-http-badauthz + shell: bash + if: always() + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running Integration Test case-http-badauthz" + # shellcheck disable=SC2001 + go test -v -timeout=30m -tags integration \ + ./test/integration/connect/envoy -run="TestEnvoy/case-http-badauthz" -win=true + + - name: Envoy Integration Tests for windows case-ingress-gateway-grpc + shell: bash + if: always() + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running Integration Test case-ingress-gateway-grpc" + # shellcheck disable=SC2001 + go test -v -timeout=30m -tags integration \ + ./test/integration/connect/envoy -run="TestEnvoy/case-ingress-gateway-grpc" -win=true + + - name: Envoy Integration Tests for windows case-ingress-gateway-http + shell: bash + if: always() + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running Integration Test case-ingress-gateway-http" + # shellcheck disable=SC2001 + go test -v -timeout=30m -tags integration \ + ./test/integration/connect/envoy -run="TestEnvoy/case-ingress-gateway-http" -win=true + + - name: Envoy Integration Tests for windows case-ingress-gateway-multiple-services + shell: bash + if: always() + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running Integration Test case-ingress-gateway-multiple-services" + # shellcheck disable=SC2001 + go test -v -timeout=30m -tags integration \ + ./test/integration/connect/envoy -run="TestEnvoy/case-ingress-gateway-multiple-services" -win=true + + - name: Envoy Integration Tests for windows case-ingress-gateway-peering-failover + shell: bash + if: always() + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running Integration Test case-ingress-gateway-peering-failover" + # shellcheck disable=SC2001 + go test -v -timeout=30m -tags integration \ + ./test/integration/connect/envoy -run="TestEnvoy/case-ingress-gateway-peering-failover" -win=true + + - name: Envoy Integration Tests for windows case-ingress-gateway-simple + shell: bash + if: always() + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running Integration Test case-ingress-gateway-simple" + # shellcheck disable=SC2001 + go test -v -timeout=30m -tags integration \ + ./test/integration/connect/envoy -run="TestEnvoy/case-ingress-gateway-simple" -win=true + + - name: Envoy Integration Tests for windows case-ingress-mesh-gateways-resolver + shell: bash + if: always() + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running Integration Test case-ingress-mesh-gateways-resolver" + # shellcheck disable=SC2001 + go test -v -timeout=30m -tags integration \ + ./test/integration/connect/envoy -run="TestEnvoy/case-ingress-mesh-gateways-resolver" -win=true + + - name: Envoy Integration Tests for windows case-l7-intentions + shell: bash + if: always() + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running Integration Test case-l7-intentions" + # shellcheck disable=SC2001 + go test -v -timeout=30m -tags integration \ + ./test/integration/connect/envoy -run="TestEnvoy/case-l7-intentions" -win=true + + - name: Envoy Integration Tests for windows case-multidc-rsa-ca + shell: bash + if: always() + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running Integration Test case-multidc-rsa-ca" + # shellcheck disable=SC2001 + go test -v -timeout=30m -tags integration \ + ./test/integration/connect/envoy -run="TestEnvoy/case-multidc-rsa-ca" -win=true + + - name: Envoy Integration Tests for windows case-prometheus + shell: bash + if: always() + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running Integration Test case-prometheus" + # shellcheck disable=SC2001 + go test -v -timeout=30m -tags integration \ + ./test/integration/connect/envoy -run="TestEnvoy/case-prometheus" -win=true + + - name: Envoy Integration Tests for windows case-property-override + shell: bash + if: always() + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running Integration Test case-property-override" + # shellcheck disable=SC2001 + go test -v -timeout=30m -tags integration \ + ./test/integration/connect/envoy -run="TestEnvoy/case-property-override" -win=true + + - name: Envoy Integration Tests for windows case-stats-proxy + shell: bash + if: always() + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running Integration Test case-stats-proxy" + # shellcheck disable=SC2001 + go test -v -timeout=30m -tags integration \ + ./test/integration/connect/envoy -run="TestEnvoy/case-stats-proxy" -win=true + + - name: Envoy Integration Tests for windows case-statsd-udp + shell: bash + if: always() + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running Integration Test case-statsd-udp" + # shellcheck disable=SC2001 + go test -v -timeout=30m -tags integration \ + ./test/integration/connect/envoy -run="TestEnvoy/case-statsd-udp" -win=true + + - name: Envoy Integration Tests for windows case-terminating-gateway-hostnames + shell: bash + if: always() + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running Integration Test case-terminating-gateway-hostnames" + # shellcheck disable=SC2001 + go test -v -timeout=30m -tags integration \ + ./test/integration/connect/envoy -run="TestEnvoy/case-terminating-gateway-hostnames" -win=true + + - name: Envoy Integration Tests for windows case-terminating-gateway-simple + shell: bash + if: always() + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running Integration Test case-terminating-gateway-simple" + # shellcheck disable=SC2001 + go test -v -timeout=30m -tags integration \ + ./test/integration/connect/envoy -run="TestEnvoy/case-terminating-gateway-simple" -win=true + + - name: Envoy Integration Tests for windows case-terminating-gateway-without-services + shell: bash + if: always() + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running Integration Test case-terminating-gateway-without-services" + # shellcheck disable=SC2001 + go test -v -timeout=30m -tags integration \ + ./test/integration/connect/envoy -run="TestEnvoy/case-terminating-gateway-without-services" -win=true + + - name: Envoy Integration Tests for windows case-upstream-config + shell: bash + if: always() + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running Integration Test case-upstream-config" + # shellcheck disable=SC2001 + go test -v -timeout=30m -tags integration \ + ./test/integration/connect/envoy -run="TestEnvoy/case-upstream-config" -win=true + + - name: Envoy Integration Tests for windows case-wanfed-gw + shell: bash + if: always() + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running Integration Test case-wanfed-gw" + # shellcheck disable=SC2001 + go test -v -timeout=30m -tags integration \ + ./test/integration/connect/envoy -run="TestEnvoy/case-wanfed-gw" -win=true + + - name: Envoy Integration Tests for windows case-ingress-gateway-sds + shell: bash + if: always() + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running Integration Test case-ingress-gateway-sds" + # shellcheck disable=SC2001 + go test -v -timeout=30m -tags integration \ + ./test/integration/connect/envoy -run="TestEnvoy/case-ingress-gateway-sds" -win=true + + - name: Envoy Integration Tests for windows case-lua + shell: bash + if: always() + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running Integration Test case-lua" + # shellcheck disable=SC2001 + go test -v -timeout=30m -tags integration \ + ./test/integration/connect/envoy -run="TestEnvoy/case-lua" -win=true + + - name: Envoy Integration Tests for windows case-terminating-gateway-subsets + shell: bash + if: always() + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running Integration Test case-terminating-gateway-subsets" + # shellcheck disable=SC2001 + go test -v -timeout=30m -tags integration \ + ./test/integration/connect/envoy -run="TestEnvoy/case-terminating-gateway-subsets" -win=true + + # Skipping this because - https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_filters/wasm_filter + # - name: Envoy Integration Tests for windows case-wasm + # shell: bash + # env: + # GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + # GOTESTSUM_FORMAT: standard-verbose + # COMPOSE_INTERACTIVE_NO_CLI: 1 + # LAMBDA_TESTS_ENABLED: "true" + # # tput complains if this isn't set to something. + # TERM: ansi + # run: | + # # shellcheck disable=SC2001 + # echo "Running Integration Test case-wasm" + # # shellcheck disable=SC2001 + # go test -v -timeout=30m -tags integration \ + # ./test/integration/connect/envoy -run="TestEnvoy/case-wasm" -win=true + + # Skipping because of - cacert is not available in curl windows + # https://www.phillipsj.net/posts/windows-curl-and-self-signed-certs/ + # - name: Envoy Integration Tests for windows case-ingress-gateway-tls + # shell: bash + # if: always() + # env: + # GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + # GOTESTSUM_FORMAT: standard-verbose + # COMPOSE_INTERACTIVE_NO_CLI: 1 + # LAMBDA_TESTS_ENABLED: "true" + # # tput complains if this isn't set to something. + # TERM: ansi + # run: | + # # shellcheck disable=SC2001 + # echo "Running Integration Test case-ingress-gateway-tls" + # # shellcheck disable=SC2001 + # go test -v -timeout=30m -tags integration \ + # ./test/integration/connect/envoy -run="TestEnvoy/case-ingress-gateway-tls" -win=true + + - name: Envoy Integration Tests for windows case-mesh-to-lambda + shell: bash + if: always() + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running Integration Test case-mesh-to-lambda" + # shellcheck disable=SC2001 + go test -v -timeout=30m -tags integration \ + ./test/integration/connect/envoy -run="TestEnvoy/case-mesh-to-lambda" -win=true + + + # NOTE: ENT specific step as we store secrets in Vault. + - name: Authenticate to Vault + if: ${{ endsWith(github.repository, '-enterprise') }} + id: vault-auth + run: vault-auth + + # NOTE: ENT specific step as we store secrets in Vault. + - name: Fetch Secrets + if: ${{ endsWith(github.repository, '-enterprise') }} + id: secrets + uses: hashicorp/vault-action@v2.5.0 + with: + url: ${{ steps.vault-auth.outputs.addr }} + caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }} + token: ${{ steps.vault-auth.outputs.token }} + secrets: | + kv/data/github/${{ github.repository }}/datadog apikey | DATADOG_API_KEY; + + - name: Prepare datadog-ci + shell: bash + if: ${{ !endsWith(github.repository, '-enterprise') }} + run: | + curl -L --fail "https://github.com/DataDog/datadog-ci/releases/download/v2.17.2/datadog-ci_win-x64.exe" --output "C:/datadog-ci" + chmod +x C:/datadog-ci + + - name: Upload coverage + # do not run on forks + if: github.event.pull_request.head.repo.full_name == github.repository + env: + DATADOG_API_KEY: "${{ endsWith(github.repository, '-enterprise') && env.DATADOG_API_KEY || secrets.DATADOG_API_KEY }}" + DD_ENV: ci + run: C:/datadog-ci junit upload --service "$GITHUB_REPOSITORY" $TEST_RESULTS_DIR/results.xml + + test-integrations-success: + needs: + - envoy-integration-test + - upgrade-integration-test + runs-on: 'ubuntu-latest' + if: ${{ always() }} + steps: + - name: evaluate upstream job results + run: | + # exit 1 if failure or cancelled result for any upstream job + if printf '${{ toJSON(needs) }}' | grep -E -i '\"result\": \"(failure|cancelled)\"'; then + printf "Tests failed or workflow cancelled:\n\n${{ toJSON(needs) }}" + exit 1 + fi \ No newline at end of file diff --git a/.release/docker/docker-entrypoint-windows.sh b/.release/docker/docker-entrypoint-windows.sh new file mode 100644 index 0000000000..776b8113ce --- /dev/null +++ b/.release/docker/docker-entrypoint-windows.sh @@ -0,0 +1,82 @@ +#!/usr/bin/dumb-init /bin/sh +set -e + +# Note above that we run dumb-init as PID 1 in order to reap zombie processes +# as well as forward signals to all processes in its session. Normally, sh +# wouldn't do either of these functions so we'd leak zombies as well as do +# unclean termination of all our sub-processes. +# As of docker 1.13, using docker run --init achieves the same outcome. + +# You can set CONSUL_BIND_INTERFACE to the name of the interface you'd like to +# bind to and this will look up the IP and pass the proper -bind= option along +# to Consul. +CONSUL_BIND= +if [ -n "$CONSUL_BIND_INTERFACE" ]; then + CONSUL_BIND_ADDRESS=$(ip -o -4 addr list $CONSUL_BIND_INTERFACE | head -n1 | awk '{print $4}' | cut -d/ -f1) + if [ -z "$CONSUL_BIND_ADDRESS" ]; then + echo "Could not find IP for interface '$CONSUL_BIND_INTERFACE', exiting" + exit 1 + fi + + CONSUL_BIND="-bind=$CONSUL_BIND_ADDRESS" + echo "==> Found address '$CONSUL_BIND_ADDRESS' for interface '$CONSUL_BIND_INTERFACE', setting bind option..." +fi + +# You can set CONSUL_CLIENT_INTERFACE to the name of the interface you'd like to +# bind client intefaces (HTTP, DNS, and RPC) to and this will look up the IP and +# pass the proper -client= option along to Consul. +CONSUL_CLIENT= +if [ -n "$CONSUL_CLIENT_INTERFACE" ]; then + CONSUL_CLIENT_ADDRESS=$(ip -o -4 addr list $CONSUL_CLIENT_INTERFACE | head -n1 | awk '{print $4}' | cut -d/ -f1) + if [ -z "$CONSUL_CLIENT_ADDRESS" ]; then + echo "Could not find IP for interface '$CONSUL_CLIENT_INTERFACE', exiting" + exit 1 + fi + + CONSUL_CLIENT="-client=$CONSUL_CLIENT_ADDRESS" + echo "==> Found address '$CONSUL_CLIENT_ADDRESS' for interface '$CONSUL_CLIENT_INTERFACE', setting client option..." +fi + +# CONSUL_DATA_DIR is exposed as a volume for possible persistent storage. The +# CONSUL_CONFIG_DIR isn't exposed as a volume but you can compose additional +# config files in there if you use this image as a base, or use CONSUL_LOCAL_CONFIG +# below. +CONSUL_DATA_DIR=C:\\consul\\data +CONSUL_CONFIG_DIR=C:\\consul\\config + +# You can also set the CONSUL_LOCAL_CONFIG environemnt variable to pass some +# Consul configuration JSON without having to bind any volumes. +if [ -n "$CONSUL_LOCAL_CONFIG" ]; then + echo "$CONSUL_LOCAL_CONFIG" > "$CONSUL_CONFIG_DIR/local.json" +fi + +# If the user is trying to run Consul directly with some arguments, then +# pass them to Consul. +if [ "${1:0:1}" = '-' ]; then + set -- consul "$@" +fi + +# Look for Consul subcommands. +if [ "$1" = 'agent' ]; then + shift + set -- consul agent \ + -data-dir="$CONSUL_DATA_DIR" \ + -config-dir="$CONSUL_CONFIG_DIR" \ + $CONSUL_BIND \ + $CONSUL_CLIENT \ + "$@" +elif [ "$1" = 'version' ]; then + # This needs a special case because there's no help output. + set -- consul "$@" +elif consul --help "$1" 2>&1 | grep -q "consul $1"; then + # We can't use the return code to check for the existence of a subcommand, so + # we have to use grep to look for a pattern in the help output. + set -- consul "$@" +fi + +# NOTE: Unlike in the regular Consul Docker image, we don't have code here +# for changing data-dir directory ownership or using su-exec because OpenShift +# won't run this container as root and so we can't change data dir ownership, +# and there's no need to use su-exec. + +exec "$@" \ No newline at end of file diff --git a/Dockerfile-windows b/Dockerfile-windows new file mode 100644 index 0000000000..14582908db --- /dev/null +++ b/Dockerfile-windows @@ -0,0 +1,51 @@ +FROM mcr.microsoft.com/windows/servercore:ltsc2019 +ARG VERSION=1.16.0 + +ENV chocolateyVersion=1.4.0 + +LABEL org.opencontainers.image.authors="Consul Team " \ + org.opencontainers.image.url="https://www.consul.io/" \ + org.opencontainers.image.documentation="https://www.consul.io/docs" \ + org.opencontainers.image.source="https://github.com/hashicorp/consul" \ + org.opencontainers.image.version=$VERSION \ + org.opencontainers.image.vendor="HashiCorp" \ + org.opencontainers.image.title="consul" \ + org.opencontainers.image.description="Consul is a datacenter runtime that provides service discovery, configuration, and orchestration." \ + version=${VERSION} + +RUN ["powershell", "Set-ExecutionPolicy", "Bypass", "-Scope", "Process", "-Force;"] +RUN ["powershell", "iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))"] + +RUN choco install git.install -yf +RUN SETX /M path "%PATH%;C:\Program Files\Git\bin" + +RUN mkdir C:\\consul +RUN mkdir C:\\consul\\data +RUN mkdir C:\\consul\\config + +# Server RPC is used for communication between Consul clients and servers for internal +# request forwarding. +EXPOSE 8300 + +# Serf LAN and WAN (WAN is used only by Consul servers) are used for gossip between +# Consul agents. LAN is within the datacenter and WAN is between just the Consul +# servers in all datacenters. +EXPOSE 8301 8301/udp 8302 8302/udp + +# HTTP and DNS (both TCP and UDP) are the primary interfaces that applications +# use to interact with Consul. +EXPOSE 8500 8600 8600/udp + +#ENV CONSUL_URL=https://releases.hashicorp.com/consul/${VERSION}/consul_${VERSION}_windows_amd64.zip +#RUN curl %CONSUL_URL% -L -o consul.zip +#RUN tar -xf consul.zip -C consul + +COPY consul.exe C:\\consul + +COPY .release/docker/docker-entrypoint-windows.sh C:\\docker-entrypoint-windows.sh +ENTRYPOINT ["bash.exe", "docker-entrypoint-windows.sh"] + +# By default you'll get an insecure single-node development server that stores +# everything in RAM, exposes a web UI and HTTP endpoints, and bootstraps itself. +# Don't use this configuration for production. +CMD ["agent", "-dev", "-client", "0.0.0.0"] diff --git a/build-support/windows/Dockerfile-consul-dev-windows b/build-support/windows/Dockerfile-consul-dev-windows new file mode 100644 index 0000000000..4e35ccb6e5 --- /dev/null +++ b/build-support/windows/Dockerfile-consul-dev-windows @@ -0,0 +1,4 @@ +ARG VERSION=1.16.0 + +FROM windows/consul:${VERSION}-local +COPY dist/ C:\\consul diff --git a/build-support/windows/Dockerfile-consul-local-windows b/build-support/windows/Dockerfile-consul-local-windows new file mode 100644 index 0000000000..992c48c286 --- /dev/null +++ b/build-support/windows/Dockerfile-consul-local-windows @@ -0,0 +1,52 @@ +ARG VERSION=1.13.3 + +FROM windows/test-sds-server as test-sds-server +FROM docker.mirror.hashicorp.services/windows/openzipkin as openzipkin +FROM windows/consul:${VERSION} + +# Fortio binary downloaded +RUN mkdir fortio +ENV FORTIO_URL=https://github.com/fortio/fortio/releases/download/v1.33.0/fortio_win_1.33.0.zip +RUN curl %FORTIO_URL% -L -o fortio.zip +RUN tar -xf fortio.zip -C fortio + +RUN choco install openssl -yf +RUN choco install jq -yf +RUN choco install netcat -yf +RUN choco install openjdk -yf + +# Install Bats +ENV BATS_URL=https://github.com/bats-core/bats-core/archive/refs/tags/v1.7.0.zip +RUN curl %BATS_URL% -L -o bats.zip +RUN mkdir bats-core +RUN tar -xf bats.zip -C bats-core --strip-components=1 +RUN cd "C:\\Program Files\\Git\\bin" && bash.exe -c "/c/bats-core/install.sh /c/bats" + +# Install Jaeger +ENV JAEGER_URL=https://github.com/jaegertracing/jaeger/releases/download/v1.11.0/jaeger-1.11.0-windows-amd64.tar.gz +RUN curl %JAEGER_URL% -L -o jaeger.tar.gz +RUN mkdir jaeger +RUN tar -xf jaeger.tar.gz -C jaeger --strip-components=1 + +# Install Socat +ENV SOCAT_URL=https://github.com/tech128/socat-1.7.3.0-windows/archive/refs/heads/master.zip +RUN curl %SOCAT_URL% -L -o socat.zip +RUN mkdir socat +RUN tar -xf socat.zip -C socat --strip-components=1 + +# Copy test-sds-server binary and certs +COPY --from=test-sds-server ["C:/go/src/", "C:/test-sds-server/"] + +# Copy openzipkin .jar file +COPY --from=openzipkin ["C:/zipkin", "C:/zipkin"] + +EXPOSE 8300 +EXPOSE 8301 8301/udp 8302 8302/udp +EXPOSE 8500 8600 8600/udp +EXPOSE 8502 + +EXPOSE 19000 19001 19002 19003 19004 +EXPOSE 21000 21001 21002 21003 21004 +EXPOSE 5000 1234 2345 + +RUN SETX /M path "%PATH%;C:\consul;C:\fortio;C:\jaeger;C:\Program Files\Git\bin;C:\Program Files\Git\usr\bin;C:\Program Files\OpenSSL-Win64\bin;C:\bats\bin\;C:\ProgramData\chocolatey\lib\jq\tools;C:\socat" diff --git a/build-support/windows/Dockerfile-openzipkin-windows b/build-support/windows/Dockerfile-openzipkin-windows new file mode 100644 index 0000000000..b23867f0b2 --- /dev/null +++ b/build-support/windows/Dockerfile-openzipkin-windows @@ -0,0 +1,12 @@ +FROM docker.mirror.hashicorp.services/windows/openjdk:1809 + +RUN mkdir zipkin +RUN curl.exe -sSL 'https://search.maven.org/remote_content?g=io.zipkin&a=zipkin-server&v=LATEST&c=exec' -o zipkin/zipkin.jar + +EXPOSE 9410/tcp + +EXPOSE 9411/tcp + +WORKDIR /zipkin + +ENTRYPOINT ["java", "-jar", "zipkin.jar"] \ No newline at end of file diff --git a/build-support/windows/build-consul-dev-image.sh b/build-support/windows/build-consul-dev-image.sh new file mode 100644 index 0000000000..4c03f2f9b0 --- /dev/null +++ b/build-support/windows/build-consul-dev-image.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +cd ../../ +rm -rf dist + +export GOOS=windows GOARCH=amd64 +VERSION=1.16.0 +CONSUL_BUILDDATE=$(date +"%Y-%m-%dT%H:%M:%SZ") +GIT_IMPORT=github.com/hashicorp/consul/version +GOLDFLAGS=" -X $GIT_IMPORT.Version=$VERSION -X $GIT_IMPORT.VersionPrerelease=dev -X $GIT_IMPORT.BuildDate=$CONSUL_BUILDDATE " + +go build -ldflags "$GOLDFLAGS" -o ./dist/ . + +docker build -t windows/consul:${VERSION}-dev -f build-support/windows/Dockerfile-consul-dev-windows . --build-arg VERSION=${VERSION} diff --git a/build-support/windows/build-consul-local-images.sh b/build-support/windows/build-consul-local-images.sh new file mode 100644 index 0000000000..eaa7cc1056 --- /dev/null +++ b/build-support/windows/build-consul-local-images.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash + +readonly HASHICORP_DOCKER_PROXY="docker.mirror.hashicorp.services" + +# Build Consul Version 1.13.3 / 1.12.6 / 1.11.11 +VERSION=${VERSION:-"1.16.0"} +export VERSION + +# Build Windows Envoy Version 1.23.1 / 1.21.5 / 1.20.7 +ENVOY_VERSION=${ENVOY_VERSION:-"1.23.1"} +export ENVOY_VERSION + +echo "Building Images" + + +# Pull Windows Servercore image +echo " " +echo "Pull Windows Servercore image" +docker pull mcr.microsoft.com/windows/servercore:1809 +# Tag Windows Servercore image +echo " " +echo "Tag Windows Servercore image" +docker tag mcr.microsoft.com/windows/servercore:1809 "${HASHICORP_DOCKER_PROXY}/windows/servercore:1809" + + +# Pull Windows Nanoserver image +echo " " +echo "Pull Windows Nanoserver image" +docker pull mcr.microsoft.com/windows/nanoserver:1809 +# Tag Windows Nanoserver image +echo " " +echo "Tag Windows Nanoserver image" +docker tag mcr.microsoft.com/windows/nanoserver:1809 "${HASHICORP_DOCKER_PROXY}/windows/nanoserver:1809" + + +# Pull Windows OpenJDK image +echo " " +echo "Pull Windows OpenJDK image" +docker pull openjdk:windowsservercore-1809 +# Tag Windows OpenJDK image +echo " " +echo "Tag Windows OpenJDK image" +docker tag openjdk:windowsservercore-1809 "${HASHICORP_DOCKER_PROXY}/windows/openjdk:1809" + +# Pull Windows Golang image +echo " " +echo "Pull Windows Golang image" +docker pull golang:1.18.1-nanoserver-1809 +# Tag Windows Golang image +echo " " +echo "Tag Windows Golang image" +docker tag golang:1.18.1-nanoserver-1809 "${HASHICORP_DOCKER_PROXY}/windows/golang:1809" + + +# Pull Kubernetes/pause image +echo " " +echo "Pull Kubernetes/pause image" +docker pull mcr.microsoft.com/oss/kubernetes/pause:3.6 +# Tag Kubernetes/pause image +echo " " +echo "Tag Kubernetes/pause image" +docker tag mcr.microsoft.com/oss/kubernetes/pause:3.6 "${HASHICORP_DOCKER_PROXY}/windows/kubernetes/pause" + +# Pull envoy-windows image +echo " " +echo "Pull envoyproxy/envoy-windows image" +docker pull envoyproxy/envoy-windows:v${ENVOY_VERSION} +# Tag envoy-windows image +echo " " +echo "Tag envoyproxy/envoy-windows image" +docker tag envoyproxy/envoy-windows:v${ENVOY_VERSION} "${HASHICORP_DOCKER_PROXY}/windows/envoy-windows:v${ENVOY_VERSION}" + +# Build Windows Openzipkin Image +docker build -t "${HASHICORP_DOCKER_PROXY}/windows/openzipkin" -f Dockerfile-openzipkin-windows . + + +# Build Windows Test sds server Image +./build-test-sds-server-image.sh + + +# Build windows/consul:${VERSION} Image +echo " " +echo "Build windows/consul:${VERSION} Image" +docker build -t "windows/consul:${VERSION}" -f ../../Dockerfile-windows ../../ --build-arg VERSION=${VERSION} + + +# Build windows/consul:${VERSION}-local Image +echo " " +echo "Build windows/consul:${VERSION}-local Image" +docker build -t windows/consul:${VERSION}-local -f ./Dockerfile-consul-local-windows . --build-arg VERSION=${VERSION} + +echo "Building Complete!" diff --git a/build-support/windows/build-test-sds-server-image.sh b/build-support/windows/build-test-sds-server-image.sh new file mode 100644 index 0000000000..25de3dadbe --- /dev/null +++ b/build-support/windows/build-test-sds-server-image.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +cd ../../test/integration/connect/envoy + +docker build -t windows/test-sds-server -f ./Dockerfile-test-sds-server-windows test-sds-server diff --git a/build-support/windows/windows-test.md b/build-support/windows/windows-test.md new file mode 100644 index 0000000000..5295e40757 --- /dev/null +++ b/build-support/windows/windows-test.md @@ -0,0 +1,119 @@ +# Dockerfiles for Windows Integration Tests + +## Index + +- [About](#about-this-file) +- [Consul Windows](#consul-windows) +- [Consul Windows Local](#consul-windows-local) +- [Consul Windows Dev](#consul-windows-dev) +- [Dockerfile-openzipkin-windows](#dockerfile-openzipkin-windows) + +## About this File + +In this file you will find which Docker images that need to be pre-built to run the Envoy integration tests on Windows, as well as information on how to run each of these files individually for testing purposes. + +## Consul Windows + +The Windows/Consul:_{VERSION}_ image is built from the "Dockerfile-windows" file located at the root of the project. +To do this, the official [windows/servercore image](https://hub.docker.com/_/microsoft-windows-servercore) is used as base image. +To build the image, use the following command: + +```shell +docker build -t windows/consul -f Dockerfile-windows . --build-arg VERSION=${VERSION} +``` + +You can test the built file by running the following command: + +```shell +docker run --rm -p 8300:8300 -p 8301:8301 -p 8302:8302 -p 8500:8500 -p 8600:8600 --name consul --hostname "consul-primary-server" --network-alias "consul-primary-server" windows/consul agent -dev -datacenter "primary" -grpc-port -1 -client "0.0.0.0" -bind "0.0.0.0" +``` + +If everything works properly you should openning the browser and check the Consul UI running on: `http://localhost:8500` + +## Consul Windows Local + +The Windows/Consul:_{VERSION}_-local custom image deployed in the "Dockerfile-consul-local-windows" DockerFile is built from the selected by the shell script _build-consul-local-images.sh_. +When executing it, all the tools required to run the Windows Connect Envoy Integration Tests will be added to the image. +It is necessary that the _"windows/consul"_ image has been built first. This script also takes care of that. + +To build this image you need to run the following command on your terminal: + +```shell +./build-consul-local-images.sh +``` + +> [!NOTE] +> Shell script execution may vary depending on your terminal, we recommend using **Git Bash** for Windows. + +```shell +docker run --rm -p 8300:8300 -p 8301:8301 -p 8302:8302 -p 8500:8500 -p 8600:8600 --name consul-local --hostname "consul-primary-server" --network-alias "consul-primary-server" windows/consul:_{VERSION}_-local agent -dev -datacenter "primary" -grpc-port -1 -client "0.0.0.0" -bind "0.0.0.0" +``` + +If everything works properly you can use your browser and check the Consul UI running on: `http://localhost:8500` + +## Consul Windows Dev + +The Windows/Consul:_{VERSION}_-dev custom image deployed in the "Dockerfile-consul-dev-windows" DockerFile is generated by the shell script named _build-consul-dev-image.sh_. +When executing it, the compilation of Consul is carried out and it is saved in the _"dist"_ directory, this file is then copied to the _"windows/consul:_{VERSION}_-dev"_ image. +It is necessary that the _"windows/consul_{VERSION}_-local"_ image has been built first. + +To build this image you need to run the following command on your terminal: + +```shell +./build-consul-dev-image.sh +``` + +> [!NOTE] +> Shell script execution may vary depending on your terminal, we recommend using **Git Bash** for Windows. + +You can test the built file by running the following command: + +```shell +docker run --rm -p 8300:8300 -p 8301:8301 -p 8302:8302 -p 8500:8500 -p 8600:8600 --name consul-local --hostname "consul-primary-server" --network-alias "consul-primary-server" windows/consul:_{VERSION}_-dev agent -dev -datacenter "primary" -grpc-port -1 -client "0.0.0.0" -bind "0.0.0.0" +``` + +If everything works properly you can use your browser and check the Consul UI running on: `http://localhost:8500` + +## Dockerfile-openzipkin-windows + +Due to the unavailability of an official Openzipkin Docker image for Windows, the [openjdk Windows image](https://hub.docker.com/layers/openjdk/library/openjdk/jdk-windowsservercore-1809/images/sha256-b0cc238d2ec5fb58109a0006ff9e1bcaf66a5301f49bcb8dece9599ac5be6331) was used, where the latest self-contained executable Openzipkin .jar file is downloaded. + +To build this image you need to run the following command on your terminal: + +```shell +docker build -t openzipkin -f Dockerfile-openzipkin-windows . +``` + +You can test the built file by running the following command: + +```shell +docker run --rm --name openzipkin +``` + +If everything works as it should, you will see the zipkin logo being displayed, along with the current version and port configuration: + +```shell +:: version 2.23.18 :: commit 4b71677 :: + +20XX-XX-XX XX:XX:XX.XXX INFO [/] 1252 --- [oss-http-*:9411] c.l.a.s.Server : Serving HTTP at /[0:0:0:0:0:0:0:0]:9411 - http://127.0.0.1:9411/ +``` + +# Testing + +During development, it may be more convenient to check your work-in-progress by running only the tests which you expect to be affected by your changes, as the full test suite can take several minutes to execute. [Go's built-in test tool](https://golang.org/pkg/cmd/go/internal/test/) allows specifying a list of packages to test and the `-run` option to only include test names matching a regular expression. +The `go test -short` flag can also be used to skip slower tests. + +Examples (run from the repository root): + +- `go test -v ./connect` will run all tests in the connect package (see `./connect` folder) +- `go test -v -run TestRetryJoin ./command/agent` will run all tests in the agent package (see `./command/agent` folder) with name substring `TestRetryJoin` + +When a pull request is opened CI will run all tests and lint to verify the change. + +If you want to run the tests on Windows images you must attach the win=true flag. + +Example: + +```shell +go test -v -timeout=30m -tags integration ./test/integration/connect/envoy -run="TestEnvoy/case-badauthz" -win=true +``` diff --git a/test/integration/connect/envoy/Dockerfile-consul-envoy-windows b/test/integration/connect/envoy/Dockerfile-consul-envoy-windows new file mode 100644 index 0000000000..b760fd5754 --- /dev/null +++ b/test/integration/connect/envoy/Dockerfile-consul-envoy-windows @@ -0,0 +1,12 @@ +# From Consul Version 1.13.3 / 1.12.6 / 1.11.11 +ARG VERSION=1.16.0-dev +# From Envoy version 1.23.1 / 1.21.5 / 1.20.7 +ARG ENVOY_VERSION + +FROM docker.mirror.hashicorp.services/windows/envoy-windows:v${ENVOY_VERSION} as envoy +FROM windows/consul:${VERSION} + +# Copy envoy.exe from FROM windows/envoy-windows:${ENVOY_VERSION} +COPY --from=envoy ["C:/Program Files/envoy/", "C:/envoy/"] + +RUN SETX /M path "%PATH%;C:\envoy;" \ No newline at end of file diff --git a/test/integration/connect/envoy/Dockerfile-tcpdump-windows b/test/integration/connect/envoy/Dockerfile-tcpdump-windows new file mode 100644 index 0000000000..cbf5041630 --- /dev/null +++ b/test/integration/connect/envoy/Dockerfile-tcpdump-windows @@ -0,0 +1,7 @@ +FROM mcr.microsoft.com/windows/servercore:ltsc2019 + +COPY ["tcpdump.exe", "C:/Program Files/"] + +ENTRYPOINT ["C:/Program Files/tcpdump.exe"] + +# docker.exe build -t envoy-tcpdump -f Dockerfile-tcpdump-windows . diff --git a/test/integration/connect/envoy/Dockerfile-test-sds-server-windows b/test/integration/connect/envoy/Dockerfile-test-sds-server-windows new file mode 100644 index 0000000000..fc5e45b88d --- /dev/null +++ b/test/integration/connect/envoy/Dockerfile-test-sds-server-windows @@ -0,0 +1,8 @@ +FROM docker.mirror.hashicorp.services/windows/golang:1809 + +WORKDIR /go/src +COPY ./ . + +RUN go build -v -o test-sds-server.exe sds.go + +CMD ["test-sds-server.exe"] diff --git a/test/integration/connect/envoy/WINDOWS-TEST.md b/test/integration/connect/envoy/WINDOWS-TEST.md new file mode 100644 index 0000000000..d9217f7a38 --- /dev/null +++ b/test/integration/connect/envoy/WINDOWS-TEST.md @@ -0,0 +1,40 @@ +# Envoy Integration Tests on Windows + +## Index + +- [About](#about) +- [Pre-built core images](#pre-built-core-images) +- [Test images](#integration-test-images) +- [Run Tests](#run-tests) + +## About + +This file is the entrypoint to understand how to execute Envoy integration tests on Windows as well as to understand the differences between Linux tests and Windows tests. Below you can find a list of relevant documentation that has been written while working on supporting the Envoy integration tests on Windows. + +- [Windows Testing Architecture](test/integration/connect/envoy/docs/windows-testing-architecture.md): On this file you will find why the testing architecture on Windows differs from Linux's. +- [Build Images](build-support-windows/BUILD-IMAGES.md): Here you will find how to build the images required for executing the tests. +- [Windows Troubleshooting](test/integration/connect/envoy/WindowsTroubleshooting.md): This file lists, among other things everything we needed to change/adapt for the existing tests to run in Windows containers. + +## Pre-built core images + +Before running the integration tests, you must pre-build the core images that the tests require to be ran on the Windows environment. Make sure to check out the `BUILD-IMAGES` file [here](build-support-windows/BUILD-IMAGES.md) for this purpose. + +## Integration test images + +During the execution of the integration tests, several images are built based-on the pre-built core images. To get more information about these and how to run them independently, please check out the `docker.windows` file [here](test/integration/connect/envoy/docker.windows.md). + +## Run tests + +To run all the integration tests, you need to execute next command + +```shell +go test -v -timeout=30s -tags integration ./test/integration/connect/envoy -run="TestEnvoy" -win=true +``` + +To run a single test case, the name should be specified. For instance, to run the `case-badauthz` test, you need to execute next command + +```shell +go test -v -timeout=30m -tags integration ./test/integration/connect/envoy -run="TestEnvoy/case-badauthz" -win=true +``` + +> :warning: Note that the flag `-win=true` must be specified as shown in the above commands. This flag is very important because the same allows to indicate that the tests will be executed on the Windows environment. When executing the Envoy integration tests the **End of Line Sequence** of every related file and or script will be automatically changed from **LF to CRLF**. diff --git a/test/integration/connect/envoy/case-dogstatsd-udp/verify.bats b/test/integration/connect/envoy/case-dogstatsd-udp/verify.bats index 55b0ad7684..dfc238ad6d 100644 --- a/test/integration/connect/envoy/case-dogstatsd-udp/verify.bats +++ b/test/integration/connect/envoy/case-dogstatsd-udp/verify.bats @@ -24,14 +24,11 @@ load helpers } @test "s1 proxy should be sending metrics to statsd" { - run retry_default cat /workdir/primary/statsd/statsd.log + run retry_default must_match_in_statsd_logs '^envoy\.' primary - echo "METRICS:" - echo "$output" - echo "COUNT: $(echo "$output" | grep -Ec '^envoy\.')" + echo "METRICS: $output" - [ "$status" == 0 ] - [ $(echo $output | grep -Ec '^envoy\.') -gt "0" ] + [ "$status" == 0 ] } @test "s1 proxy should be sending dogstatsd tagged metrics" { diff --git a/test/integration/connect/envoy/case-gateways-local/secondary/setup.sh b/test/integration/connect/envoy/case-gateways-local/secondary/setup.sh index 938135c9f4..68fd765ad0 100644 --- a/test/integration/connect/envoy/case-gateways-local/secondary/setup.sh +++ b/test/integration/connect/envoy/case-gateways-local/secondary/setup.sh @@ -9,4 +9,4 @@ register_services secondary gen_envoy_bootstrap s2 19001 secondary gen_envoy_bootstrap mesh-gateway 19003 secondary true -retry_default docker_consul secondary curl -s "http://localhost:8500/v1/catalog/service/consul?dc=primary" >/dev/null +retry_default docker_consul secondary curl -s "http://localhost:8500/v1/catalog/service/consul?dc=primary" > /dev/null diff --git a/test/integration/connect/envoy/case-grpc/service_s1.hcl b/test/integration/connect/envoy/case-grpc/service_s1.hcl index b22ba221fa..1a72954240 100644 --- a/test/integration/connect/envoy/case-grpc/service_s1.hcl +++ b/test/integration/connect/envoy/case-grpc/service_s1.hcl @@ -20,7 +20,7 @@ services { protocol = "grpc" envoy_dogstatsd_url = "udp://127.0.0.1:8125" envoy_stats_tags = ["foo=bar"] - envoy_stats_flush_interval = "1s" + envoy_stats_flush_interval = "5s" } } } diff --git a/test/integration/connect/envoy/case-grpc/verify.bats b/test/integration/connect/envoy/case-grpc/verify.bats index 422258a0c0..fc7ca15b5b 100644 --- a/test/integration/connect/envoy/case-grpc/verify.bats +++ b/test/integration/connect/envoy/case-grpc/verify.bats @@ -43,7 +43,7 @@ load helpers metrics_query='envoy.cluster.grpc.PingServer.total.*[#,]local_cluster:s1(,|$)' fi - run retry_default must_match_in_statsd_logs "${metrics_query}" + run retry_long must_match_in_statsd_logs "${metrics_query}" echo "OUTPUT: $output" [ "$status" == 0 ] diff --git a/test/integration/connect/envoy/case-http-badauthz/setup.sh b/test/integration/connect/envoy/case-http-badauthz/setup.sh index 6bd91b1f4e..66a5874129 100644 --- a/test/integration/connect/envoy/case-http-badauthz/setup.sh +++ b/test/integration/connect/envoy/case-http-badauthz/setup.sh @@ -5,10 +5,10 @@ set -eEuo pipefail +register_services primary + # Setup deny intention setup_upsert_l4_intention s1 s2 deny -register_services primary - gen_envoy_bootstrap s1 19000 primary gen_envoy_bootstrap s2 19001 primary diff --git a/test/integration/connect/envoy/case-ingress-gateway-tls/verify.bats b/test/integration/connect/envoy/case-ingress-gateway-tls/verify.bats index 61eaaf97cc..92a158e51b 100644 --- a/test/integration/connect/envoy/case-ingress-gateway-tls/verify.bats +++ b/test/integration/connect/envoy/case-ingress-gateway-tls/verify.bats @@ -19,7 +19,7 @@ load helpers } @test "ingress-gateway should have healthy endpoints for s1" { - assert_upstream_has_endpoints_in_status 127.0.0.1:20000 s1 HEALTHY 1 + assert_upstream_has_endpoints_in_status 127.0.0.1:20000 s1 HEALTHY 1 } @test "should be able to connect to s1 through the TLS-enabled ingress port" { @@ -29,8 +29,8 @@ load helpers run retry_default curl --cacert <(get_ca_root) -s -f -d hello \ --resolve s1.ingress.consul:9998:127.0.0.1 \ https://s1.ingress.consul:9998 - [ "$status" -eq 0 ] - [[ "$output" == *"hello"* ]] + [ "$status" -eq 0 ] + [[ "$output" == *"hello"* ]] } @test "should be able to connect to s1 through the TLS-enabled ingress port using the custom host" { @@ -38,6 +38,6 @@ load helpers run retry_default curl --cacert <(get_ca_root) -s -f -d hello \ --resolve test.example.com:9999:127.0.0.1 \ https://test.example.com:9999 - [ "$status" -eq 0 ] - [[ "$output" == *"hello"* ]] + [ "$status" -eq 0 ] + [[ "$output" == *"hello"* ]] } diff --git a/test/integration/connect/envoy/case-wanfed-gw/global-setup-windows.sh b/test/integration/connect/envoy/case-wanfed-gw/global-setup-windows.sh new file mode 100644 index 0000000000..c63b12a12f --- /dev/null +++ b/test/integration/connect/envoy/case-wanfed-gw/global-setup-windows.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +# initialize the outputs for each dc +for dc in primary secondary; do + rm -rf "workdir/${dc}/tls" + mkdir -p "workdir/${dc}/tls" +done + +container="consul-envoy-integ-tls-init--${CASE_NAME}" + +scriptlet=" +mkdir /out ; +cd /out ; +consul tls ca create ; +consul tls cert create -dc=primary -server -node=pri ; +consul tls cert create -dc=secondary -server -node=sec ; +" + +docker.exe rm -f "$container" &>/dev/null || true +docker.exe run -i --net=none --name="$container" windows/consul:local bash -c "${scriptlet}" + +# primary +for f in \ + consul-agent-ca.pem \ + primary-server-consul-0-key.pem \ + primary-server-consul-0.pem \ + ; do + docker.exe cp "${container}:C:\\Program Files\\Git\\out\\$f" workdir/primary/tls +done + +# secondary +for f in \ + consul-agent-ca.pem \ + secondary-server-consul-0-key.pem \ + secondary-server-consul-0.pem \ + ; do + docker.exe cp "${container}:C:\\Program Files\\Git\\out\\$f" workdir/secondary/tls +done + +# Private keys have 600 perms but tests are run as another user +chmod 666 workdir/primary/tls/primary-server-consul-0-key.pem +chmod 666 workdir/secondary/tls/secondary-server-consul-0-key.pem + +docker.exe rm -f "$container" >/dev/null || true \ No newline at end of file diff --git a/test/integration/connect/envoy/case-zipkin/verify.bats b/test/integration/connect/envoy/case-zipkin/verify.bats index d771d52363..5093456890 100644 --- a/test/integration/connect/envoy/case-zipkin/verify.bats +++ b/test/integration/connect/envoy/case-zipkin/verify.bats @@ -35,14 +35,17 @@ load helpers # Send traced request through upstream. Debug echoes headers back which we can # use to get the traceID generated (no way to force one I can find with Envoy # currently?) - run curl -s -f -H 'x-client-trace-id:test-sentinel' localhost:5000/Debug + # Fixed from /Debug -> /debug. Reason: /Debug return null + run curl -s -f -H 'x-client-trace-id:test-sentinel' localhost:5000/debug -m 5 echo "OUTPUT $output" [ "$status" == "0" ] # Get the traceID from the output - TRACEID=$(echo $output | grep 'X-B3-Traceid:' | cut -c 15-) + # Replaced grep by jq to filter the TraceId. + # Reason: Grep did not filter and return the entire raw string and the test was failing + TRACEID=$(echo $output | jq -rR 'split("X-B3-Traceid: ") | last' | cut -c -16) # Get the trace from Jaeger. Won't bother parsing it just seeing it show up # there is enough to know that the tracing config worked. diff --git a/test/integration/connect/envoy/docker-windows.md b/test/integration/connect/envoy/docker-windows.md new file mode 100644 index 0000000000..a9f2bbcaca --- /dev/null +++ b/test/integration/connect/envoy/docker-windows.md @@ -0,0 +1,42 @@ +# Docker Files for Windows Integration Tests + +## Index + +- [About](#about-this-file) +- [Pre-requisites](#pre-requisites) +- [Dockerfile-test-sds-server-windows](#dockerfile-test-sds-server-windows) + +## About this File + +In this file you will find which Dockerfiles are needed to run the Envoy integration tests on Windows, as well as information on how to run each of these files individually for testing purposes. + +## Pre-requisites + +After building and running the images and containers, you need to have pre-built the base images used by these Dockerfiles. See [pre-built images required in Windows](../../../../build-support-windows/BUILD-IMAGES.md) + +## Dockerfile-test-sds-server-windows + +This file sole purpose is to build the test-sds-server executable using Go. To do so, we use an official [golang image](https://hub.docker.com/_/golang/) provided in docker hub with Windows nano server. +To build this image you need to run the following command on your terminal: + +```shell +docker build -t test-sds-server -f Dockerfile-test-sds-server-windows test-sds-server +``` + +This is the same command used in run-tests.sh + +You can test the built file by running the following command: + +```shell +docker run --rm -p 1234:1234 --name test-sds-server test-sds-server +``` + +If everything works properly you should get the following output: + +```shell +20XX-XX-XXTXX:XX:XX.XXX-XXX [INFO] Loaded cert from file: name=ca-root +20XX-XX-XXTXX:XX:XX.XXX-XXX [INFO] Loaded cert from file: name=foo.example.com +20XX-XX-XXTXX:XX:XX.XXX-XXX [INFO] Loaded cert from file: name=wildcard.ingress.consul +20XX-XX-XXTXX:XX:XX.XXX-XXX [INFO] Loaded cert from file: name=www.example.com +20XX-XX-XXTXX:XX:XX.XXX-XXX [INFO] ==> SDS listening: addr=0.0.0.0:1234 +``` diff --git a/test/integration/connect/envoy/docs/img/linux-arch.png b/test/integration/connect/envoy/docs/img/linux-arch.png new file mode 100644 index 0000000000..710b83b245 Binary files /dev/null and b/test/integration/connect/envoy/docs/img/linux-arch.png differ diff --git a/test/integration/connect/envoy/docs/img/windows-arch-singlecontainer.png b/test/integration/connect/envoy/docs/img/windows-arch-singlecontainer.png new file mode 100644 index 0000000000..1ad5f4aabb Binary files /dev/null and b/test/integration/connect/envoy/docs/img/windows-arch-singlecontainer.png differ diff --git a/test/integration/connect/envoy/docs/img/windows-linux-arch.png b/test/integration/connect/envoy/docs/img/windows-linux-arch.png new file mode 100644 index 0000000000..d1c05533e5 Binary files /dev/null and b/test/integration/connect/envoy/docs/img/windows-linux-arch.png differ diff --git a/test/integration/connect/envoy/docs/windows-testing-architecture.md b/test/integration/connect/envoy/docs/windows-testing-architecture.md new file mode 100644 index 0000000000..73a29e4f17 --- /dev/null +++ b/test/integration/connect/envoy/docs/windows-testing-architecture.md @@ -0,0 +1,106 @@ +# Windows Testing Architecture + +## Index + +- [About](#about) +- [Testing Architectures](#testing-architectures) + - [Linux Test Architecture](#linux-test-architecture) + - [Replicating the Linux Test Architecture on Windows](#replicating-the-linux-test-architecture-on-windows) + - [Single Container Test Architecture](#single-container-test-architecture) + - [Docker Image Components](#docker-image-components) + - Main Components: + - [Bats](#bats) + - [Fortio](#fortio) + - [Jaegertracing](#jaegertracing) + - [Openzipkin](#openzipkin) + - [Socat](#socat) + - Additional tools: + - [Git Bash](#git-bash) + - [JQ](#jq) + - [Netcat](#netcat) + - [Openssl](#openssl) + +## About + +The purpose of this document is not only to explain why the testing architecture is different on Windows but also to describe how the Single Container test architecture is composed. + +## Testing Architectures + +### Linux Test Architecture + +On Linux, tests take advantage of the Host network feature (only available for Linux containers). This means that every container within the network shares the host’s networking namespace. The network stack for every container that uses this network mode won’t be isolated from the Docker host and won’t get their own IP address. + +![linux-architecture](./img/linux-arch.png) + +Every time a test is run, a directory called workdir is created, here all the required files to run the tests are copied. Then this same directory is mounted as a **named volume**, a container with a Kubernetes pause image tagged as *envoy_workdir_1* is run to keep the volume accessible as other containers start while running the tests. Linux containers allow file system operations on runtime unlike Windows containers. + +### Replicating the Linux Test Architecture on Windows + +As we previously mentioned, on Windows there is no Host networking feature, so we went with NAT network instead. The main consequences of this is that now each container has their own networking stack (IP address) separated from each other, they can communicate among themselves using Docker's DNS feature (using the containers name) but no longer through localhost. +Another problem we are facing while sticking to this architecture, is that configuration files assume that every service (services run by fortio and Envoy's sidecar proxy service) are running in localhost. Though we had some partial success on modifying those files on runtime still we are finding issues related to this. +Test's assertions are composed of either function calls or curl executions, we managed this by mapping those calls to the corresponding container name. + +![windows-linux-architecture](./img/windows-linux-arch.png) + +Above, the failing connections are depicted. We kept the same architecture as on Linux and worked around trying to solve those connectivity issues. +Finally, after serveral tries, it was decided that instead of replicating the Linux architecture on Windows, it was more straightforward just to have a single container with all the required components to run the tests. This **single container** test architecture is the approach that works best on Windows. + +## Single Container Test Architecture + +As mentioned above, the single container approach, means building a Windows Docker image not only with Consul and Envoy, but also with all the tools required to execute the existing Envoy integration tests. + +![windows-linux-singlecontainer](./img/windows-singlecontainer.png) + +Below you can find a list and a brief description of those components. + +### Docker Image Components + +The Docker image used for the Consul - Envoy integration tests has several components needed to run those tests. + +- Main Components: + - [Bats](#bats) + - [Fortio](#fortio) + - [Jaegertracing](#jaegertracing) + - [Openzipkin](#openzipkin) + - [Socat](#socat) +- Additional tools: + - [Git Bash](#git-bash) + - [JQ](#jq) + - [Netcat](#netcat) + - [Openssl](#openssl) + +#### Bats + +BATS stands for Bash Automated Testing System and is the one in charge of executing the tests. + +#### Fortio + +Fortio is a microservices (http, grpc) load testing library, command line tool, advanced echo server, and web UI. It is used to run the services registered into Consul during the integration tests. + +#### Jaegertracing + +Jaeger is open source software for tracing transactions between distributed services. It's used for monitoring and troubleshooting complex microservices environments. It is used along with Openzipkin in some test cases. + +#### Openzipkin + +Zipkin is also a tracing software. + +#### Socat + +Socat is a command line based utility that establishes two bidirectional byte streams and transfers data between them. On this integration tests it is used to redirect Envoy's stats. There is no official Windows version. We are using this unofficial release available [here](https://github.com/tech128/socat-1.7.3.0-windows). + +#### Git Bash + +This tool is only used in Windows tests, it was added to the Docker image to be able to use some Linux commands during test execution. + +#### JQ + +Jq is a lightweight and flexible command-line JSON processor. It is used in several tests to modify and filter JSON outputs. + +#### Netcat + +Netcat is a simple program that reads and writes data across networks, much the same way that cat reads and writes data to files. + +#### Openssl + +Open SSL is an all-around cryptography library that offers open-source application of the TLS protocol. It is used to verify that the correct tls certificates are being provisioned during tests. diff --git a/test/integration/connect/envoy/helpers.windows.bash b/test/integration/connect/envoy/helpers.windows.bash new file mode 100644 index 0000000000..2e4a5d4fac --- /dev/null +++ b/test/integration/connect/envoy/helpers.windows.bash @@ -0,0 +1,1192 @@ +#!/bin/bash + +CONSUL_HOSTNAME="" +MOD_ARG="" + +function split_hostport { + local HOSTPORT="$@" + + if [[ $HOSTPORT == *":"* ]]; then + MOD_ARG=$( <<< $HOSTPORT sed 's/:/ /' ) + fi +} + +function get_consul_hostname { + local DC=${1:-primary} + + [[ $XDS_TARGET = "client" ]] && CONSUL_HOSTNAME="consul-$DC-client" || CONSUL_HOSTNAME="consul-$DC" +} + +# retry based on +# https://github.com/fernandoacorreia/azure-docker-registry/blob/master/tools/scripts/create-registry-server +# under MIT license. +function retry { + local n=1 + local max=$1 + shift + local delay=$1 + shift + + local errtrace=0 + if grep -q "errtrace" <<<"$SHELLOPTS" + then + errtrace=1 + set +E + fi + + if [[ $1 == "curl" ]]; then + set -- "${@}" -m 10 + elif [[ $1 == "nc" ]] + then + split_hostport $3 + set -- "${@:1:2}" $MOD_ARG "${@:4}" + fi + + # This if block, was added to check if curl is being executed directly on a test, + # if so, we replace the url parameter with the correct one. + + for ((i=1;i<=$max;i++)) + do + if "$@" + then + if test $errtrace -eq 1 + then + set -E + fi + return 0 + else + echo "Command failed. Attempt $i/$max:" + sleep $delay + fi + done + + if test $errtrace -eq 1; then + set -E + fi + return 1 +} + +function retry_default { + local DEFAULT_TOTAL_RETRIES=5 + set +E + ret=0 + retry $DEFAULT_TOTAL_RETRIES 1 "$@" || ret=1 + set -E + return $ret +} + +function retry_long { + retry 30 1 "$@" +} + +function is_set { + # Arguments: + # $1 - string value to check its truthiness + # + # Return: + # 0 - is truthy (backwards I know but allows syntax like `if is_set ` to work) + # 1 - is not truthy + + local val=$(tr '[:upper:]' '[:lower:]' <<<"$1") + case $val in + 1 | t | true | y | yes) + return 0 + ;; + *) + return 1 + ;; + esac +} + +function get_cert { + local HOSTPORT=$1 + local SERVER_NAME=$2 + local CA_FILE=$3 + local SNI_FLAG="" + if [ -n "$SERVER_NAME" ]; then + SNI_FLAG="-servername $SERVER_NAME" + fi + CERT=$(openssl s_client -connect $HOSTPORT $SNI_FLAG -showcerts /dev/null) + + echo "WANT CN: ${CN} (SNI: ${SERVER_NAME})" + echo "GOT CERT:" + echo "$CERT" + + echo "$CERT" | grep "CN = ${CN}" +} + +function get_upstream_endpoint { + local HOSTPORT=$1 + local CLUSTER_NAME=$2 + run curl -s -f "http://${HOSTPORT}/clusters?format=json" + [ "$status" -eq 0 ] + echo "$output" | jq --raw-output " +.cluster_statuses[] +| select(.name|startswith(\"${CLUSTER_NAME}\"))" +} + +function assert_upstream_missing_once { + local HOSTPORT=$1 + local CLUSTER_NAME=$2 + + run get_upstream_endpoint $HOSTPORT $CLUSTER_NAME + [ "$status" -eq 0 ] + echo "$output" + [ "" == "$output" ] +} + +function assert_upstream_missing { + local HOSTPORT=$1 + local CLUSTER_NAME=$2 + run retry_long assert_upstream_missing_once $HOSTPORT $CLUSTER_NAME + echo "OUTPUT: $output $status" + + [ "$status" -eq 0 ] +} + +function assert_envoy_version { + local ADMINPORT=$1 + run retry_default curl -f -s localhost:$ADMINPORT/server_info + [ "$status" -eq 0 ] + # Envoy 1.8.0 returns a plain text line like + # envoy 5d25f466c3410c0dfa735d7d4358beb76b2da507/1.8.0/Clean/DEBUG live 3 3 0 + # Later versions return JSON. + if (echo $output | grep '^envoy'); then + VERSION=$(echo $output | cut -d ' ' -f 2) + else + VERSION=$(echo $output | jq -r '.version') + fi + echo "Status=$status" + echo "Output=$output" + echo "---" + echo "Got version=$VERSION" + echo "Want version=$ENVOY_VERSION" + + # 1.20.2, 1.19.3 and 1.18.6 are special snowflakes in that the version for + # the release is reported with a '-dev' suffix (eg 1.20.2-dev). + if [ "$ENVOY_VERSION" = "1.20.2" ]; then + ENVOY_VERSION="1.20.2-dev" + elif [ "$ENVOY_VERSION" = "1.19.3" ]; then + ENVOY_VERSION="1.19.3-dev" + elif [ "$ENVOY_VERSION" = "1.18.6" ]; then + ENVOY_VERSION="1.18.6-dev" + fi + + echo $VERSION | grep "/$ENVOY_VERSION/" +} + +function assert_envoy_expose_checks_listener_count { + local HOSTPORT=$1 + local EXPECT_PATH=$2 + + # scrape this once + BODY=$(get_envoy_expose_checks_listener_once $HOSTPORT) + echo "BODY = $BODY" + + CHAINS=$(echo "$BODY" | jq '.active_state.listener.filter_chains | length') + echo "CHAINS = $CHAINS (expect 1)" + [ "${CHAINS:-0}" -eq 1 ] + + RANGES=$(echo "$BODY" | jq '.active_state.listener.filter_chains[0].filter_chain_match.source_prefix_ranges | length') + echo "RANGES = $RANGES (expect 3)" + # note: if IPv6 is not supported in the kernel per + # agent/xds:kernelSupportsIPv6() then this will only be 2 + [ "${RANGES:-0}" -eq 3 ] + + HCM=$(echo "$BODY" | jq '.active_state.listener.filter_chains[0].filters[0]') + HCM_NAME=$(echo "$HCM" | jq -r '.name') + HCM_PATH=$(echo "$HCM" | jq -r '.typed_config.route_config.virtual_hosts[0].routes[0].match.path') + echo "HCM = $HCM" + [ "${HCM_NAME:-}" == "envoy.filters.network.http_connection_manager" ] + [ "${HCM_PATH:-}" == "${EXPECT_PATH}" ] +} + +function get_envoy_expose_checks_listener_once { + local HOSTPORT=$1 + run curl -m 5 -s -f $HOSTPORT/config_dump + [ "$status" -eq 0 ] + echo "$output" | jq --raw-output '.configs[] | select(.["@type"] == "type.googleapis.com/envoy.admin.v3.ListenersConfigDump") | .dynamic_listeners[] | select(.name | startswith("exposed_path_"))' +} + +function get_envoy_public_listener_once { + local HOSTPORT=$1 + run curl -s -f $HOSTPORT/config_dump + [ "$status" -eq 0 ] + echo "$output" | jq --raw-output '.configs[] | select(.["@type"] == "type.googleapis.com/envoy.admin.v3.ListenersConfigDump") | .dynamic_listeners[] | select(.name | startswith("public_listener:"))' +} + +function assert_envoy_http_rbac_policy_count { + local HOSTPORT=$1 + local EXPECT_COUNT=$2 + + GOT_COUNT=$(get_envoy_http_rbac_once $HOSTPORT | jq '.rules.policies | length') + echo "GOT_COUNT = $GOT_COUNT" + [ "${GOT_COUNT:-0}" -eq $EXPECT_COUNT ] +} + +function get_envoy_http_rbac_once { + local HOSTPORT=$1 + run curl -m 5 -s -f $HOSTPORT/config_dump + [ "$status" -eq 0 ] + echo "$output" | jq --raw-output '.configs[2].dynamic_listeners[].active_state.listener.filter_chains[0].filters[0].typed_config.http_filters[] | select(.name == "envoy.filters.http.rbac") | .typed_config' +} + +function assert_envoy_network_rbac_policy_count { + local HOSTPORT=$1 + local EXPECT_COUNT=$2 + + GOT_COUNT=$(get_envoy_network_rbac_once $HOSTPORT | jq '.rules.policies | length') + echo "GOT_COUNT = $GOT_COUNT" + [ "${GOT_COUNT:-0}" -eq $EXPECT_COUNT ] +} + +function get_envoy_network_rbac_once { + local HOSTPORT=$1 + run curl -m 5 -s -f $HOSTPORT/config_dump + [ "$status" -eq 0 ] + echo "$output" | jq --raw-output '.configs[2].dynamic_listeners[].active_state.listener.filter_chains[0].filters[] | select(.name == "envoy.filters.network.rbac") | .typed_config' +} + +function get_envoy_listener_filters { + local HOSTPORT=$1 + run retry_default curl -s -f $HOSTPORT/config_dump + [ "$status" -eq 0 ] + echo "$output" | jq --raw-output '.configs[2].dynamic_listeners[].active_state.listener | "\(.name) \( .filter_chains[0].filters | map(.name) | join(","))"' +} + +function get_envoy_http_filter { + local HOSTPORT=$1 + local FILTER_NAME=$2 + run retry_default curl -s -f $HOSTPORT/config_dump + [ "$status" -eq 0 ] + echo "$output" | jq --raw-output ".configs[2].dynamic_listeners[] | .active_state.listener.filter_chains[].filters[] | select(.name == \"envoy.filters.network.http_connection_manager\") | .typed_config.http_filters[] | select(.name == \"${FILTER_NAME}\")" +} + +function get_envoy_http_filters { + local HOSTPORT=$1 + run retry_default curl -s -f $HOSTPORT/config_dump + [ "$status" -eq 0 ] + echo "$output" | jq --raw-output '.configs[2].dynamic_listeners[].active_state.listener | "\(.name) \( .filter_chains[0].filters[] | select(.name == "envoy.filters.network.http_connection_manager") | .typed_config.http_filters | map(.name) | join(","))"' +} + +function get_envoy_dynamic_cluster_once { + local HOSTPORT=$1 + local NAME_PREFIX=$2 + run curl -m 5 -s -f $HOSTPORT/config_dump + [ "$status" -eq 0 ] + echo "$output" | jq --raw-output ".configs[] | select (.[\"@type\"] == \"type.googleapis.com/envoy.admin.v3.ClustersConfigDump\") | .dynamic_active_clusters[] | select(.cluster.name | startswith(\"${NAME_PREFIX}\"))" +} + +function assert_envoy_dynamic_cluster_exists_once { + local HOSTPORT=$1 + local NAME_PREFIX=$2 + local EXPECT_SNI=$3 + BODY="$(get_envoy_dynamic_cluster_once $HOSTPORT $NAME_PREFIX)" + [ -n "$BODY" ] + + SNI="$(echo "$BODY" | jq --raw-output ".cluster.transport_socket.typed_config.sni | select(. | startswith(\"${EXPECT_SNI}\"))")" + [ -n "$SNI" ] +} + +function assert_envoy_dynamic_cluster_exists { + local HOSTPORT=$1 + local NAME_PREFIX=$2 + local EXPECT_SNI=$3 + run retry_long assert_envoy_dynamic_cluster_exists_once $HOSTPORT $NAME_PREFIX $EXPECT_SNI + [ "$status" -eq 0 ] +} + +function get_envoy_cluster_config { + local HOSTPORT=$1 + local CLUSTER_NAME=$2 + run retry_default curl -s -f $HOSTPORT/config_dump + [ "$status" -eq 0 ] + echo "$output" | jq --raw-output " + .configs[1].dynamic_active_clusters[] + | select(.cluster.name|startswith(\"${CLUSTER_NAME}\")) + | .cluster + " +} + +function get_envoy_stats_flush_interval { + local HOSTPORT=$1 + run retry_default curl -s -f $HOSTPORT/config_dump + [ "$status" -eq 0 ] + #echo "$output" > /workdir/s1_envoy_dump.json + echo "$output" | jq --raw-output '.configs[0].bootstrap.stats_flush_interval' +} + +# snapshot_envoy_admin is meant to be used from a teardown scriptlet from the host. +function snapshot_envoy_admin { + local HOSTPORT=$1 + local ENVOY_NAME=$2 + local DC=${3:-primary} + local OUTDIR="${LOG_DIR}/envoy-snapshots/${DC}/${ENVOY_NAME}" + + mkdir -p "${OUTDIR}" + docker_consul_exec "$DC" bash -c "curl -s http://${HOSTPORT}/config_dump" > "${OUTDIR}/config_dump.json" + docker_consul_exec "$DC" bash -c "curl -s http://${HOSTPORT}/clusters?format=json" > "${OUTDIR}/clusters.json" + docker_consul_exec "$DC" bash -c "curl -s http://${HOSTPORT}/stats" > "${OUTDIR}/stats.txt" + docker_consul_exec "$DC" bash -c "curl -s http://${HOSTPORT}/stats/prometheus" > "${OUTDIR}/stats_prometheus.txt" +} + +function reset_envoy_metrics { + local HOSTPORT=$1 + curl -m 5 -s -f -XPOST $HOSTPORT/reset_counters + return $? +} + +function get_all_envoy_metrics { + local HOSTPORT=$1 + curl -m 5 -s -f $HOSTPORT/stats + return $? +} + +function get_envoy_metrics { + local HOSTPORT=$1 + local METRICS=$2 + + get_all_envoy_metrics $HOSTPORT | grep "$METRICS" +} + +function assert_upstream_has_endpoint_port { + local HOSTPORT=$1 + local CLUSTER_NAME=$2 + local PORT_VALUE=$3 + + run retry_long assert_upstream_has_endpoint_port_once $HOSTPORT $CLUSTER_NAME $PORT_VALUE + [ "$status" -eq 0 ] +} + +function get_upstream_endpoint_in_status_count { + local HOSTPORT=$1 + local CLUSTER_NAME=$2 + local HEALTH_STATUS=$3 + run curl -m 5 -s -f "http://${HOSTPORT}/clusters?format=json" + [ "$status" -eq 0 ] + echo "$output" | jq --raw-output " +.cluster_statuses[] +| select(.name|startswith(\"${CLUSTER_NAME}\")) +| [.host_statuses[].health_status.eds_health_status] +| [select(.[] == \"${HEALTH_STATUS}\")] +| length" +} + +function assert_upstream_has_endpoints_in_status_once { + local HOSTPORT=$1 + local CLUSTER_NAME=$2 + local HEALTH_STATUS=$3 + local EXPECT_COUNT=$4 + + GOT_COUNT=$(get_upstream_endpoint_in_status_count $HOSTPORT $CLUSTER_NAME $HEALTH_STATUS) + + [ "$GOT_COUNT" -eq $EXPECT_COUNT ] +} + +function assert_upstream_has_endpoints_in_status { + local HOSTPORT=$1 + local CLUSTER_NAME=$2 + local HEALTH_STATUS=$3 + local EXPECT_COUNT=$4 + run retry_long assert_upstream_has_endpoints_in_status_once $HOSTPORT $CLUSTER_NAME $HEALTH_STATUS $EXPECT_COUNT + [ "$status" -eq 0 ] +} + +function assert_envoy_metric { + set -eEuo pipefail + local HOSTPORT=$1 + local METRIC=$2 + local EXPECT_COUNT=$3 + + METRICS=$(get_envoy_metrics $HOSTPORT "$METRIC") + + if [ -z "${METRICS}" ]; then + echo "Metric not found" 1>&2 + return 1 + fi + + GOT_COUNT=$(awk -F: '{print $2}' <<<"$METRICS" | head -n 1 | tr -d ' ') + + if [ -z "$GOT_COUNT" ]; then + echo "Couldn't parse metric count" 1>&2 + return 1 + fi + + if [ $EXPECT_COUNT -ne $GOT_COUNT ]; then + echo "$METRIC - expected count: $EXPECT_COUNT, actual count: $GOT_COUNT" 1>&2 + return 1 + fi +} + +function assert_envoy_metric_at_least { + set -eEuo pipefail + local HOSTPORT=$1 + local METRIC=$2 + local EXPECT_COUNT=$3 + + METRICS=$(get_envoy_metrics $HOSTPORT "$METRIC") + + if [ -z "${METRICS}" ]; then + echo "Metric not found" 1>&2 + return 1 + fi + + GOT_COUNT=$(awk -F: '{print $2}' <<<"$METRICS" | head -n 1 | tr -d ' ') + + if [ -z "$GOT_COUNT" ]; then + echo "Couldn't parse metric count" 1>&2 + return 1 + fi + + if [ $EXPECT_COUNT -gt $GOT_COUNT ]; then + echo "$METRIC - expected >= count: $EXPECT_COUNT, actual count: $GOT_COUNT" 1>&2 + return 1 + fi +} + +function assert_envoy_aggregate_metric_at_least { + set -eEuo pipefail + local HOSTPORT=$1 + local METRIC=$2 + local EXPECT_COUNT=$3 + + METRICS=$(get_envoy_metrics $HOSTPORT "$METRIC") + + if [ -z "${METRICS}" ]; then + echo "Metric not found" 1>&2 + return 1 + fi + + GOT_COUNT=$(awk '{ sum += $2 } END { print sum }' <<<"$METRICS") + + if [ -z "$GOT_COUNT" ]; then + echo "Couldn't parse metric count" 1>&2 + return 1 + fi + + if [ $EXPECT_COUNT -gt $GOT_COUNT ]; then + echo "$METRIC - expected >= count: $EXPECT_COUNT, actual count: $GOT_COUNT" 1>&2 + return 1 + fi +} + +function get_healthy_service_count { + local SERVICE_NAME=$1 + local DC=$2 + local NS=$3 + local AP=$4 + local PEER_NAME=$5 + + run curl -m 5 -s -f ${HEADERS} "consul-${DC}-client:8500/v1/health/connect/${SERVICE_NAME}?passing&ns=${NS}&partition=${AP}&peer=${PEER_NAME}" + + [ "$status" -eq 0 ] + echo "$output" | jq --raw-output '. | length' +} + +function assert_alive_wan_member_count { + local DC=$1 + local EXPECT_COUNT=$2 + run retry_long assert_alive_wan_member_count_once $DC $EXPECT_COUNT + [ "$status" -eq 0 ] +} + +function assert_alive_wan_member_count_once { + local DC=$1 + local EXPECT_COUNT=$2 + + GOT_COUNT=$(get_alive_wan_member_count $DC) + + [ "$GOT_COUNT" -eq "$EXPECT_COUNT" ] +} + +function get_alive_wan_member_count { + local DC=$1 + run retry_default curl -sL -f "consul-${DC}-server:8500/v1/agent/members?wan=1" + [ "$status" -eq 0 ] + # echo "$output" >&3 + echo "$output" | jq '.[] | select(.Status == 1) | .Name' | wc -l +} + +function assert_service_has_healthy_instances_once { + local SERVICE_NAME=$1 + local EXPECT_COUNT=$2 + local DC=${3:-primary} + local NS=${4:-} + local AP=${5:-} + local PEER_NAME=${6:-} + + GOT_COUNT=$(get_healthy_service_count "$SERVICE_NAME" "$DC" "$NS" "$AP" "$PEER_NAME") + + [ "$GOT_COUNT" -eq $EXPECT_COUNT ] +} + +function assert_service_has_healthy_instances { + local SERVICE_NAME=$1 + local EXPECT_COUNT=$2 + local DC=${3:-primary} + local NS=${4:-} + local AP=${5:-} + local PEER_NAME=${6:-} + + run retry_long assert_service_has_healthy_instances_once "$SERVICE_NAME" "$EXPECT_COUNT" "$DC" "$NS" "$AP" "$PEER_NAME" + [ "$status" -eq 0 ] +} + +function check_intention { + local SOURCE=$1 + local DESTINATION=$2 + get_consul_hostname primary + + curl -m 5 -s -f "${CONSUL_HOSTNAME}:8500/v1/connect/intentions/check?source=${SOURCE}&destination=${DESTINATION}" | jq ".Allowed" +} + +function assert_intention_allowed { + local SOURCE=$1 + local DESTINATION=$2 + + run check_intention "${SOURCE}" "${DESTINATION}" + [ "$status" -eq 0 ] + [ "$output" = "true" ] +} + +function assert_intention_denied { + local SOURCE=$1 + local DESTINATION=$2 + + run check_intention "${SOURCE}" "${DESTINATION}" + [ "$status" -eq 0 ] + [ "$output" = "false" ] +} + +function docker_consul { + local DC=$1 + shift 1 + retry_default docker_exec envoy_consul-${DC}_1 "$@" +} + +function docker_consul_for_proxy_bootstrap { + local DC=$1 + shift 1 + + local CONTAINER_NAME="$SINGLE_CONTAINER_BASE_NAME"-"$DC"_1 + + docker.exe exec -i $CONTAINER_NAME bash.exe -c "$@" +} + +function docker_exec { + if ! docker.exe exec -i "$@"; then + echo "Failed to execute: docker exec -i $@" 1>&2 + return 1 + fi +} + +function docker_consul_exec { + local DC=$1 + shift 1 + docker_exec envoy_consul-${DC}_1 "$@" +} + +function kill_envoy { + local BOOTSTRAP_NAME=$1 + local DC=${2:-primary} + + PORT=$( cat /c/workdir/$DC/envoy/${BOOTSTRAP_NAME}-bootstrap.json | jq .admin.address.socket_address.port_value ) + PID=$( netstat -qo | grep "127.0.0.1:$PORT" | sed -r "s/.* //g" ) + tskill $PID +} + +function must_match_in_statsd_logs { + local DC=${2:-primary} + local FILE="/c/workdir/${DC}/statsd/statsd.log" + + COUNT=$( grep -Ec $1 $FILE ) + echo "COUNT of '$1' matches: $COUNT" + + [ "$COUNT" -gt "0" ] +} + +function must_match_in_prometheus_response { + run curl -m 5 -f -s $1/metrics + COUNT=$( echo "$output" | grep -Ec $2 ) + + echo "OUTPUT head -n 10" + echo "$output" | head -n 10 + echo "COUNT of '$2' matches: $COUNT" + + [ "$status" == 0 ] + [ "$COUNT" -gt "0" ] +} + +function must_match_in_stats_proxy_response { + run curl -m 5 -f -s $1/$2 + COUNT=$( echo "$output" | grep -Ec $3 ) + + echo "OUTPUT head -n 10" + echo "$output" | head -n 10 + echo "COUNT of '$3' matches: $COUNT" + + [ "$status" == 0 ] + [ "$COUNT" -gt "0" ] +} + +# must_fail_tcp_connection checks that a request made through an upstream fails, +# probably due to authz being denied if all other tests passed already. Although +# we are using curl, this only works as expected for TCP upstreams as we are +# checking TCP-level errors. HTTP upstreams will return a valid 503 generated by +# Envoy rather than a connection-level error. +function must_fail_tcp_connection { + # Attempt to curl through upstream + run curl -m 5 --no-keepalive -s -v -f -d hello $1 + + echo "OUTPUT $output" + + # Should fail during handshake and return "got nothing" error + [ "$status" == "52" ] + + # Verbose output should enclude empty reply + echo "$output" | grep 'Empty reply from server' +} + +function must_pass_tcp_connection { + run curl -m 5 --no-keepalive -s -f -d hello $1 + + echo "OUTPUT $output" + + [ "$status" == "0" ] + [ "$output" = "hello" ] +} + +# must_fail_http_connection see must_fail_tcp_connection but this expects Envoy +# to generate a 503 response since the upstreams have refused connection. +function must_fail_http_connection { + # Attempt to curl through upstream + run curl -m 5 --no-keepalive -s -i -d hello "$1" + + echo "OUTPUT $output" + + [ "$status" == "0" ] + + local expect_response="${2:-403 Forbidden}" + # Should fail request with 503 + echo "$output" | grep "${expect_response}" +} + +# must_pass_http_request allows you to craft a specific http request to assert +# that envoy will NOT reject the request. Primarily of use for testing L7 +# intentions. +function must_pass_http_request { + local METHOD=$1 + local URL=$2 + local DEBUG_HEADER_VALUE="${3:-""}" + + local extra_args + if [[ -n "${DEBUG_HEADER_VALUE}" ]]; then + extra_args="-H x-test-debug:${DEBUG_HEADER_VALUE}" + fi + case "$METHOD" in + GET) ;; + + DELETE) + extra_args="$extra_args -X${METHOD}" + ;; + PUT | POST) + extra_args="$extra_args -d'{}' -X${METHOD}" + ;; + *) + return 1 + ;; + esac + + run curl -m 5 --no-keepalive -v -s -f $extra_args "$URL" + [ "$status" == 0 ] +} + +# must_fail_http_request allows you to craft a specific http request to assert +# that envoy will reject the request. Primarily of use for testing L7 +# intentions. +function must_fail_http_request { + local METHOD=$1 + local URL=$2 + local DEBUG_HEADER_VALUE="${3:-""}" + + local extra_args + if [[ -n "${DEBUG_HEADER_VALUE}" ]]; then + extra_args="-H x-test-debug:${DEBUG_HEADER_VALUE}" + fi + case "$METHOD" in + HEAD) + extra_args="$extra_args -I" + ;; + GET) ;; + + DELETE) + extra_args="$extra_args -X${METHOD}" + ;; + PUT | POST) + extra_args="$extra_args -d'{}' -X${METHOD}" + ;; + *) + return 1 + ;; + esac + + # Attempt to curl through upstream + run curl -m 5 --no-keepalive -s -i $extra_args "$URL" + + echo "OUTPUT $output" + + echo "$output" | grep "403 Forbidden" +} + +function upsert_config_entry { + local DC="$1" + local BODY="$2" + + echo "$BODY" | docker_consul "$DC" consul config write - +} + +function gen_envoy_bootstrap { + SERVICE=$1 + ADMIN_PORT=$2 + DC=${3:-primary} + IS_GW=${4:-0} + EXTRA_ENVOY_BS_ARGS="${5-}" + ADMIN_HOST="0.0.0.0" + + PROXY_ID="$SERVICE" + if ! is_set "$IS_GW"; then + PROXY_ID="$SERVICE-sidecar-proxy" + ADMIN_HOST="127.0.0.1" + fi + + if output=$(docker_consul_for_proxy_bootstrap $DC "consul connect envoy -bootstrap \ + -proxy-id $PROXY_ID \ + -envoy-version "$ENVOY_VERSION" \ + -http-addr envoy_consul-${DC}_1:8500 \ + -grpc-addr envoy_consul-${DC}_1:8502 \ + -admin-access-log-path="C:/envoy/envoy.log" \ + -admin-bind $ADMIN_HOST:$ADMIN_PORT ${EXTRA_ENVOY_BS_ARGS} \ + > /c/workdir/${DC}/envoy/${SERVICE}-bootstrap.json"); then + + # All OK, write config to file + echo $output + #echo "$output" > /c/workdir/${DC}/envoy/$SERVICE-bootstrap.json + else + status=$? + # Command failed, instead of swallowing error (printed on stdout by docker + # it seems) by writing it to file, echo it + echo "$output" + #return $status + fi + + +} + +function read_config_entry { + local KIND=$1 + local NAME=$2 + local DC=${3:-primary} + get_consul_hostname $DC + docker_consul_exec "$DC" bash -c "consul config read -kind $KIND -name $NAME -http-addr=\"$CONSUL_HOSTNAME:8500\"" +} + +function wait_for_namespace { + local NS="${1}" + local DC=${2:-primary} + get_consul_hostname $DC + retry_default docker_consul_exec "$DC" bash -c "curl -sLf http://${CONSUL_HOSTNAME}:8500/v1/namespace/${NS} >/dev/null" +} + +function wait_for_config_entry { + retry_default read_config_entry "$@" +} + +function assert_config_entry_status { + local TYPE="$1" + local STATUS="$2" + local REASON="$3" + local DC="$4" + local KIND="$5" + local NAME="$6" + local NS=${7:-} + local AP=${8:-} + local PEER=${9:-} + + status=$(curl -s -f "consul-${DC}-client:8500/v1/config/${KIND}/${NAME}?passing&ns=${NS}&partition=${AP}&peer=${PEER}" | jq ".Status.Conditions[] | select(.Type == \"$TYPE\" and .Status == \"$STATUS\" and .Reason == \"$REASON\")") + [ -n "$status" ] +} + +function delete_config_entry { + local KIND=$1 + local NAME=$2 + get_consul_hostname primary + retry_default curl -sL -XDELETE "http://${CONSUL_HOSTNAME}:8500/v1/config/${KIND}/${NAME}" +} + +function register_services { + local DC=${1:-primary} + wait_for_leader "$DC" + docker_consul_exec ${DC} bash -c "consul services register workdir/${DC}/register/service_*.hcl" +} + +# wait_for_leader waits until a leader is elected. +# Its first argument must be the datacenter name. +function wait_for_leader { + get_consul_hostname primary + retry_default docker_consul_exec "$1" bash -c "[[ $(curl --fail -sS http://${CONSUL_HOSTNAME}:8500/v1/status/leader) ]]" +} + +function setup_upsert_l4_intention { + local SOURCE=$1 + local DESTINATION=$2 + local ACTION=$3 + get_consul_hostname primary + retry_default docker_consul_exec primary bash -c "curl -sL -X PUT -d '{\"Action\": \"${ACTION}\"}' 'http://${CONSUL_HOSTNAME}:8500/v1/connect/intentions/exact?source=${SOURCE}&destination=${DESTINATION}'" +} + +function upsert_l4_intention { + local SOURCE=$1 + local DESTINATION=$2 + local ACTION=$3 + get_consul_hostname primary + retry_default curl -sL -XPUT "http://${CONSUL_HOSTNAME}:8500/v1/connect/intentions/exact?source=${SOURCE}&destination=${DESTINATION}" \ + -d"{\"Action\": \"${ACTION}\"}" >/dev/null +} + +function get_ca_root { + get_consul_hostname primary + curl -s -f "http://${CONSUL_HOSTNAME}:8500/v1/connect/ca/roots" | jq -r ".Roots[0].RootCert" +} + +function wait_for_agent_service_register { + local SERVICE_ID=$1 + local DC=${2:-primary} + get_consul_hostname $DC + retry_default docker_consul_exec "$DC" bash -c "curl -sLf 'http://${CONSUL_HOSTNAME}:8500/v1/agent/service/${SERVICE_ID}' >/dev/null" +} + +function set_ttl_check_state { + local CHECK_ID=$1 + local CHECK_STATE=$2 + local DC=${3:-primary} + get_consul_hostname $DC + + case "$CHECK_STATE" in + pass) ;; + + warn) ;; + + fail) ;; + + *) + echo "invalid ttl check state '${CHECK_STATE}'" >&2 + return 1 + ;; + esac + + retry_default docker_consul_exec "$DC" bash -c "curl -sL -XPUT 'http://${CONSUL_HOSTNAME}:8500/v1/agent/check/warn/${CHECK_ID}' >/dev/null" +} + +function get_upstream_fortio_name { + local HOST=$1 + local PORT=$2 + local PREFIX=$3 + local DEBUG_HEADER_VALUE="${4:-""}" + local extra_args + if [[ -n "${DEBUG_HEADER_VALUE}" ]]; then + extra_args="-H x-test-debug:${DEBUG_HEADER_VALUE}" + fi + # split proto if https:// is at the front of the host since the --resolve + # string needs just a bare host. + local PROTO="" + local CA_FILE="" + if [ "${HOST:0:8}" = "https://" ]; then + HOST="${HOST:8}" + PROTO="https://" + # Fix in the CA_FILE parameter: for Windows environments, the root path starts with "/c" + extra_args="${extra_args} --cacert /c/workdir/test-sds-server/certs/ca-root.crt" + fi + # We use --resolve instead of setting a Host header since we need the right + # name to be sent for SNI in some cases too. + run retry_default curl --ssl-revoke-best-effort -v -s -f --resolve "${HOST}:${PORT}:127.0.0.1" $extra_args \ + "${PROTO}${HOST}:${PORT}${PREFIX}/debug?env=dump" + + # Useful Debugging but breaks the expectation that the value output is just + # the grep output when things don't fail + if [ "$status" != 0 ]; then + echo "GOT FORTIO OUTPUT: $output" + fi + [ "$status" == 0 ] + echo "$output" | grep -E "^FORTIO_NAME=" +} + +function get_upstream_endpoint_port { + local HOSTPORT=$1 + local CLUSTER_NAME=$2 + local PORT_VALUE=$3 + run curl -s -f "http://${HOSTPORT}/clusters?format=json" + [ "$status" -eq 0 ] + echo "$output" | jq --raw-output " +.cluster_statuses[] +| select(.name|startswith(\"${CLUSTER_NAME}\")) +| [.host_statuses[].address.socket_address.port_value] +| [select(.[] == ${PORT_VALUE})] +| length" +} + +function assert_upstream_has_endpoint_port_once { + local HOSTPORT=$1 + local CLUSTER_NAME=$2 + local PORT_VALUE=$3 + + GOT_COUNT=$(get_upstream_endpoint_port $HOSTPORT $CLUSTER_NAME $PORT_VALUE) + + [ "$GOT_COUNT" -eq 1 ] +} + +function assert_expected_fortio_name { + local EXPECT_NAME=$1 + local HOST=${2:-"localhost"} + local PORT=${3:-5000} + local URL_PREFIX=${4:-""} + local DEBUG_HEADER_VALUE="${5:-""}" + + run get_upstream_fortio_name ${HOST} ${PORT} "${URL_PREFIX}" "${DEBUG_HEADER_VALUE}" + + echo "GOT: $output" + + [ "$status" == 0 ] + [ "$output" == "FORTIO_NAME=${EXPECT_NAME}" ] +} + +function assert_expected_fortio_name_pattern { + local EXPECT_NAME_PATTERN=$1 + local HOST=${2:-"localhost"} + local PORT=${3:-5000} + local URL_PREFIX=${4:-""} + local DEBUG_HEADER_VALUE="${5:-""}" + + GOT=$(get_upstream_fortio_name ${HOST} ${PORT} "${URL_PREFIX}" "${DEBUG_HEADER_VALUE}") + + if [[ "$GOT" =~ $EXPECT_NAME_PATTERN ]]; then + : + else + echo "expected name pattern: $EXPECT_NAME_PATTERN, actual name: $GOT" 1>&2 + return 1 + fi +} + +function get_upstream_fortio_host_header { + local HOST=$1 + local PORT=$2 + local PREFIX=$3 + local DEBUG_HEADER_VALUE="${4:-""}" + local extra_args + if [[ -n "${DEBUG_HEADER_VALUE}" ]]; then + extra_args="-H x-test-debug:${DEBUG_HEADER_VALUE}" + fi + run retry_default curl -v -s -f -H"Host: ${HOST}" $extra_args \ + "localhost:${PORT}${PREFIX}/debug" + [ "$status" == 0 ] + echo "$output" | grep -E "^Host: " +} + +function assert_expected_fortio_host_header { + local EXPECT_HOST=$1 + local HOST=${2:-"localhost"} + local PORT=${3:-5000} + local URL_PREFIX=${4:-""} + local DEBUG_HEADER_VALUE="${5:-""}" + + GOT=$(get_upstream_fortio_host_header ${HOST} ${PORT} "${URL_PREFIX}" "${DEBUG_HEADER_VALUE}") + + if [ "$GOT" != "Host: ${EXPECT_HOST}" ]; then + echo "expected Host header: $EXPECT_HOST, actual Host header: $GOT" 1>&2 + return 1 + fi +} + +function create_peering { + local GENERATE_PEER=$1 + local ESTABLISH_PEER=$2 + run curl -m 5 -sL -XPOST "http://consul-${GENERATE_PEER}-client:8500/v1/peering/token" -d"{ \"PeerName\" : \"${GENERATE_PEER}-to-${ESTABLISH_PEER}\" }" + # echo "$output" >&3 + [ "$status" == 0 ] + + local token + token="$(echo "$output" | jq -r .PeeringToken)" + [ -n "$token" ] + + run curl -m 5 -sLv -XPOST "http://consul-${ESTABLISH_PEER}-client:8500/v1/peering/establish" -d"{ \"PeerName\" : \"${ESTABLISH_PEER}-to-${GENERATE_PEER}\", \"PeeringToken\" : \"${token}\" }" + # echo "$output" >&3 + [ "$status" == 0 ] +} + +function assert_service_has_imported { + local DC=${1:-primary} + local SERVICE_NAME=$2 + local PEER_NAME=$3 + + run curl -s -f "http://consul-${DC}-client:8500/v1/peering/${PEER_NAME}" + [ "$status" == 0 ] + + echo "$output" | jq --raw-output '.StreamStatus.ImportedServices' | grep -e "${SERVICE_NAME}" + if [ $? -ne 0 ]; then + echo "Error finding service: ${SERVICE_NAME}" + return 1 + fi +} + +function get_lambda_envoy_http_filter { + local HOSTPORT=$1 + local NAME_PREFIX=$2 + run retry_default curl -s -f $HOSTPORT/config_dump + [ "$status" -eq 0 ] + # get the full http filter object so the individual fields can be validated. + echo "$output" | jq --raw-output ".configs[2].dynamic_listeners[] | .active_state.listener.filter_chains[].filters[] | select(.name == \"envoy.filters.network.http_connection_manager\") | .typed_config.http_filters[] | select(.name == \"envoy.filters.http.aws_lambda\") | .typed_config" +} + +function register_lambdas { + local DC=${1:-primary} + # register lambdas to the catalog + for f in $(find workdir/${DC}/register -type f -name 'lambda_*.json'); do + retry_default curl -sL -XPUT -d @${f} "http://localhost:8500/v1/catalog/register" >/dev/null && \ + echo "Registered Lambda: $(jq -r .Service.Service $f)" + done + # write service-defaults config entries for lambdas + for f in $(find workdir/${DC}/register -type f -name 'service_defaults_*.json'); do + varsub ${f} AWS_LAMBDA_REGION AWS_LAMBDA_ARN + retry_default curl -sL -XPUT -d @${f} "http://localhost:8500/v1/config" >/dev/null && \ + echo "Wrote config: $(jq -r '.Kind + " / " + .Name' $f)" + done +} + +function assert_lambda_envoy_dynamic_cluster_exists { + local HOSTPORT=$1 + local NAME_PREFIX=$2 + + local BODY=$(get_envoy_dynamic_cluster_once $HOSTPORT $NAME_PREFIX) + [ -n "$BODY" ] + + [ "$(echo $BODY | jq -r '.cluster.transport_socket.typed_config.sni')" == '*.amazonaws.com' ] +} + +function assert_lambda_envoy_dynamic_http_filter_exists { + local HOSTPORT=$1 + local NAME_PREFIX=$2 + local ARN=$3 + + local FILTER=$(get_lambda_envoy_http_filter $HOSTPORT $NAME_PREFIX) + [ -n "$FILTER" ] + + [ "$(echo $FILTER | jq -r '.arn')" == "$ARN" ] +} + +function varsub { + local file=$1 + shift + for v in "$@"; do + sed -i "s/\${$v}/${!v}/g" $file + done +} + +function get_url_header { + local URL=$1 + local HEADER=$2 + run curl -s -f -X GET -I "${URL}" + [ "$status" == 0 ] + RESP=$(echo "$output" | tr -d '\r') + RESP=$(echo "$RESP" | grep -E "^${HEADER}: ") + RESP=$(echo "$RESP" | sed "s/^${HEADER}: //g") + echo "$RESP" +} + +function assert_url_header { + local URL=$1 + local HEADER=$2 + local VALUE=$3 + run get_url_header "$URL" "$HEADER" + [ "$status" == 0 ] + [ "$VALUE" = "$output" ] +} + +# assert_upstream_message asserts both the returned code +# and message from upstream service +function assert_upstream_message { + local HOSTPORT=$1 + run curl -s -d hello localhost:$HOSTPORT + + if [ "$status" -ne 0 ]; then + echo "Command failed" + return 1 + fi + + if (echo $output | grep 'hello'); then + return 0 + fi + + echo "expected message not found in $output" + return 1 +} \ No newline at end of file diff --git a/test/integration/connect/envoy/main_test.go b/test/integration/connect/envoy/main_test.go index b81a72e37d..a035280553 100644 --- a/test/integration/connect/envoy/main_test.go +++ b/test/integration/connect/envoy/main_test.go @@ -7,6 +7,9 @@ package envoy import ( + "flag" + "io/ioutil" + "log" "os" "os/exec" "sort" @@ -16,11 +19,23 @@ import ( "github.com/stretchr/testify/require" ) +var ( + flagWin = flag.Bool("win", false, "Execute tests on windows") +) + func TestEnvoy(t *testing.T) { + flag.Parse() + + if *flagWin == true { + dir := "../../../" + check_dir_files(dir) + } + testcases, err := discoverCases() require.NoError(t, err) runCmd(t, "suite_setup") + defer runCmd(t, "suite_teardown") for _, tc := range testcases { @@ -40,7 +55,8 @@ func TestEnvoy(t *testing.T) { } } -func runCmd(t *testing.T, c string, env ...string) { + +func runCmdLinux(t *testing.T, c string, env ...string) { t.Helper() cmd := exec.Command("./run-tests.sh", c) @@ -52,6 +68,34 @@ func runCmd(t *testing.T, c string, env ...string) { } } +func runCmdWindows(t *testing.T, c string, env ...string) { + t.Helper() + + param_5 := "false" + if env != nil { + param_5 = strings.Join(env, " ") + } + + cmd := exec.Command("cmd", "/C", "bash run-tests.windows.sh", c, param_5) + cmd.Env = append(os.Environ(), env...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + t.Fatalf("command failed: %v", err) + } +} + +func runCmd(t *testing.T, c string, env ...string) { + t.Helper() + + if *flagWin == true { + runCmdWindows(t, c, env...) + + } else { + runCmdLinux(t, c, env...) + } +} + // Discover the cases so we pick up both oss and ent copies. func discoverCases() ([]string, error) { cwd, err := os.Getwd() @@ -74,3 +118,57 @@ func discoverCases() ([]string, error) { sort.Strings(out) return out, nil } + +// CRLF convert functions +// Recursively iterates through the directory passed by parameter looking for the sh and bash files. +// Upon finding them, it calls crlf_file_check. +func check_dir_files(path string) { + files, err := ioutil.ReadDir(path) + if err != nil { + log.Fatal(err) + } + for _, fil := range files { + + v := strings.Split(fil.Name(), ".") + file_extension := v[len(v)-1] + + file_path := path + "/" + fil.Name() + + if fil.IsDir() == true { + check_dir_files(file_path) + } + + if file_extension == "sh" || file_extension == "bash" { + crlf_file_check(file_path) + } + } +} + +// Check if a file contains CRLF line endings if so call crlf_normalize +func crlf_file_check(file_name string) { + + file, err := ioutil.ReadFile(file_name) + text := string(file) + + if edit := crlf_verify(text); edit != -1 { + crlf_normalize(file_name, text) + } + + if err != nil { + log.Fatal(err) + } +} + +// Checks for the existence of CRLF line endings. +func crlf_verify(text string) int { + position := strings.Index(text, "\r\n") + return position +} + +// Replace CRLF line endings with LF. +func crlf_normalize(filename, text string) { + text = strings.Replace(text, "\r\n", "\n", -1) + data := []byte(text) + + ioutil.WriteFile(filename, data, 0644) +} diff --git a/test/integration/connect/envoy/run-tests.windows.sh b/test/integration/connect/envoy/run-tests.windows.sh new file mode 100644 index 0000000000..2388bcd5b6 --- /dev/null +++ b/test/integration/connect/envoy/run-tests.windows.sh @@ -0,0 +1,908 @@ +#!/usr/bin/env bash + +if [ $2 != "false" ] +then + export $2 +fi + +readonly self_name="$0" + +readonly HASHICORP_DOCKER_PROXY="docker.mirror.hashicorp.services" + +readonly SINGLE_CONTAINER_BASE_NAME=envoy_consul + +# DEBUG=1 enables set -x for this script so echos every command run +DEBUG=${DEBUG:-} + +XDS_TARGET=${XDS_TARGET:-server} + +# ENVOY_VERSION to run each test against +ENVOY_VERSION=${ENVOY_VERSION:-"1.23.1"} +export ENVOY_VERSION + +export DOCKER_BUILDKIT=0 + +if [ ! -z "$DEBUG" ] ; then + set -x +fi + +source helpers.windows.bash + +function command_error { + echo "ERR: command exited with status $1" 1>&2 + echo " command: $2" 1>&2 + echo " line: $3" 1>&2 + echo " function: $4" 1>&2 + echo " called at: $5" 1>&2 + # printf '%s\n' "${FUNCNAME[@]}" + # printf '%s\n' "${BASH_SOURCE[@]}" + # printf '%s\n' "${BASH_LINENO[@]}" +} + +trap 'command_error $? "${BASH_COMMAND}" "${LINENO}" "${FUNCNAME[0]:-main}" "${BASH_SOURCE[0]}:${BASH_LINENO[0]}"' ERR + +readonly WORKDIR_SNIPPET="-v envoy_workdir:C:\workdir" + +function network_snippet { + local DC="$1" + echo "--net=envoy-tests" +} + +function aws_snippet { + LAMBDA_TESTS_ENABLED=${LAMBDA_TESTS_ENABLED:-false} + if [ "$LAMBDA_TESTS_ENABLED" != false ]; then + local snippet="" + + # The Lambda integration cases assume that a Lambda function exists in $AWS_REGION with an ARN of $AWS_LAMBDA_ARN. + # The AWS credentials must have permission to invoke the Lambda function. + [ -n "$(set | grep '^AWS_ACCESS_KEY_ID=')" ] && snippet="${snippet} -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID" + [ -n "$(set | grep '^AWS_SECRET_ACCESS_KEY=')" ] && snippet="${snippet} -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY" + [ -n "$(set | grep '^AWS_SESSION_TOKEN=')" ] && snippet="${snippet} -e AWS_SESSION_TOKEN=$AWS_SESSION_TOKEN" + [ -n "$(set | grep '^AWS_LAMBDA_REGION=')" ] && snippet="${snippet} -e AWS_LAMBDA_REGION=$AWS_LAMBDA_REGION" + [ -n "$(set | grep '^AWS_LAMBDA_ARN=')" ] && snippet="${snippet} -e AWS_LAMBDA_ARN=$AWS_LAMBDA_ARN" + + echo "$snippet" + fi +} + +function init_workdir { + local CLUSTER="$1" + + if test -z "$CLUSTER" + then + CLUSTER=primary + fi + + # Note, we use explicit set of dirs so we don't delete .gitignore. Also, + # don't wipe logs between runs as they are already split and we need them to + # upload as artifacts later. + rm -rf workdir/${CLUSTER} + rm -rf workdir/logs + mkdir -p workdir/${CLUSTER}/{consul,consul-server,register,envoy,bats,statsd,data} + + # Reload consul config from defaults + cp consul-base-cfg/*.hcl workdir/${CLUSTER}/consul/ + + # Add any overrides if there are any (no op if not) + find ${CASE_DIR} -maxdepth 1 -name '*.hcl' -type f -exec cp -f {} workdir/${CLUSTER}/consul \; + + # Copy all the test files + find ${CASE_DIR} -maxdepth 1 -name '*.bats' -type f -exec cp -f {} workdir/${CLUSTER}/bats \; + # Copy CLUSTER specific bats + cp helpers.windows.bash workdir/${CLUSTER}/bats/helpers.bash + + # Add any CLUSTER overrides + if test -d "${CASE_DIR}/${CLUSTER}" + then + find ${CASE_DIR}/${CLUSTER} -type f -name '*.hcl' -exec cp -f {} workdir/${CLUSTER}/consul \; + find ${CASE_DIR}/${CLUSTER} -type f -name '*.bats' -exec cp -f {} workdir/${CLUSTER}/bats \; + fi + + # move all of the registration files OUT of the consul config dir now + find workdir/${CLUSTER}/consul -type f -name 'service_*.hcl' -exec mv -f {} workdir/${CLUSTER}/register \; + + # move the server.hcl out of the consul dir so that it doesn't get picked up + # by the client agent (if we're running with XDS_TARGET=client). + if test -f "workdir/${CLUSTER}/consul/server.hcl" + then + mv workdir/${CLUSTER}/consul/server.hcl workdir/${CLUSTER}/consul-server/server.hcl + fi + + # copy the ca-certs for SDS so we can verify the right ones are served + mkdir -p workdir/test-sds-server/certs + cp test-sds-server/certs/ca-root.crt workdir/test-sds-server/certs/ca-root.crt + + if test -d "${CASE_DIR}/data" + then + cp -r ${CASE_DIR}/data/* workdir/${CLUSTER}/data + fi + + return 0 +} + +function docker_kill_rm { + local name + local todo=() + for name in "$@"; do + name="envoy_${name}_1" + if docker.exe container inspect $name &>/dev/null; then + if [[ "$name" == envoy_tcpdump-* ]]; then + echo -n "Gracefully stopping $name..." + docker.exe stop $name &> /dev/null + echo "done" + fi + todo+=($name) + fi + done + + if [[ ${#todo[@]} -eq 0 ]]; then + return 0 + fi + + echo -n "Killing and removing: ${todo[@]}..." + docker.exe rm -v -f ${todo[@]} &> /dev/null + echo "done" +} + +function start_consul { + local DC=${1:-primary} + + # 8500/8502 are for consul + # 9411 is for zipkin which shares the network with consul + # 16686 is for jaeger ui which also shares the network with consul + ports=( + '-p=8500:8500' + '-p=8502:8502' + '-p=9411:9411' + '-p=16686:16686' + ) + case "$DC" in + secondary) + ports=( + '-p=9500:8500' + '-p=9502:8502' + ) + ;; + alpha) + ports=( + '-p=9510:8500' + '-p=9512:8502' + ) + ;; + esac + + license="${CONSUL_LICENSE:-}" + # load the consul license so we can pass it into the consul + # containers as an env var in the case that this is a consul + # enterprise test + if test -z "$license" -a -n "${CONSUL_LICENSE_PATH:-}" + then + license=$(cat $CONSUL_LICENSE_PATH) + fi + + # We currently run these integration tests in two modes: one in which Envoy's + # xDS sessions are served directly by a Consul server, and another in which it + # goes through a client agent. + # + # This is necessary because servers and clients source configuration data in + # different ways (client agents use an RPC-backed cache and servers use their + # own local data) and we want to catch regressions in both. + # + # In the future we should also expand these tests to register services to the + # catalog directly (agentless) rather than relying on the server also being + # an agent. + # + # When XDS_TARGET=client we'll start a Consul server with its gRPC port + # disabled (but only if REQUIRE_PEERS is not set), and a client agent with + # its gRPC port enabled. + # + # When XDS_TARGET=server (or anything else) we'll run a single Consul server + # with its gRPC port enabled. + # + # In either case, the hostname `consul-${DC}-server` should be used as a + # server address (e.g. for WAN joining) and `consul-${DC}-client` should be + # used as a client address (e.g. for interacting with the HTTP API). + # + # Both hostnames work in both modes because we set network aliases on the + # containers such that both hostnames will resolve to the same container when + # XDS_TARGET=server. + # + # We also join containers to the network `container:consul-${DC}_1` in many + # places (see: network_snippet) so that we can curl localhost etc. In both + # modes, you can assume that this name refers to the client's container. + # + # Any .hcl files in the case/cluster directory will be given to both clients + # and servers (via the -config-dir flag) *except for* server.hcl which will + # only be applied to the server (and service registrations which will be made + # against the client). + if [[ "$XDS_TARGET" == "client" ]] + then + docker_kill_rm consul-${DC}-server + docker_kill_rm consul-${DC} + + server_grpc_port="-1" + if is_set $REQUIRE_PEERS; then + server_grpc_port="8502" + fi + + docker.exe run -d --name envoy_consul-${DC}-server_1 \ + --net=envoy-tests \ + $WORKDIR_SNIPPET \ + --hostname "consul-${DC}-server" \ + --network-alias "consul-${DC}-server" \ + -e "CONSUL_LICENSE=$license" \ + windows/consul:local \ + agent -dev -datacenter "${DC}" \ + -config-dir "C:\\workdir\\${DC}\\consul" \ + -config-dir "C:\\workdir\\${DC}\\consul-server" \ + -grpc-port $server_grpc_port \ + -client "0.0.0.0" \ + -bind "0.0.0.0" >/dev/null + + docker.exe run -d --name envoy_consul-${DC}_1 \ + --net=envoy-tests \ + $WORKDIR_SNIPPET \ + --hostname "consul-${DC}-client" \ + --network-alias "consul-${DC}-client" \ + -e "CONSUL_LICENSE=$license" \ + ${ports[@]} \ + windows/consul:local \ + agent -datacenter "${DC}" \ + -config-dir "C:\\workdir\\${DC}\\consul" \ + -data-dir "/tmp/consul" \ + -client "0.0.0.0" \ + -grpc-port 8502 \ + -datacenter "${DC}" \ + -retry-join "consul-${DC}-server" >/dev/null + else + docker_kill_rm consul-${DC} + + docker.exe run -d --name envoy_consul-${DC}_1 \ + --net=envoy-tests \ + $WORKDIR_SNIPPET \ + --memory 4096m \ + --cpus 2 \ + --hostname "consul-${DC}" \ + --network-alias "consul-${DC}-client" \ + --network-alias "consul-${DC}-server" \ + -e "CONSUL_LICENSE=$license" \ + ${ports[@]} \ + windows/consul:local \ + agent -dev -datacenter "${DC}" \ + -config-dir "C:\\workdir\\${DC}\\consul" \ + -config-dir "C:\\workdir\\${DC}\\consul-server" \ + -client "0.0.0.0" >/dev/null + fi +} + +function start_partitioned_client { + local PARTITION=${1:-ap1} + + # Start consul now as setup script needs it up + docker_kill_rm consul-${PARTITION} + + license="${CONSUL_LICENSE:-}" + # load the consul license so we can pass it into the consul + # containers as an env var in the case that this is a consul + # enterprise test + if test -z "$license" -a -n "${CONSUL_LICENSE_PATH:-}" + then + license=$(cat $CONSUL_LICENSE_PATH) + fi + + sh -c "rm -rf /workdir/${PARTITION}/data" + + # Run consul and expose some ports to the host to make debugging locally a + # bit easier. + # + docker.exe run -d --name envoy_consul-${PARTITION}_1 \ + --net=envoy-tests \ + $WORKDIR_SNIPPET \ + --hostname "consul-${PARTITION}-client" \ + --network-alias "consul-${PARTITION}-client" \ + -e "CONSUL_LICENSE=$license" \ + windows/consul:local agent \ + -datacenter "primary" \ + -retry-join "consul-primary-server" \ + -grpc-port 8502 \ + -data-dir "/tmp/consul" \ + -config-dir "C:\\workdir\\${PARTITION}/consul" \ + -client "0.0.0.0" >/dev/null +} + +function pre_service_setup { + local CLUSTER=${1:-primary} + + # Run test case setup (e.g. generating Envoy bootstrap, starting containers) + if [ -f "${CASE_DIR}/${CLUSTER}/setup.sh" ] + then + source ${CASE_DIR}/${CLUSTER}/setup.sh + else + source ${CASE_DIR}/setup.sh + fi +} + +function start_services { + # Start containers required + if [ ! -z "$REQUIRED_SERVICES" ] ; then + docker_kill_rm $REQUIRED_SERVICES + run_containers $REQUIRED_SERVICES + fi + + return 0 +} + +function verify { + local CLUSTER="$1" + if test -z "$CLUSTER"; then + CLUSTER="primary" + fi + + # Execute tests + res=0 + + # Nuke any previous case's verify container. + docker_kill_rm verify-${CLUSTER} + + echo "Running ${CLUSTER} verification step for ${CASE_DIR}..." + + # need to tell the PID 1 inside of the container that it won't be actual PID + # 1 because we're using --pid=host so we use TINI_SUBREAPER + if docker.exe exec -i ${SINGLE_CONTAINER_BASE_NAME}-${CLUSTER}_1 bash \ + -c "TINI_SUBREAPER=1 \ + ENVOY_VERSION=${ENVOY_VERSION} \ + XDS_TARGET=${XDS_TARGET} \ + /c/bats/bin/bats \ + --pretty /c/workdir/${CLUSTER}/bats" ; then + echo "✓ PASS" + else + echo "⨯ FAIL" + res=1 + fi + + return $res +} + +function capture_logs { + local LOG_DIR="workdir/logs/${CASE_DIR}/${ENVOY_VERSION}" + + init_vars + + echo "Capturing Logs" + mkdir -p "$LOG_DIR" + + services="$REQUIRED_SERVICES consul-primary" + if [[ "$XDS_TARGET" == "client" ]] + then + services="$services consul-primary-server" + fi + + if is_set $REQUIRE_SECONDARY + then + services="$services consul-secondary" + + if [[ "$XDS_TARGET" == "client" ]] + then + services="$services consul-secondary-server" + fi + fi + + if is_set $REQUIRE_PARTITIONS + then + services="$services consul-ap1" + fi + if is_set $REQUIRE_PEERS + then + services="$services consul-alpha" + + if [[ "$XDS_TARGET" == "client" ]] + then + services="$services consul-alpha-server" + fi + fi + + if [ -f "${CASE_DIR}/capture.sh" ] + then + echo "Executing ${CASE_DIR}/capture.sh" + source ${CASE_DIR}/capture.sh || true + fi + + for cont in $services; do + echo "Capturing log for $cont" + docker.exe logs "envoy_${cont}_1" &> "${LOG_DIR}/${cont}.log" || { + echo "EXIT CODE $?" > "${LOG_DIR}/${cont}.log" + } + done +} + +function stop_services { + # Teardown + docker_kill_rm $REQUIRED_SERVICES + + docker_kill_rm consul-primary consul-primary-server consul-secondary consul-secondary-server consul-ap1 consul-alpha consul-alpha-server +} + +function init_vars { + source "defaults.sh" + if [ -f "${CASE_DIR}/vars.sh" ] ; then + source "${CASE_DIR}/vars.sh" + fi +} + +function global_setup { + if [ -f "${CASE_DIR}/global-setup-windows.sh" ] ; then + source "${CASE_DIR}/global-setup-windows.sh" + fi +} + +function wipe_volumes { + docker.exe exec -w "C:\workdir" envoy_workdir_1 cmd /c "rd /s /q . 2>nul" +} + +# Windows containers does not allow cp command while running. +function stop_and_copy_files { + # Create CMD file to execute within the container + echo "icacls C:\workdir /grant:r Everyone:(OI)(CI)F /T" > copy.cmd + echo "XCOPY C:\workdir_bak C:\workdir /e /h /c /i /y" > copy.cmd + # Stop dummy container to copy local workdir to container's workdir_bak + docker.exe stop envoy_workdir_1 > /dev/null + docker.exe cp workdir/. envoy_workdir_1:/workdir_bak + # Copy CMD file into container + docker.exe cp copy.cmd envoy_workdir_1:/ + # Start dummy container and execute the CMD file + docker.exe start envoy_workdir_1 > /dev/null + docker.exe exec envoy_workdir_1 copy.cmd + # Delete local CMD file after execution + rm copy.cmd +} + +function run_tests { + CASE_DIR="${CASE_DIR?CASE_DIR must be set to the path of the test case}" + CASE_NAME=$( basename $CASE_DIR | cut -c6- ) + export CASE_NAME + export SKIP_CASE="" + + init_vars + + # Initialize the workdir + init_workdir primary + + if is_set $REQUIRE_SECONDARY + then + init_workdir secondary + fi + if is_set $REQUIRE_PARTITIONS + then + init_workdir ap1 + fi + if is_set $REQUIRE_PEERS + then + init_workdir alpha + fi + + global_setup + + # Allow vars.sh to set a reason to skip this test case based on the ENV + if [ "$SKIP_CASE" != "" ] ; then + echo "SKIPPING CASE: $SKIP_CASE" + return 0 + fi + + # Wipe state + wipe_volumes + + # Copying base files to shared volume + stop_and_copy_files + + # Starting Consul primary cluster + start_consul primary + + if is_set $REQUIRE_SECONDARY; then + start_consul secondary + fi + if is_set $REQUIRE_PARTITIONS; then + docker_consul "primary" consul partition create -name ap1 > /dev/null + start_partitioned_client ap1 + fi + if is_set $REQUIRE_PEERS; then + start_consul alpha + fi + + echo "Setting up the primary datacenter" + pre_service_setup primary + + if is_set $REQUIRE_SECONDARY; then + echo "Setting up the secondary datacenter" + pre_service_setup secondary + fi + if is_set $REQUIRE_PARTITIONS; then + echo "Setting up the non-default partition" + pre_service_setup ap1 + fi + if is_set $REQUIRE_PEERS; then + echo "Setting up the alpha peer" + pre_service_setup alpha + fi + + echo "Starting services" + start_services + + # Run the verify container and report on the output + echo "Verifying the primary datacenter" + verify primary + + if is_set $REQUIRE_SECONDARY; then + echo "Verifying the secondary datacenter" + verify secondary + fi + if is_set $REQUIRE_PEERS; then + echo "Verifying the alpha peer" + verify alpha + fi +} + +function test_teardown { + init_vars + + stop_services +} + +function workdir_cleanup { + docker_kill_rm workdir + docker.exe volume rm -f envoy_workdir &>/dev/null || true +} + + +function suite_setup { + # Cleanup from any previous unclean runs. + suite_teardown + + docker.exe network create -d "nat" envoy-tests &>/dev/null + + # Start the volume container + # + # This is a dummy container that we use to create volume and keep it + # accessible while other containers are down. + docker.exe volume create envoy_workdir &>/dev/null + docker.exe run -d --name envoy_workdir_1 \ + $WORKDIR_SNIPPET \ + --user ContainerAdministrator \ + --net=none \ + "${HASHICORP_DOCKER_PROXY}/windows/kubernetes/pause" &>/dev/null + + # pre-build the consul+envoy container + echo "Rebuilding 'windows/consul:local' image with envoy $ENVOY_VERSION..." + retry_default docker.exe build -t windows/consul:local \ + --build-arg ENVOY_VERSION=${ENVOY_VERSION} \ + -f Dockerfile-consul-envoy-windows . + + + local CONSUL_VERSION=$(docker image inspect --format='{{.ContainerConfig.Labels.version}}' \ + windows/consul:local) + echo "Running Tests with Consul=$CONSUL_VERSION - Envoy=$ENVOY_VERSION - XDS_TARGET=$XDS_TARGET" +} + +function suite_teardown { + docker_kill_rm verify-primary verify-secondary verify-alpha + + # this is some hilarious magic + docker_kill_rm $(grep "^function run_container_" $self_name | \ + sed 's/^function run_container_\(.*\) {/\1/g') + + docker_kill_rm consul-primary consul-primary-server consul-secondary consul-secondary-server consul-ap1 consul-alpha consul-alpha-server + + if docker.exe network inspect envoy-tests &>/dev/null ; then + echo -n "Deleting network 'envoy-tests'..." + docker.exe network rm envoy-tests + echo "done" + fi + + workdir_cleanup +} + +function run_containers { + for name in $@ ; do + run_container $name + done +} + +function run_container { + docker_kill_rm "$1" + "run_container_$1" +} + +function common_run_container_service { + local service="$1" + local CLUSTER="$2" + local httpPort="$3" + local grpcPort="$4" + local CONTAINER_NAME="$SINGLE_CONTAINER_BASE_NAME"-"$CLUSTER"_1 + + docker.exe exec -d $CONTAINER_NAME bash \ + -c "FORTIO_NAME=${service} \ + fortio.exe server \ + -http-port ":$httpPort" \ + -grpc-port ":$grpcPort" \ + -redirect-port disabled" +} + +function run_container_s1 { + common_run_container_service s1 primary 8080 8079 +} + +function run_container_s1-ap1 { + common_run_container_service s1 ap1 8080 8079 +} + +function run_container_s2 { + common_run_container_service s2 primary 8181 8179 +} +function run_container_s2-v1 { + common_run_container_service s2-v1 primary 8182 8178 +} +function run_container_s2-v2 { + common_run_container_service s2-v2 primary 8183 8177 +} + +function run_container_s3 { + common_run_container_service s3 primary 8282 8279 +} +function run_container_s3-v1 { + common_run_container_service s3-v1 primary 8283 8278 +} +function run_container_s3-v2 { + common_run_container_service s3-v2 primary 8284 8277 +} +function run_container_s3-alt { + common_run_container_service s3-alt primary 8286 8280 +} + +function run_container_s4 { + common_run_container_service s4 primary 8382 8281 +} + +function run_container_s1-secondary { + common_run_container_service s1-secondary secondary 8080 8079 +} + +function run_container_s2-secondary { + common_run_container_service s2-secondary secondary 8181 8179 +} + +function run_container_s2-ap1 { + common_run_container_service s2 ap1 8480 8479 +} + +function run_container_s3-ap1 { + common_run_container_service s3 ap1 8580 8579 +} + +function run_container_s1-alpha { + common_run_container_service s1-alpha alpha 8080 8079 +} + +function run_container_s2-alpha { + common_run_container_service s2-alpha alpha 8181 8179 +} + +function run_container_s3-alpha { + common_run_container_service s3-alpha alpha 8282 8279 +} + +function common_run_container_sidecar_proxy { + local service="$1" + local CLUSTER="$2" + local CONTAINER_NAME="$SINGLE_CONTAINER_BASE_NAME"-"$CLUSTER"_1 + + # Hot restart breaks since both envoys seem to interact with each other + # despite separate containers that don't share IPC namespace. Not quite + # sure how this happens but may be due to unix socket being in some shared + # location? + docker.exe exec -d $CONTAINER_NAME bash \ + -c "envoy.exe \ + -c /c/workdir/${CLUSTER}/envoy/${service}-bootstrap.json \ + -l trace \ + --disable-hot-restart \ + --drain-time-s 1 >/dev/null" +} + +function run_container_s1-sidecar-proxy { + common_run_container_sidecar_proxy s1 primary +} + +function run_container_s1-ap1-sidecar-proxy { + common_run_container_sidecar_proxy s1 ap1 +} + +function run_container_s1-sidecar-proxy-consul-exec { + local CLUSTER="primary" + local CONTAINER_NAME="$SINGLE_CONTAINER_BASE_NAME"-"$CLUSTER"_1 + local ADMIN_HOST="127.0.0.1" + local ADMIN_PORT="19000" + + docker.exe exec -d $CONTAINER_NAME bash \ + -c "consul connect envoy -sidecar-for s1 \ + -http-addr $CONTAINER_NAME:8500 \ + -grpc-addr $CONTAINER_NAME:8502 \ + -admin-bind $ADMIN_HOST:$ADMIN_PORT \ + -envoy-version ${ENVOY_VERSION} \ + -- \ + -l trace >/dev/null" +} + +function run_container_s2-sidecar-proxy { + common_run_container_sidecar_proxy s2 primary +} +function run_container_s2-v1-sidecar-proxy { + common_run_container_sidecar_proxy s2-v1 primary +} +function run_container_s2-v2-sidecar-proxy { + common_run_container_sidecar_proxy s2-v2 primary +} + +function run_container_s3-sidecar-proxy { + common_run_container_sidecar_proxy s3 primary +} +function run_container_s3-v1-sidecar-proxy { + common_run_container_sidecar_proxy s3-v1 primary +} +function run_container_s3-v2-sidecar-proxy { + common_run_container_sidecar_proxy s3-v2 primary +} + +function run_container_s3-alt-sidecar-proxy { + common_run_container_sidecar_proxy s3-alt primary +} + +function run_container_s1-sidecar-proxy-secondary { + common_run_container_sidecar_proxy s1 secondary +} +function run_container_s2-sidecar-proxy-secondary { + common_run_container_sidecar_proxy s2 secondary +} + +function run_container_s2-ap1-sidecar-proxy { + common_run_container_sidecar_proxy s2 ap1 +} + +function run_container_s3-ap1-sidecar-proxy { + common_run_container_sidecar_proxy s3 ap1 +} + +function run_container_s1-sidecar-proxy-alpha { + common_run_container_sidecar_proxy s1 alpha +} +function run_container_s2-sidecar-proxy-alpha { + common_run_container_sidecar_proxy s2 alpha +} +function run_container_s3-sidecar-proxy-alpha { + common_run_container_sidecar_proxy s3 alpha +} + +function common_run_container_gateway { + local name="$1" + local DC="$2" + local CONTAINER_NAME="$SINGLE_CONTAINER_BASE_NAME"-"$DC"_1 + + # Hot restart breaks since both envoys seem to interact with each other + # despite separate containers that don't share IPC namespace. Not quite + # sure how this happens but may be due to unix socket being in some shared + # location? + docker.exe exec -d $CONTAINER_NAME bash \ + -c "envoy.exe \ + -c /c/workdir/${DC}/envoy/${name}-bootstrap.json \ + -l trace \ + --disable-hot-restart \ + --drain-time-s 1 >/dev/null" +} + +function run_container_gateway-primary { + common_run_container_gateway mesh-gateway primary +} +function run_container_gateway-secondary { + common_run_container_gateway mesh-gateway secondary +} +function run_container_gateway-alpha { + common_run_container_gateway mesh-gateway alpha +} + +function run_container_ingress-gateway-primary { + common_run_container_gateway ingress-gateway primary +} + +function run_container_api-gateway-primary { + common_run_container_gateway api-gateway primary +} + +function run_container_terminating-gateway-primary { + common_run_container_gateway terminating-gateway primary +} + +function run_container_fake-statsd { + local CONTAINER_NAME="$SINGLE_CONTAINER_BASE_NAME"-"primary"_1 + # This magic SYSTEM incantation is needed since Envoy doesn't add newlines and so + # we need each packet to be passed to echo to add a new line before + # appending. But it does not work on Windows. + docker.exe exec -d $CONTAINER_NAME bash -c "socat -u UDP-RECVFROM:8125,fork,reuseaddr OPEN:/workdir/primary/statsd/statsd.log,create,append" +} + +function run_container_zipkin { + docker.exe run -d --name $(container_name) \ + $WORKDIR_SNIPPET \ + $(network_snippet primary) \ + "${HASHICORP_DOCKER_PROXY}/windows/openzipkin" +} + +function run_container_jaeger { + echo "Starting Jaeger service..." + + local DC=${1:-primary} + local CONTAINER_NAME="$SINGLE_CONTAINER_BASE_NAME"-"$DC"_1 + + docker.exe exec -d $CONTAINER_NAME bash -c "jaeger-all-in-one.exe \ + --collector.zipkin.http-port=9411" +} + +function run_container_test-sds-server { + echo "Starting test-sds-server" + + local DC=${1:-primary} + local CONTAINER_NAME="$SINGLE_CONTAINER_BASE_NAME"-"$DC"_1 + + docker.exe exec -d $CONTAINER_NAME bash -c "cd /c/test-sds-server && + ./test-sds-server.exe" +} + +function container_name { + echo "envoy_${FUNCNAME[1]/#run_container_/}_1" +} +function container_name_prev { + echo "envoy_${FUNCNAME[2]/#run_container_/}_1" +} + +# This is a debugging tool. Run via 'bash run-tests.sh debug_dump_volumes' on Powershell +function debug_dump_volumes { + local LINUX_PATH=$(pwd) + local WIN_PATH=$( echo "$LINUX_PATH" | sed 's/^\/mnt//' | sed -e 's/^\///' -e 's/\//\\/g' -e 's/^./\0:/' ) + docker.exe run -it \ + $WORKDIR_SNIPPET \ + -v "$WIN_PATH":"C:\\cwd" \ + --net=none \ + "${HASHICORP_DOCKER_PROXY}/windows/nanoserver:1809" \ + cmd /c "xcopy \workdir \cwd\workdir /E /H /C /I /Y" +} + +function run_container_tcpdump-primary { + # To use add "tcpdump-primary" to REQUIRED_SERVICES + common_run_container_tcpdump primary +} +function run_container_tcpdump-secondary { + # To use add "tcpdump-secondary" to REQUIRED_SERVICES + common_run_container_tcpdump secondary +} +function run_container_tcpdump-alpha { + # To use add "tcpdump-alpha" to REQUIRED_SERVICES + common_run_container_tcpdump alpha +} + +function common_run_container_tcpdump { + local DC="$1" + + # we cant run this in circle but its only here to temporarily enable. + +# docker.exe build --rm=false -t envoy-tcpdump -f Dockerfile-tcpdump-windows . + + docker.exe run -d --name $(container_name_prev) \ + $(network_snippet $DC) \ + envoy-tcpdump \ + -v -i any \ + -w "/data/${DC}.pcap" +} + +case "${1-}" in + "") + echo "command required" + exit 1 ;; + *) + "$@" ;; +esac diff --git a/test/integration/connect/envoy/windows-troubleshooting.md b/test/integration/connect/envoy/windows-troubleshooting.md new file mode 100644 index 0000000000..a3a83e0888 --- /dev/null +++ b/test/integration/connect/envoy/windows-troubleshooting.md @@ -0,0 +1,90 @@ +# Envoy Integration Tests on Windows + +## Index + +- [About this Guide](#about-this-guide) +- [Prerequisites](#prerequisites) +- [Running the Tests](#running-the-tests) +- [Troubleshooting](#troubleshooting) + - [About Envoy Integration Tests on Windows](#about-envoy-integration-tests-on-windows) + - [Common Errors](#common-errors) +- [Windows Scripts Changes](#windows-scripts-changes) +- [Volume Issues](#volume-issues) + +## About this Guide + +On this guide you will find all the information required to run the Envoy integration tests on Windows. + +## Prerequisites + +To run the integration tests yo will need to have the following installed on your System: + +- GO v1.18(or later). +- Gotestsum library [installation](https://pkg.go.dev/gotest.tools/gotestsum). +- Docker. + +Before running the tests, you will need to build the required Docker images, to do so, you can use the script provided [here](../../../../build-support-windows/build-images.sh): + +- Build Images Script Execution + - From a Bash console (GitBash or WSL) execute: `./build-images.sh` + +## Running the Tests + +To execute the tests you need to run the following command depending on the shell you are using: +**On Powershell**: +`go test -v -timeout=30m -tags integration ./test/integration/connect/envoy -run="TestEnvoy/" -win=true` +Where **TEST CASE** is the individual test case we want to execute (e.g. case-badauthz). + +**On Git Bash**: +`ENVOY_VERSION= go test -v -timeout=30m -tags integration ./test/integration/connect/envoy -run="TestEnvoy/" -win=true` +Where **TEST CASE** is the individual test case we want to execute (e.g. case-badauthz), and **ENVOY VERSION** is the version which you are currently testing. + +> [!TIP] +> When executing the integration tests using **Powershell** you may need to set the ENVOY_VERSION value manually in line 20 of the [run-tests.windows.sh](run-tests.windows.sh) file. + +> [!WARNING] +> When executing the integration tests for Windows environments, the **End of Line Sequence** of every related file and/or script will be changed from **LF** to **CRLF**. + +### About Envoy Integration Tests on Windows + +Integration tests on Linux run a multi-container architecture that take advantage of the Host Network Docker feature, using this feature means that the container's network stack is not isolated from the Docker host (the container shares the host’s networking namespace), and the container does not get its own IP-address allocated (read more about this [here](https://docs.docker.com/network/host/)). This feature is only available for Linux, which made migrating the tests to Windows challenging, since replicating the same architecture created more issues, that's why a **single container** architecture was chosen to run the Envoy integration tests. +Using a single container architecture meant that we could use the same tests as on linux, moreover we were able to speed-up their execution by replacing *docker run* commands which started utility containers, for *docker exec* commands. + +### Common errors + +If the tests are executed without docker running, the following error will be seen: + +```powershell +error during connect: This error may indicate that the docker daemon is not running.: Post "http://%2F%2F.%2Fpipe%2Fdocker_engine/v1.24/build?buildargs=%7B%7D&cachefrom=%5B%5D&cgroupparent=&cpuperiod=0&cpuquota=0&cpusetcpus=&cpusetmems=&cpushares=0&dockerfile=Dockerfile-bats-windows&labels=%7B%7D&memory=0&memswap=0&networkmode=default&rm=1&shmsize=0&t=bats-verify&target=&ulimits=null&version=1": open //./pipe/docker_engine: The system cannot find the file specified. +``` + +If any of the docker images does not exist or is mistagged, an error similar to the following will be displayed: + +```powershell +Error response from daemon: No such container: envoy_workdir_1 +``` + +If you run the Windows tests from WSL you will get the following error message: + +```bash +main_test.go:34: command failed: exec: "cmd": executable file not found in $PATH +``` + +## Windows Scripts Changes + +- The "http-addr", "grpc-addr" and "admin-access-log-path" flags were added to the creation of the Envoy Bootstrap files. +- To execute commands sh was replaced by bash on our Windows container. +- All paths were updated to use Windows format. +- Created *stop_and_copy_files* function to copy files into the shared volume (see [volume issues](#volume-issues)). +- Changed the *-admin-bind* value from `0.0.0.0` to `127.0.0.1` when generating the Envoy Bootstrap files. +- Removed the *&&* from the *common_run_container_service's* docker exec command and replaced it with *\*. +- Removed *docker_wget* and *docker_curl* functions from [helpers.windows.bash](helpers.windows.bash) file and replaced them with **docker_consul_exec**, this way we avoid starting intermediate containers when capturing logs. +- The function *wipe_volumes* uses a `docker exec` command instead of the original `docker run`, this way we speed up test execution by avoiding to start a new container just to delete volume content before each test run. +- For **case-grpc** we increased the `envoy_stats_flush_interval` value from 1s to 5s, on Windows, the original value caused the test to pass or fail randomly. +- For **case-wanfed-gw** a new script was created: **global-setup-windows.sh**, this file replaces global-setup.sh when running this test in Windows. The new script uses the windows/consul:local Docker image to generate the required TLS files and copies them into host's workdir directory. +- To use the **debug_dump_volumes** function, you need to use it via Powershell and execute the following command: `bash run-tests.windows.sh debug_dump_volumes` Make sure to be positioned with your terminal in the correct directory. +- For **case-consul-exec** this case can only be run when using the consul-dev Docker image on this repository, since it relies on features implemented only here. These features are: Windows valid default value for "-admin-access-log-path" and `consul connect envoy` command starts Envoy. This features have also been submitted in [PR#15114](https://github.com/hashicorp/consul/pull/15114). + +## Volume Issues + +Another difference that arose when migrating the tests from Linux to Windows, is that file system operations can't be executed while Windows containers are running. Currently, when running the tests a **named volume** is created and all of the required files are copied into that volume. Because of the constraint mentioned before, the workaround we implemented was creating a function (**stop_and_copy_files**) that stops the *kubernetes/pause* container and executes a script to copy the required files and finally starts the container again.