diff --git a/.circleci/config.yml b/.circleci/config.yml index 4425e83b2d..925d1a47a7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -520,7 +520,7 @@ jobs: # Circle images but pick Go since we have to pick one of them. - image: *GOLANG_IMAGE environment: - ENVOY_VERSIONS: "1.11.2" + ENVOY_VERSION: "1.11.2" steps: &ENVOY_INTEGRATION_TEST_STEPS - checkout # Get go binary from workspace @@ -531,33 +531,42 @@ jobs: - run: docker build -t consul-dev -f ./build-support/docker/Consul-Dev.dockerfile . - run: name: Envoy Integration Tests - command: make test-envoy-integ SKIP_DOCKER_BUILD=1 + command: | + mkdir -p /tmp/test-results/ + gotestsum -- -timeout=30m -tags integration ./test/integration/connect/envoy environment: + GOTESTSUM_JUNITFILE: /tmp/test-results/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 # tput complains if this isn't set to something. TERM: ansi - store_artifacts: path: ./test/integration/connect/envoy/workdir/logs destination: container-logs + - store_test_results: + path: *TEST_RESULTS_DIR + - store_artifacts: + path: *TEST_RESULTS_DIR envoy-integration-test-1.12.2: docker: - image: *GOLANG_IMAGE environment: - ENVOY_VERSIONS: "1.12.2" + ENVOY_VERSION: "1.12.2" steps: *ENVOY_INTEGRATION_TEST_STEPS envoy-integration-test-1.13.1: docker: - image: *GOLANG_IMAGE environment: - ENVOY_VERSIONS: "1.13.1" + ENVOY_VERSION: "1.13.1" steps: *ENVOY_INTEGRATION_TEST_STEPS envoy-integration-test-1.14.1: docker: - image: *GOLANG_IMAGE environment: - ENVOY_VERSIONS: "1.14.1" + ENVOY_VERSION: "1.14.1" steps: *ENVOY_INTEGRATION_TEST_STEPS # run integration tests for the connect ca providers diff --git a/GNUmakefile b/GNUmakefile index fa6a711213..fc7c3b3070 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -354,7 +354,7 @@ ui-docker: ui-build-image @$(SHELL) $(CURDIR)/build-support/scripts/build-docker.sh ui test-envoy-integ: $(ENVOY_INTEG_DEPS) - @$(SHELL) $(CURDIR)/test/integration/connect/envoy/run-tests.sh + @go test -v -timeout=30m -tags integration ./test/integration/connect/envoy test-connect-ca-providers: ifeq ("$(CIRCLECI)","true") diff --git a/test/integration/connect/envoy/helpers.bash b/test/integration/connect/envoy/helpers.bash index e5430474f5..a163d261f1 100755 --- a/test/integration/connect/envoy/helpers.bash +++ b/test/integration/connect/envoy/helpers.bash @@ -431,7 +431,7 @@ function docker_wget { function docker_curl { local DC=$1 shift 1 - docker run -ti --rm --network container:envoy_consul-${DC}_1 --entrypoint curl consul-dev "$@" + docker run --rm --network container:envoy_consul-${DC}_1 --entrypoint curl consul-dev "$@" } function docker_exec { diff --git a/test/integration/connect/envoy/main_test.go b/test/integration/connect/envoy/main_test.go new file mode 100644 index 0000000000..68e1d9e02c --- /dev/null +++ b/test/integration/connect/envoy/main_test.go @@ -0,0 +1,69 @@ +// +build integration + +package envoy + +import ( + "os" + "os/exec" + "testing" +) + +func TestEnvoy(t *testing.T) { + var testcases = []string{ + "case-badauthz", + "case-basic", + "case-centralconf", + "case-cfg-resolver-dc-failover-gateways-none", + "case-cfg-resolver-dc-failover-gateways-remote", + "case-cfg-resolver-defaultsubset", + "case-cfg-resolver-subset-onlypassing", + "case-cfg-resolver-subset-redirect", + "case-cfg-resolver-svc-failover", + "case-cfg-resolver-svc-redirect-http", + "case-cfg-resolver-svc-redirect-tcp", + "case-consul-exec", + "case-dogstatsd-udp", + "case-gateways-local", + "case-gateways-remote", + "case-gateway-without-services", + "case-grpc", + "case-http", + "case-http2", + "case-http-badauthz", + "case-ingress-gateway-http", + "case-ingress-gateway-multiple-services", + "case-ingress-gateway-simple", + "case-ingress-mesh-gateways-resolver", + "case-multidc-rsa-ca", + "case-prometheus", + "case-statsd-udp", + "case-stats-proxy", + "case-terminating-gateway-simple", + "case-terminating-gateway-subsets", + "case-terminating-gateway-without-services", + "case-upstream-config", + "case-wanfed-gw", + "case-zipkin", + } + + runCmd(t, "suite_setup") + defer runCmd(t, "suite_teardown") + + for _, tc := range testcases { + t.Run(tc, func(t *testing.T) { + runCmd(t, "run_tests", "CASE_DIR="+tc) + }) + } +} + +func runCmd(t *testing.T, c string, env ...string) { + t.Helper() + + cmd := exec.Command("./run-tests.sh", c) + cmd.Env = append(os.Environ(), env...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + t.Fatalf("command failed: %v", err) + } +} diff --git a/test/integration/connect/envoy/run-tests.sh b/test/integration/connect/envoy/run-tests.sh index 55ce31b0fd..8058912075 100755 --- a/test/integration/connect/envoy/run-tests.sh +++ b/test/integration/connect/envoy/run-tests.sh @@ -1,57 +1,20 @@ -#!/bin/bash +#!/usr/bin/env bash set -eEuo pipefail # DEBUG=1 enables set -x for this script so echos every command run DEBUG=${DEBUG:-} -# FILTER_TESTS="" skips any test whose CASENAME doesn't match the -# pattern. CASENAME is combination of the name from the case- dir and the -# envoy version for example: "http, envoy 1.8.0". The pattern is passed to grep -# over that string. -FILTER_TESTS=${FILTER_TESTS:-} - -# STOP_ON_FAIL exits after a case fails so the workdir state can be viewed and -# the components interacted with to debug the failure. This is useful when tests -# only fail when run as part of a whole suite but work in isolation. -STOP_ON_FAIL=${STOP_ON_FAIL:-} - -# ENVOY_VERSIONS is the list of envoy versions to run each test against -ENVOY_VERSIONS=${ENVOY_VERSIONS:-"1.11.2 1.12.3 1.13.1 1.14.1"} +# ENVOY_VERSION to run each test against +ENVOY_VERSION=${ENVOY_VERSION:-"1.14.1"} +export ENVOY_VERSION if [ ! -z "$DEBUG" ] ; then set -x fi -DIR=$(cd -P -- "$(dirname -- "$0")" && pwd -P) - -cd $DIR - -LEAVE_CONSUL_UP=${LEAVE_CONSUL_UP:-} -PROXY_LOGS_ON_FAIL=${PROXY_LOGS_ON_FAIL:-} - source helpers.bash -RESULT=1 -CLEANED_UP=0 - -function cleanup { - local STATUS="$?" - - if [ "$CLEANED_UP" != 0 ] ; then - return - fi - CLEANED_UP=1 - - if [ "$STATUS" -ne 0 ] - then - capture_logs - fi - - docker-compose down -v --remove-orphans -} -trap cleanup EXIT - function command_error { echo "ERR: command exited with status $1" 1>&2 echo " command: $2" 1>&2 @@ -65,12 +28,6 @@ function command_error { trap 'command_error $? "${BASH_COMMAND}" "${LINENO}" "${FUNCNAME[0]:-main}" "${BASH_SOURCE[0]}:${BASH_LINENO[0]}"' ERR -# Cleanup from any previous unclean runs. -docker-compose down -v --remove-orphans - -# Start the volume container -docker-compose up -d workdir - function init_workdir { local DC="$1" @@ -123,11 +80,11 @@ function pre_service_setup { local DC=${1:-primary} # Run test case setup (e.g. generating Envoy bootstrap, starting containers) - if [ -f "${CASE_DIR}${DC}/setup.sh" ] + if [ -f "${CASE_DIR}/${DC}/setup.sh" ] then - source ${CASE_DIR}${DC}/setup.sh + source ${CASE_DIR}/${DC}/setup.sh else - source ${CASE_DIR}setup.sh + source ${CASE_DIR}/setup.sh fi } @@ -147,18 +104,13 @@ function start_services { function verify { local DC=$1 - if test -z "$DC" - then + if test -z "$DC"; then DC=primary fi # Execute tests res=0 - echo "- - - - - - - - - - - - - - - - - - - - - - - -" - echoblue -n "CASE $CASE_STR" - echo -n ": " - # Nuke any previous case's verify container. docker-compose rm -s -v -f verify-${DC} || true @@ -168,13 +120,12 @@ function verify { echored "⨯ FAIL" res=1 fi - echo "================================================" return $res } function capture_logs { - echo "Capturing Logs for $CASE_STR" + echo "Capturing Logs" mkdir -p "$LOG_DIR" services="$REQUIRED_SERVICES consul-primary" if is_set $REQUIRE_SECONDARY @@ -182,10 +133,10 @@ function capture_logs { services="$services consul-secondary" fi - if [ -f "${CASE_DIR}capture.sh" ] + if [ -f "${CASE_DIR}/capture.sh" ] then - echo "Executing ${CASE_DIR}capture.sh" - source ${CASE_DIR}capture.sh || true + echo "Executing ${CASE_DIR}/capture.sh" + source ${CASE_DIR}/capture.sh || true fi @@ -197,29 +148,34 @@ function capture_logs { } function stop_services { - # Teardown - if [ -f "${CASE_DIR}teardown.sh" ] ; then - source "${CASE_DIR}teardown.sh" + if [ -f "${CASE_DIR}/teardown.sh" ] ; then + source "${CASE_DIR}/teardown.sh" fi docker-compose rm -s -v -f $REQUIRED_SERVICES || true } -function initVars { +function init_vars { source "defaults.sh" - if [ -f "${CASE_DIR}vars.sh" ] ; then - source "${CASE_DIR}vars.sh" + if [ -f "${CASE_DIR}/vars.sh" ] ; then + source "${CASE_DIR}/vars.sh" fi } function global_setup { - if [ -f "${CASE_DIR}global-setup.sh" ] ; then - source "${CASE_DIR}global-setup.sh" + if [ -f "${CASE_DIR}/global-setup.sh" ] ; then + source "${CASE_DIR}/global-setup.sh" fi } -function runTest { - initVars +function run_tests { + CASE_DIR="${CASE_DIR?CASE_DIR must be set to the path of the test case}" + CASE_NAME=$( basename $CASE_DIR | cut -c6- ) + export CASE_NAME + + export LOG_DIR="workdir/logs/${CASE_DIR}/${ENVOY_VERSION}" + + init_vars # Initialize the workdir init_workdir primary @@ -315,43 +271,29 @@ function runTest { return $TESTRESULT } +function suite_setup { + # Set a log dir to prevent docker-compose warning about unset var + export LOG_DIR="workdir/logs/" + # Cleanup from any previous unclean runs. + docker-compose down --volumes --timeout 0 --remove-orphans -RESULT=0 + # Start the volume container + docker-compose up -d workdir +} -for c in ./case-*/ ; do - for ev in $ENVOY_VERSIONS ; do - export CASE_DIR="${c}" - export CASE_NAME=$( basename $c | cut -c6- ) - export CASE_ENVOY_VERSION="envoy $ev" - export CASE_STR="$CASE_NAME, $CASE_ENVOY_VERSION" - export ENVOY_VERSION="${ev}" - export LOG_DIR="workdir/logs/${CASE_DIR}/${ENVOY_VERSION}" - echo "================================================" - echoblue "CASE $CASE_STR" - echo "- - - - - - - - - - - - - - - - - - - - - - - -" +function suite_teardown { + # Set a log dir to prevent docker-compose warning about unset var + export LOG_DIR="workdir/logs/" - if [ ! -z "$FILTER_TESTS" ] && echo "$CASE_STR" | grep -v "$FILTER_TESTS" > /dev/null ; then - echo " SKIPPED: doesn't match FILTER_TESTS=$FILTER_TESTS" - continue 1 - fi + docker-compose down --volumes --timeout 0 --remove-orphans +} - if ! runTest - then - RESULT=1 - fi - if [ $RESULT -ne 0 ] && [ ! -z "$STOP_ON_FAIL" ] ; then - echo " => STOPPING because STOP_ON_FAIL set" - break 2 - fi - done -done +case "${1-}" in + "") + echo "command required" + exit 1 ;; + *) + "$@" ;; +esac -cleanup - -if [ $RESULT -eq 0 ] ; then - echogreen "✓ PASS" -else - echored "⨯ FAIL" - exit 1 -fi