From 94d232ab1e764d146dcaf1b0f098c18bc33d2ed1 Mon Sep 17 00:00:00 2001 From: cskh Date: Wed, 2 Nov 2022 14:08:17 -0400 Subject: [PATCH] integ test: reduce flakiness due to compound output from retry (#15233) * integ test: avoid flakiness due to compound output from retry * changelog --- .changelog/15233.txt | 3 + .../connect/envoy/case-basic/verify.bats | 6 +- .../envoy/case-centralconf/verify.bats | 2 +- .../primary/verify.bats | 8 +- .../verify.bats | 3 +- .../verify.bats | 3 +- .../primary/verify.bats | 2 +- .../primary/verify.bats | 2 +- .../case-cross-peers-http/primary/verify.bats | 2 +- .../primary/verify.bats | 2 +- .../case-cross-peers/primary/verify.bats | 2 +- .../envoy/case-expose-checks/verify.bats | 2 +- .../case-gateways-local/primary/verify.bats | 6 +- .../case-gateways-remote/primary/verify.bats | 2 +- .../connect/envoy/case-http/verify.bats | 14 +- .../case-ingress-gateway-simple/verify.bats | 2 +- .../case-ingress-gateway-tls/verify.bats | 8 +- .../primary/verify.bats | 10 +- .../case-multidc-rsa-ca/primary/verify.bats | 4 +- .../connect/envoy/case-prometheus/verify.bats | 2 +- .../envoy/case-stats-proxy/verify.bats | 8 +- .../verify.bats | 4 +- .../verify.bats | 4 +- test/integration/connect/envoy/helpers.bash | 208 ++++++++---------- 24 files changed, 147 insertions(+), 162 deletions(-) create mode 100644 .changelog/15233.txt diff --git a/.changelog/15233.txt b/.changelog/15233.txt new file mode 100644 index 0000000000..7c121db067 --- /dev/null +++ b/.changelog/15233.txt @@ -0,0 +1,3 @@ +```release-note: improvement +integ test: fix flakiness due to test condition from retry app endoint +``` diff --git a/test/integration/connect/envoy/case-basic/verify.bats b/test/integration/connect/envoy/case-basic/verify.bats index 65e70523d3..d89194bdab 100644 --- a/test/integration/connect/envoy/case-basic/verify.bats +++ b/test/integration/connect/envoy/case-basic/verify.bats @@ -33,13 +33,13 @@ load helpers @test "s1 upstream should be able to connect to s2" { run retry_default curl -s -f -d hello localhost:5000 [ "$status" -eq 0 ] - [ "$output" = "hello" ] + [[ "$output" == *"hello"* ]] } @test "s1 proxy should have been configured with one rbac listener filter at L4" { LISTEN_FILTERS=$(get_envoy_listener_filters localhost:19000) - PUB=$(echo "$LISTEN_FILTERS" | grep -E "^public_listener:" | cut -f 2 -d ' ' ) - UPS=$(echo "$LISTEN_FILTERS" | grep -E "^(default\/default\/)?s2:" | cut -f 2 -d ' ' ) + PUB=$(echo "$LISTEN_FILTERS" | grep -E "^public_listener:" | cut -f 2 -d ' ') + UPS=$(echo "$LISTEN_FILTERS" | grep -E "^(default\/default\/)?s2:" | cut -f 2 -d ' ') echo "LISTEN_FILTERS = $LISTEN_FILTERS" echo "PUB = $PUB" diff --git a/test/integration/connect/envoy/case-centralconf/verify.bats b/test/integration/connect/envoy/case-centralconf/verify.bats index 0f4221ab5f..f2668ffcd2 100644 --- a/test/integration/connect/envoy/case-centralconf/verify.bats +++ b/test/integration/connect/envoy/case-centralconf/verify.bats @@ -29,7 +29,7 @@ load helpers @test "s1 upstream should be able to connect to s2 with http/1.1" { run retry_default curl --http1.1 -s -f -d hello localhost:5000 [ "$status" -eq 0 ] - [ "$output" = "hello" ] + [[ "$output" == *"hello"* ]] } @test "s1 proxy should be exposing metrics to prometheus from central config" { diff --git a/test/integration/connect/envoy/case-cfg-resolver-cluster-peering-failover/primary/verify.bats b/test/integration/connect/envoy/case-cfg-resolver-cluster-peering-failover/primary/verify.bats index 5b059beceb..7c8d9ee8ae 100644 --- a/test/integration/connect/envoy/case-cfg-resolver-cluster-peering-failover/primary/verify.bats +++ b/test/integration/connect/envoy/case-cfg-resolver-cluster-peering-failover/primary/verify.bats @@ -49,11 +49,10 @@ load helpers assert_upstream_has_endpoints_in_status 127.0.0.1:19000 failover-target~s2.default.primary-to-alpha.external HEALTHY 1 } - @test "s1 upstream should be able to connect to s2" { run retry_default curl -s -f -d hello localhost:5000 [ "$status" -eq 0 ] - [ "$output" = "hello" ] + [[ "$output" == *"hello"* ]] } @test "s1 upstream made 1 connection" { @@ -68,7 +67,6 @@ load helpers assert_service_has_healthy_instances s2 0 primary } - @test "s1 upstream should have healthy endpoints for s2 in the failover cluster peer" { assert_upstream_has_endpoints_in_status 127.0.0.1:19000 failover-target~s2.default.primary.internal UNHEALTHY 1 assert_upstream_has_endpoints_in_status 127.0.0.1:19000 failover-target~s2.default.primary-to-alpha.external HEALTHY 1 @@ -85,7 +83,7 @@ load helpers @test "s1 upstream should be able to connect to s2 in the failover cluster peer" { run retry_default curl -s -f -d hello localhost:5000 [ "$status" -eq 0 ] - [ "$output" = "hello" ] + [[ "$output" == *"hello"* ]] } @test "s1 upstream made 1 connection to s2 through the cluster peer" { @@ -105,7 +103,7 @@ load helpers @test "s1 upstream should be able to connect to s2 via virtual-s2" { run retry_default curl -s -f -d hello localhost:5001 [ "$status" -eq 0 ] - [ "$output" = "hello" ] + [[ "$output" == *"hello"* ]] } @test "s1 upstream made 1 connection to s2 via virtual-s2 through the cluster peer" { diff --git a/test/integration/connect/envoy/case-cfg-resolver-svc-redirect-http/verify.bats b/test/integration/connect/envoy/case-cfg-resolver-svc-redirect-http/verify.bats index 39462d6e15..d252e07059 100644 --- a/test/integration/connect/envoy/case-cfg-resolver-svc-redirect-http/verify.bats +++ b/test/integration/connect/envoy/case-cfg-resolver-svc-redirect-http/verify.bats @@ -37,10 +37,9 @@ load helpers @test "s1 upstream should be able to connect to its upstream simply" { run retry_default curl -s -f -d hello localhost:5000 [ "$status" -eq 0 ] - [ "$output" = "hello" ] + [[ "$output" == *"hello"* ]] } @test "s1 upstream should be able to connect to s3 via upstream s2" { assert_expected_fortio_name s3 } - diff --git a/test/integration/connect/envoy/case-cfg-resolver-svc-redirect-tcp/verify.bats b/test/integration/connect/envoy/case-cfg-resolver-svc-redirect-tcp/verify.bats index 39462d6e15..d252e07059 100644 --- a/test/integration/connect/envoy/case-cfg-resolver-svc-redirect-tcp/verify.bats +++ b/test/integration/connect/envoy/case-cfg-resolver-svc-redirect-tcp/verify.bats @@ -37,10 +37,9 @@ load helpers @test "s1 upstream should be able to connect to its upstream simply" { run retry_default curl -s -f -d hello localhost:5000 [ "$status" -eq 0 ] - [ "$output" = "hello" ] + [[ "$output" == *"hello"* ]] } @test "s1 upstream should be able to connect to s3 via upstream s2" { assert_expected_fortio_name s3 } - diff --git a/test/integration/connect/envoy/case-cross-peer-control-plane-mgw/primary/verify.bats b/test/integration/connect/envoy/case-cross-peer-control-plane-mgw/primary/verify.bats index bc21d5ee3f..f14e12dd8a 100644 --- a/test/integration/connect/envoy/case-cross-peer-control-plane-mgw/primary/verify.bats +++ b/test/integration/connect/envoy/case-cross-peer-control-plane-mgw/primary/verify.bats @@ -57,7 +57,7 @@ load helpers @test "s1 upstream should be able to connect to s2" { run retry_default curl -s -f -d hello localhost:5000 [ "$status" -eq 0 ] - [ "$output" = "hello" ] + [[ "$output" == *"hello"* ]] } @test "s1 upstream made 1 connection to s2" { diff --git a/test/integration/connect/envoy/case-cross-peers-http-router/primary/verify.bats b/test/integration/connect/envoy/case-cross-peers-http-router/primary/verify.bats index 68a3fe5011..c6b2d59d6f 100644 --- a/test/integration/connect/envoy/case-cross-peers-http-router/primary/verify.bats +++ b/test/integration/connect/envoy/case-cross-peers-http-router/primary/verify.bats @@ -57,7 +57,7 @@ load helpers @test "s1 upstream should be able to connect to s2 with http/1.1" { run retry_default curl --http1.1 -s -f -d hello localhost:5000 [ "$status" -eq 0 ] - [ "$output" = "hello" ] + [[ "$output" == *"hello"* ]] } @test "s1 upstream should be able to connect to s2 via s2" { diff --git a/test/integration/connect/envoy/case-cross-peers-http/primary/verify.bats b/test/integration/connect/envoy/case-cross-peers-http/primary/verify.bats index bdc7ac390f..9e59677b9b 100644 --- a/test/integration/connect/envoy/case-cross-peers-http/primary/verify.bats +++ b/test/integration/connect/envoy/case-cross-peers-http/primary/verify.bats @@ -49,7 +49,7 @@ load helpers @test "s1 upstream should be able to connect to s2 with http/1.1" { run retry_default curl --http1.1 -s -f -d hello localhost:5000 [ "$status" -eq 0 ] - [ "$output" = "hello" ] + [[ "$output" == *"hello"* ]] } @test "s1 upstream made 1 connection to s2" { diff --git a/test/integration/connect/envoy/case-cross-peers-resolver-redirect-tcp/primary/verify.bats b/test/integration/connect/envoy/case-cross-peers-resolver-redirect-tcp/primary/verify.bats index 1f2b26d7ac..e2673269ef 100644 --- a/test/integration/connect/envoy/case-cross-peers-resolver-redirect-tcp/primary/verify.bats +++ b/test/integration/connect/envoy/case-cross-peers-resolver-redirect-tcp/primary/verify.bats @@ -49,7 +49,7 @@ load helpers @test "s1 upstream should be able to connect to s2" { run retry_default curl -s -f -d hello localhost:5000 [ "$status" -eq 0 ] - [ "$output" = "hello" ] + [[ "$output" == *"hello"* ]] } @test "s1 upstream should be able to connect to s3 via s2 due to redirect" { diff --git a/test/integration/connect/envoy/case-cross-peers/primary/verify.bats b/test/integration/connect/envoy/case-cross-peers/primary/verify.bats index af45d2ad3f..840db4021f 100644 --- a/test/integration/connect/envoy/case-cross-peers/primary/verify.bats +++ b/test/integration/connect/envoy/case-cross-peers/primary/verify.bats @@ -49,7 +49,7 @@ load helpers @test "s1 upstream should be able to connect to s2" { run retry_default curl -s -f -d hello localhost:5000 [ "$status" -eq 0 ] - [ "$output" = "hello" ] + [[ "$output" == *"hello"* ]] } @test "s1 upstream made 1 connection to s2" { diff --git a/test/integration/connect/envoy/case-expose-checks/verify.bats b/test/integration/connect/envoy/case-expose-checks/verify.bats index 0c4dcbb2c1..7467e6c55c 100644 --- a/test/integration/connect/envoy/case-expose-checks/verify.bats +++ b/test/integration/connect/envoy/case-expose-checks/verify.bats @@ -33,7 +33,7 @@ load helpers @test "s1 upstream should be able to connect to s2" { run retry_default curl -s -f -d hello localhost:5000 [ "$status" -eq 0 ] - [ "$output" = "hello" ] + [[ "$output" == *"hello"* ]] } @test "s2 exposes checks on a new listener" { diff --git a/test/integration/connect/envoy/case-gateways-local/primary/verify.bats b/test/integration/connect/envoy/case-gateways-local/primary/verify.bats index 1d55c794e0..cde8cedb0d 100644 --- a/test/integration/connect/envoy/case-gateways-local/primary/verify.bats +++ b/test/integration/connect/envoy/case-gateways-local/primary/verify.bats @@ -24,17 +24,17 @@ load helpers } @test "gateway-primary should have healthy endpoints for secondary" { - assert_upstream_has_endpoints_in_status 127.0.0.1:19002 secondary HEALTHY 1 + assert_upstream_has_endpoints_in_status 127.0.0.1:19002 secondary HEALTHY 1 } @test "gateway-secondary should have healthy endpoints for s2" { - assert_upstream_has_endpoints_in_status consul-secondary-client:19003 s2 HEALTHY 1 + assert_upstream_has_endpoints_in_status consul-secondary-client:19003 s2 HEALTHY 1 } @test "s1 upstream should be able to connect to s2" { run retry_default curl -s -f -d hello localhost:5000 [ "$status" -eq 0 ] - [ "$output" = "hello" ] + [[ "$output" == *"hello"* ]] } @test "s1 upstream made 1 connection" { diff --git a/test/integration/connect/envoy/case-gateways-remote/primary/verify.bats b/test/integration/connect/envoy/case-gateways-remote/primary/verify.bats index 825872e57d..9bbfeb6a89 100644 --- a/test/integration/connect/envoy/case-gateways-remote/primary/verify.bats +++ b/test/integration/connect/envoy/case-gateways-remote/primary/verify.bats @@ -22,7 +22,7 @@ load helpers @test "s1 upstream should be able to connect to s2" { run retry_default curl -s -f -d hello localhost:5000 [ "$status" -eq 0 ] - [ "$output" = "hello" ] + [[ "$output" == *"hello"* ]] } @test "s1 upstream made 1 connection" { diff --git a/test/integration/connect/envoy/case-http/verify.bats b/test/integration/connect/envoy/case-http/verify.bats index 0f65d46d30..434abbd209 100644 --- a/test/integration/connect/envoy/case-http/verify.bats +++ b/test/integration/connect/envoy/case-http/verify.bats @@ -30,13 +30,13 @@ load helpers @test "s1 upstream should be able to connect to s2 with http/1.1" { run retry_default curl --http1.1 -s -f -d hello localhost:5000 [ "$status" -eq 0 ] - [ "$output" = "hello" ] + [[ "$output" == *"hello"* ]] } @test "s1 proxy should have been configured with http connection managers" { LISTEN_FILTERS=$(get_envoy_listener_filters localhost:19000) - PUB=$(echo "$LISTEN_FILTERS" | grep -E "^public_listener:" | cut -f 2 -d ' ' ) - UPS=$(echo "$LISTEN_FILTERS" | grep -E "^(default\/default\/)?s2:" | cut -f 2 -d ' ' ) + PUB=$(echo "$LISTEN_FILTERS" | grep -E "^public_listener:" | cut -f 2 -d ' ') + UPS=$(echo "$LISTEN_FILTERS" | grep -E "^(default\/default\/)?s2:" | cut -f 2 -d ' ') echo "LISTEN_FILTERS = $LISTEN_FILTERS" echo "PUB = $PUB" @@ -48,7 +48,7 @@ load helpers @test "s2 proxy should have been configured with an http connection manager" { LISTEN_FILTERS=$(get_envoy_listener_filters localhost:19001) - PUB=$(echo "$LISTEN_FILTERS" | grep -E "^public_listener:" | cut -f 2 -d ' ' ) + PUB=$(echo "$LISTEN_FILTERS" | grep -E "^public_listener:" | cut -f 2 -d ' ') echo "LISTEN_FILTERS = $LISTEN_FILTERS" echo "PUB = $PUB" @@ -58,8 +58,8 @@ load helpers @test "s1 proxy should have been configured with http rbac filters" { HTTP_FILTERS=$(get_envoy_http_filters localhost:19000) - PUB=$(echo "$HTTP_FILTERS" | grep -E "^public_listener:" | cut -f 2 -d ' ' ) - UPS=$(echo "$HTTP_FILTERS" | grep -E "^(default\/default\/)?s2:" | cut -f 2 -d ' ' ) + PUB=$(echo "$HTTP_FILTERS" | grep -E "^public_listener:" | cut -f 2 -d ' ') + UPS=$(echo "$HTTP_FILTERS" | grep -E "^(default\/default\/)?s2:" | cut -f 2 -d ' ') echo "HTTP_FILTERS = $HTTP_FILTERS" echo "PUB = $PUB" @@ -71,7 +71,7 @@ load helpers @test "s2 proxy should have been configured with http rbac filters" { HTTP_FILTERS=$(get_envoy_http_filters localhost:19001) - PUB=$(echo "$HTTP_FILTERS" | grep -E "^public_listener:" | cut -f 2 -d ' ' ) + PUB=$(echo "$HTTP_FILTERS" | grep -E "^public_listener:" | cut -f 2 -d ' ') echo "HTTP_FILTERS = $HTTP_FILTERS" echo "PUB = $PUB" diff --git a/test/integration/connect/envoy/case-ingress-gateway-simple/verify.bats b/test/integration/connect/envoy/case-ingress-gateway-simple/verify.bats index 9d0735b42c..ea83b36d14 100644 --- a/test/integration/connect/envoy/case-ingress-gateway-simple/verify.bats +++ b/test/integration/connect/envoy/case-ingress-gateway-simple/verify.bats @@ -42,5 +42,5 @@ load helpers @test "ingress should be able to connect to s1 via configured port" { run retry_default curl -s -f -d hello localhost:9999 [ "$status" -eq 0 ] - [ "$output" = "hello" ] + [[ "$output" == *"hello"* ]] } diff --git a/test/integration/connect/envoy/case-ingress-gateway-tls/verify.bats b/test/integration/connect/envoy/case-ingress-gateway-tls/verify.bats index 6ee2288908..61eaaf97cc 100644 --- a/test/integration/connect/envoy/case-ingress-gateway-tls/verify.bats +++ b/test/integration/connect/envoy/case-ingress-gateway-tls/verify.bats @@ -19,7 +19,7 @@ load helpers } @test "ingress-gateway should have healthy endpoints for s1" { - assert_upstream_has_endpoints_in_status 127.0.0.1:20000 s1 HEALTHY 1 + assert_upstream_has_endpoints_in_status 127.0.0.1:20000 s1 HEALTHY 1 } @test "should be able to connect to s1 through the TLS-enabled ingress port" { @@ -30,7 +30,7 @@ load helpers --resolve s1.ingress.consul:9998:127.0.0.1 \ https://s1.ingress.consul:9998 [ "$status" -eq 0 ] - [ "$output" = "hello" ] + [[ "$output" == *"hello"* ]] } @test "should be able to connect to s1 through the TLS-enabled ingress port using the custom host" { @@ -39,5 +39,5 @@ load helpers --resolve test.example.com:9999:127.0.0.1 \ https://test.example.com:9999 [ "$status" -eq 0 ] - [ "$output" = "hello" ] -} \ No newline at end of file + [[ "$output" == *"hello"* ]] +} diff --git a/test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/primary/verify.bats b/test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/primary/verify.bats index f8376ee342..b621bdfc51 100644 --- a/test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/primary/verify.bats +++ b/test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/primary/verify.bats @@ -19,21 +19,21 @@ load helpers } @test "gateway-primary should have healthy endpoints for secondary" { - assert_upstream_has_endpoints_in_status 127.0.0.1:19002 secondary HEALTHY 1 + assert_upstream_has_endpoints_in_status 127.0.0.1:19002 secondary HEALTHY 1 } @test "gateway-secondary should have healthy endpoints for s1" { - assert_upstream_has_endpoints_in_status consul-secondary-client:19003 s1 HEALTHY 1 + assert_upstream_has_endpoints_in_status consul-secondary-client:19003 s1 HEALTHY 1 } @test "gateway-secondary should have healthy endpoints for s2" { - assert_upstream_has_endpoints_in_status consul-secondary-client:19003 s2 HEALTHY 1 + assert_upstream_has_endpoints_in_status consul-secondary-client:19003 s2 HEALTHY 1 } @test "ingress should be able to connect to s1" { run retry_default curl -s -f -d hello localhost:10000 [ "$status" -eq 0 ] - [ "$output" = "hello" ] + [[ "$output" == *"hello"* ]] } @test "ingress made 1 connection to s1" { @@ -47,7 +47,7 @@ load helpers @test "ingress should be able to connect to s2" { run retry_default curl -s -f -d hello localhost:9999 [ "$status" -eq 0 ] - [ "$output" = "hello" ] + [[ "$output" == *"hello"* ]] } @test "ingress made 1 connection to s2" { diff --git a/test/integration/connect/envoy/case-multidc-rsa-ca/primary/verify.bats b/test/integration/connect/envoy/case-multidc-rsa-ca/primary/verify.bats index 57f3f4a640..00d2afc34d 100644 --- a/test/integration/connect/envoy/case-multidc-rsa-ca/primary/verify.bats +++ b/test/integration/connect/envoy/case-multidc-rsa-ca/primary/verify.bats @@ -21,7 +21,7 @@ load helpers @test "s1 upstream should be able to connect to s2" { run retry_default curl -s -f -d hello localhost:5000 [ "$status" -eq 0 ] - [ "$output" = "hello" ] + [[ "$output" == *"hello"* ]] } @test "s1 upstream made 1 connection" { @@ -40,4 +40,4 @@ load helpers echo "KEY_TYPE: $KEY_TYPE" [ "$KEY_TYPE" == "rsa" ] -} \ No newline at end of file +} diff --git a/test/integration/connect/envoy/case-prometheus/verify.bats b/test/integration/connect/envoy/case-prometheus/verify.bats index 3d886b60bd..efaba38157 100644 --- a/test/integration/connect/envoy/case-prometheus/verify.bats +++ b/test/integration/connect/envoy/case-prometheus/verify.bats @@ -30,7 +30,7 @@ load helpers @test "s1 upstream should be able to connect to s2 with http/1.1" { run retry_default curl --http1.1 -s -f -d hello localhost:5000 [ "$status" -eq 0 ] - [ "$output" = "hello" ] + [[ "$output" == *"hello"* ]] } @test "s1 proxy should be exposing metrics to prometheus" { diff --git a/test/integration/connect/envoy/case-stats-proxy/verify.bats b/test/integration/connect/envoy/case-stats-proxy/verify.bats index 285cdd6cbc..2d02010dc1 100644 --- a/test/integration/connect/envoy/case-stats-proxy/verify.bats +++ b/test/integration/connect/envoy/case-stats-proxy/verify.bats @@ -30,7 +30,7 @@ load helpers @test "s1 upstream should be able to connect to s2 with http/1.1" { run retry_default curl --http1.1 -s -f -d hello localhost:5000 [ "$status" -eq 0 ] - [ "$output" = "hello" ] + [[ "$output" == *"hello"* ]] } @test "s1 proxy should be exposing the /stats prefix" { @@ -47,16 +47,16 @@ load helpers # Response should include the http public listener. retry_default \ - must_match_in_stats_proxy_response localhost:1239 \ + must_match_in_stats_proxy_response localhost:1239 \ 'stats' 'http.public_listener' # /stats/prometheus should also be reachable and labelling the local cluster. retry_default \ - must_match_in_stats_proxy_response localhost:1239 \ + must_match_in_stats_proxy_response localhost:1239 \ 'stats/prometheus' '[\{,]consul_source_service="s1"[,}]' # /stats/prometheus should also be reachable and exposing metrics. retry_default \ - must_match_in_stats_proxy_response localhost:1239 \ + must_match_in_stats_proxy_response localhost:1239 \ 'stats/prometheus' 'envoy_http_downstream_rq_active' } diff --git a/test/integration/connect/envoy/case-terminating-gateway-hostnames/verify.bats b/test/integration/connect/envoy/case-terminating-gateway-hostnames/verify.bats index af76cb426b..f00a783eef 100644 --- a/test/integration/connect/envoy/case-terminating-gateway-hostnames/verify.bats +++ b/test/integration/connect/envoy/case-terminating-gateway-hostnames/verify.bats @@ -15,7 +15,7 @@ load helpers } @test "terminating-gateway should have healthy endpoints for s4" { - assert_upstream_has_endpoints_in_status 127.0.0.1:20000 s4 HEALTHY 1 + assert_upstream_has_endpoints_in_status 127.0.0.1:20000 s4 HEALTHY 1 } @test "s1 upstream should have healthy endpoints for s4" { @@ -25,7 +25,7 @@ load helpers @test "s1 upstream should be able to connect to s4" { run retry_default curl -s -f -d hello localhost:5000 [ "$status" -eq 0 ] - [ "$output" = "hello" ] + [[ "$output" == *"hello"* ]] } @test "terminating-gateway is used for the upstream connection" { diff --git a/test/integration/connect/envoy/case-terminating-gateway-simple/verify.bats b/test/integration/connect/envoy/case-terminating-gateway-simple/verify.bats index 48983be21e..13ba7a9115 100644 --- a/test/integration/connect/envoy/case-terminating-gateway-simple/verify.bats +++ b/test/integration/connect/envoy/case-terminating-gateway-simple/verify.bats @@ -15,7 +15,7 @@ load helpers } @test "terminating-gateway should have healthy endpoints for s2" { - assert_upstream_has_endpoints_in_status 127.0.0.1:20000 s2 HEALTHY 1 + assert_upstream_has_endpoints_in_status 127.0.0.1:20000 s2 HEALTHY 1 } @test "s1 upstream should have healthy endpoints for s2" { @@ -25,7 +25,7 @@ load helpers @test "s1 upstream should be able to connect to s2" { run retry_default curl -s -f -d hello localhost:5000 [ "$status" -eq 0 ] - [ "$output" = "hello" ] + [[ "$output" == *"hello"* ]] } @test "terminating-gateway is used for the upstream connection" { diff --git a/test/integration/connect/envoy/helpers.bash b/test/integration/connect/envoy/helpers.bash index 760eaea1a8..d0837a3ba9 100755 --- a/test/integration/connect/envoy/helpers.bash +++ b/test/integration/connect/envoy/helpers.bash @@ -11,18 +11,14 @@ function retry { shift local errtrace=0 - if grep -q "errtrace" <<< "$SHELLOPTS" - then + if grep -q "errtrace" <<<"$SHELLOPTS"; then errtrace=1 set +E fi - for ((i=1;i<=$max;i++)) - do - if "$@" - then - if test $errtrace -eq 1 - then + for ((i = 1; i <= $max; i++)); do + if "$@"; then + if test $errtrace -eq 1; then set -E fi return 0 @@ -32,8 +28,7 @@ function retry { fi done - if test $errtrace -eq 1 - then + if test $errtrace -eq 1; then set -E fi return 1 @@ -52,22 +47,22 @@ function retry_long { } function is_set { - # Arguments: - # $1 - string value to check its truthiness - # - # Return: - # 0 - is truthy (backwards I know but allows syntax like `if is_set ` to work) - # 1 - is not truthy + # Arguments: + # $1 - string value to check its truthiness + # + # Return: + # 0 - is truthy (backwards I know but allows syntax like `if is_set ` to work) + # 1 - is not truthy - local val=$(tr '[:upper:]' '[:lower:]' <<< "$1") - case $val in - 1 | t | true | y | yes) - return 0 - ;; - *) - return 1 - ;; - esac + local val=$(tr '[:upper:]' '[:lower:]' <<<"$1") + case $val in + 1 | t | true | y | yes) + return 0 + ;; + *) + return 1 + ;; + esac } function get_cert { @@ -79,7 +74,7 @@ function get_cert { SNI_FLAG="-servername $SERVER_NAME" fi CERT=$(openssl s_client -connect $HOSTPORT $SNI_FLAG -showcerts "${OUTDIR}/config_dump.json" - docker_wget "$DC" "http://${HOSTPORT}/clusters?format=json" -q -O - > "${OUTDIR}/clusters.json" - docker_wget "$DC" "http://${HOSTPORT}/stats" -q -O - > "${OUTDIR}/stats.txt" - docker_wget "$DC" "http://${HOSTPORT}/stats/prometheus" -q -O - > "${OUTDIR}/stats_prometheus.txt" + docker_wget "$DC" "http://${HOSTPORT}/config_dump" -q -O - >"${OUTDIR}/config_dump.json" + docker_wget "$DC" "http://${HOSTPORT}/clusters?format=json" -q -O - >"${OUTDIR}/clusters.json" + docker_wget "$DC" "http://${HOSTPORT}/stats" -q -O - >"${OUTDIR}/stats.txt" + docker_wget "$DC" "http://${HOSTPORT}/stats/prometheus" -q -O - >"${OUTDIR}/stats_prometheus.txt" } function reset_envoy_metrics { @@ -365,22 +360,19 @@ function assert_envoy_metric { METRICS=$(get_envoy_metrics $HOSTPORT "$METRIC") - if [ -z "${METRICS}" ] - then + if [ -z "${METRICS}" ]; then echo "Metric not found" 1>&2 return 1 fi - GOT_COUNT=$(awk -F: '{print $2}' <<< "$METRICS" | head -n 1 | tr -d ' ') + GOT_COUNT=$(awk -F: '{print $2}' <<<"$METRICS" | head -n 1 | tr -d ' ') - if [ -z "$GOT_COUNT" ] - then + if [ -z "$GOT_COUNT" ]; then echo "Couldn't parse metric count" 1>&2 return 1 fi - if [ $EXPECT_COUNT -ne $GOT_COUNT ] - then + if [ $EXPECT_COUNT -ne $GOT_COUNT ]; then echo "$METRIC - expected count: $EXPECT_COUNT, actual count: $GOT_COUNT" 1>&2 return 1 fi @@ -394,22 +386,19 @@ function assert_envoy_metric_at_least { METRICS=$(get_envoy_metrics $HOSTPORT "$METRIC") - if [ -z "${METRICS}" ] - then + if [ -z "${METRICS}" ]; then echo "Metric not found" 1>&2 return 1 fi - GOT_COUNT=$(awk -F: '{print $2}' <<< "$METRICS" | head -n 1 | tr -d ' ') + GOT_COUNT=$(awk -F: '{print $2}' <<<"$METRICS" | head -n 1 | tr -d ' ') - if [ -z "$GOT_COUNT" ] - then + if [ -z "$GOT_COUNT" ]; then echo "Couldn't parse metric count" 1>&2 return 1 fi - if [ $EXPECT_COUNT -gt $GOT_COUNT ] - then + if [ $EXPECT_COUNT -gt $GOT_COUNT ]; then echo "$METRIC - expected >= count: $EXPECT_COUNT, actual count: $GOT_COUNT" 1>&2 return 1 fi @@ -423,22 +412,19 @@ function assert_envoy_aggregate_metric_at_least { METRICS=$(get_envoy_metrics $HOSTPORT "$METRIC") - if [ -z "${METRICS}" ] - then + if [ -z "${METRICS}" ]; then echo "Metric not found" 1>&2 return 1 fi - GOT_COUNT=$(awk '{ sum += $2 } END { print sum }' <<< "$METRICS") + GOT_COUNT=$(awk '{ sum += $2 } END { print sum }' <<<"$METRICS") - if [ -z "$GOT_COUNT" ] - then + if [ -z "$GOT_COUNT" ]; then echo "Couldn't parse metric count" 1>&2 return 1 fi - if [ $EXPECT_COUNT -gt $GOT_COUNT ] - then + if [ $EXPECT_COUNT -gt $GOT_COUNT ]; then echo "$METRIC - expected >= count: $EXPECT_COUNT, actual count: $GOT_COUNT" 1>&2 return 1 fi @@ -541,7 +527,7 @@ function docker_consul_for_proxy_bootstrap { local DC=$1 shift 1 - docker run -i --rm --network container:envoy_consul-${DC}_1 consul:local "$@" 2> /dev/null + docker run -i --rm --network container:envoy_consul-${DC}_1 consul:local "$@" 2>/dev/null } function docker_wget { @@ -557,8 +543,7 @@ function docker_curl { } function docker_exec { - if ! docker exec -i "$@" - then + if ! docker exec -i "$@"; then echo "Failed to execute: docker exec -i $@" 1>&2 return 1 fi @@ -582,7 +567,7 @@ function must_match_in_statsd_logs { run cat /workdir/${DC}/statsd/statsd.log echo "$output" - COUNT=$( echo "$output" | grep -Ec $1 ) + COUNT=$(echo "$output" | grep -Ec $1) echo "COUNT of '$1' matches: $COUNT" @@ -592,7 +577,7 @@ function must_match_in_statsd_logs { function must_match_in_prometheus_response { run curl -f -s $1/metrics - COUNT=$( echo "$output" | grep -Ec $2 ) + COUNT=$(echo "$output" | grep -Ec $2) echo "OUTPUT head -n 10" echo "$output" | head -n 10 @@ -604,7 +589,7 @@ function must_match_in_prometheus_response { function must_match_in_stats_proxy_response { run curl -f -s $1/$2 - COUNT=$( echo "$output" | grep -Ec $3 ) + COUNT=$(echo "$output" | grep -Ec $3) echo "OUTPUT head -n 10" echo "$output" | head -n 10 @@ -638,7 +623,7 @@ function must_pass_tcp_connection { echo "OUTPUT $output" [ "$status" == "0" ] - [ "$output" = "hello" ] + [[ "$output" == *"hello"* ]] } # must_fail_http_connection see must_fail_tcp_connection but this expects Envoy @@ -669,17 +654,17 @@ function must_pass_http_request { extra_args="-H x-test-debug:${DEBUG_HEADER_VALUE}" fi case "$METHOD" in - GET) - ;; - DELETE) - extra_args="$extra_args -X${METHOD}" - ;; - PUT|POST) - extra_args="$extra_args -d'{}' -X${METHOD}" - ;; - *) - return 1 - ;; + GET) ;; + + DELETE) + extra_args="$extra_args -X${METHOD}" + ;; + PUT | POST) + extra_args="$extra_args -d'{}' -X${METHOD}" + ;; + *) + return 1 + ;; esac run curl --no-keepalive -v -s -f $extra_args "$URL" @@ -696,23 +681,23 @@ function must_fail_http_request { local extra_args if [[ -n "${DEBUG_HEADER_VALUE}" ]]; then - extra_args="-H x-test-debug:${DEBUG_HEADER_VALUE}" + extra_args="-H x-test-debug:${DEBUG_HEADER_VALUE}" fi case "$METHOD" in - HEAD) - extra_args="$extra_args -I" - ;; - GET) - ;; - DELETE) - extra_args="$extra_args -X${METHOD}" - ;; - PUT|POST) - extra_args="$extra_args -d'{}' -X${METHOD}" - ;; - *) - return 1 - ;; + HEAD) + extra_args="$extra_args -I" + ;; + GET) ;; + + DELETE) + extra_args="$extra_args -X${METHOD}" + ;; + PUT | POST) + extra_args="$extra_args -d'{}' -X${METHOD}" + ;; + *) + return 1 + ;; esac # Attempt to curl through upstream @@ -731,8 +716,7 @@ function gen_envoy_bootstrap { EXTRA_ENVOY_BS_ARGS="${5-}" PROXY_ID="$SERVICE" - if ! is_set "$IS_GW" - then + if ! is_set "$IS_GW"; then PROXY_ID="$SERVICE-sidecar-proxy" fi @@ -742,7 +726,7 @@ function gen_envoy_bootstrap { -admin-bind 0.0.0.0:$ADMIN_PORT ${EXTRA_ENVOY_BS_ARGS} 2>&1); then # All OK, write config to file - echo "$output" > workdir/${DC}/envoy/$SERVICE-bootstrap.json + echo "$output" >workdir/${DC}/envoy/$SERVICE-bootstrap.json else status=$? # Command failed, instead of swallowing error (printed on stdout by docker @@ -794,7 +778,7 @@ function setup_upsert_l4_intention { local ACTION=$3 retry_default docker_curl primary -sL -XPUT "http://127.0.0.1:8500/v1/connect/intentions/exact?source=${SOURCE}&destination=${DESTINATION}" \ - -d"{\"Action\": \"${ACTION}\"}" >/dev/null + -d"{\"Action\": \"${ACTION}\"}" >/dev/null } function upsert_l4_intention { @@ -803,7 +787,7 @@ function upsert_l4_intention { local ACTION=$3 retry_default curl -sL -XPUT "http://127.0.0.1:8500/v1/connect/intentions/exact?source=${SOURCE}&destination=${DESTINATION}" \ - -d"{\"Action\": \"${ACTION}\"}" >/dev/null + -d"{\"Action\": \"${ACTION}\"}" >/dev/null } function get_ca_root { @@ -822,15 +806,16 @@ function set_ttl_check_state { local DC=${3:-primary} case "$CHECK_STATE" in - pass) - ;; - warn) - ;; - fail) - ;; - *) - echo "invalid ttl check state '${CHECK_STATE}'" >&2 - return 1 + pass) ;; + + warn) ;; + + fail) ;; + + *) + echo "invalid ttl check state '${CHECK_STATE}'" >&2 + return 1 + ;; esac retry_default docker_curl "${DC}" -sL -XPUT "http://localhost:8500/v1/agent/check/warn/${CHECK_ID}" @@ -843,7 +828,7 @@ function get_upstream_fortio_name { local DEBUG_HEADER_VALUE="${4:-""}" local extra_args if [[ -n "${DEBUG_HEADER_VALUE}" ]]; then - extra_args="-H x-test-debug:${DEBUG_HEADER_VALUE}" + extra_args="-H x-test-debug:${DEBUG_HEADER_VALUE}" fi # split proto if https:// is at the front of the host since the --resolve # string needs just a bare host. @@ -857,7 +842,7 @@ function get_upstream_fortio_name { # We use --resolve instead of setting a Host header since we need the right # name to be sent for SNI in some cases too. run retry_default curl -v -s -f --resolve "${HOST}:${PORT}:127.0.0.1" $extra_args \ - "${PROTO}${HOST}:${PORT}${PREFIX}/debug?env=dump" + "${PROTO}${HOST}:${PORT}${PREFIX}/debug?env=dump" # Useful Debugging but breaks the expectation that the value output is just # the grep output when things don't fail @@ -893,7 +878,7 @@ function assert_expected_fortio_name_pattern { GOT=$(get_upstream_fortio_name ${HOST} ${PORT} "${URL_PREFIX}" "${DEBUG_HEADER_VALUE}") if [[ "$GOT" =~ $EXPECT_NAME_PATTERN ]]; then - : + : else echo "expected name pattern: $EXPECT_NAME_PATTERN, actual name: $GOT" 1>&2 return 1 @@ -907,10 +892,10 @@ function get_upstream_fortio_host_header { local DEBUG_HEADER_VALUE="${4:-""}" local extra_args if [[ -n "${DEBUG_HEADER_VALUE}" ]]; then - extra_args="-H x-test-debug:${DEBUG_HEADER_VALUE}" + extra_args="-H x-test-debug:${DEBUG_HEADER_VALUE}" fi run retry_default curl -v -s -f -H"Host: ${HOST}" $extra_args \ - "localhost:${PORT}${PREFIX}/debug" + "localhost:${PORT}${PREFIX}/debug" [ "$status" == 0 ] echo "$output" | grep -E "^Host: " } @@ -959,13 +944,13 @@ function register_lambdas { local DC=${1:-primary} # register lambdas to the catalog for f in $(find workdir/${DC}/register -type f -name 'lambda_*.json'); do - retry_default curl -sL -XPUT -d @${f} "http://localhost:8500/v1/catalog/register" >/dev/null && \ + retry_default curl -sL -XPUT -d @${f} "http://localhost:8500/v1/catalog/register" >/dev/null && echo "Registered Lambda: $(jq -r .Service.Service $f)" done # write service-defaults config entries for lambdas for f in $(find workdir/${DC}/register -type f -name 'service_defaults_*.json'); do varsub ${f} AWS_LAMBDA_REGION AWS_LAMBDA_ARN - retry_default curl -sL -XPUT -d @${f} "http://localhost:8500/v1/config" >/dev/null && \ + retry_default curl -sL -XPUT -d @${f} "http://localhost:8500/v1/config" >/dev/null && echo "Wrote config: $(jq -r '.Kind + " / " + .Name' $f)" done } @@ -992,7 +977,8 @@ function assert_lambda_envoy_dynamic_http_filter_exists { } function varsub { - local file=$1 ; shift + local file=$1 + shift for v in "$@"; do sed -i "s/\${$v}/${!v}/g" $file done @@ -1016,4 +1002,4 @@ function assert_url_header { run get_url_header "$URL" "$HEADER" [ "$status" == 0 ] [ "$VALUE" = "$output" ] -} \ No newline at end of file +}