integ test: reduce flakiness due to compound output from retry (#15233)

* integ test: avoid flakiness due to compound output from retry

* changelog

.changelog/15233.txt

@@ -0,0 +1,3 @@
+```release-note: improvement
+integ test: fix flakiness due to test condition from retry app endoint
+```

test/integration/connect/envoy/case-basic/verify.bats

@@ -33,13 +33,13 @@ load helpers
 @test "s1 upstream should be able to connect to s2" {
   run retry_default curl -s -f -d hello localhost:5000
   [ "$status" -eq 0 ]
-  [ "$output" = "hello" ]
+  [[ "$output" == *"hello"* ]]
 }
 
 @test "s1 proxy should have been configured with one rbac listener filter at L4" {
   LISTEN_FILTERS=$(get_envoy_listener_filters localhost:19000)
-  PUB=$(echo "$LISTEN_FILTERS" | grep -E "^public_listener:" | cut -f 2 -d ' ' )
-  UPS=$(echo "$LISTEN_FILTERS" | grep -E "^(default\/default\/)?s2:" | cut -f 2 -d ' ' )
+  PUB=$(echo "$LISTEN_FILTERS" | grep -E "^public_listener:" | cut -f 2 -d ' ')
+  UPS=$(echo "$LISTEN_FILTERS" | grep -E "^(default\/default\/)?s2:" | cut -f 2 -d ' ')
 
   echo "LISTEN_FILTERS = $LISTEN_FILTERS"
   echo "PUB = $PUB"

test/integration/connect/envoy/case-centralconf/verify.bats

@@ -29,7 +29,7 @@ load helpers
 @test "s1 upstream should be able to connect to s2 with http/1.1" {
   run retry_default curl --http1.1 -s -f -d hello localhost:5000
   [ "$status" -eq 0 ]
-  [ "$output" = "hello" ]
+  [[ "$output" == *"hello"* ]]
 }
 
 @test "s1 proxy should be exposing metrics to prometheus from central config" {

test/integration/connect/envoy/case-cfg-resolver-cluster-peering-failover/primary/verify.bats

@@ -49,11 +49,10 @@ load helpers
   assert_upstream_has_endpoints_in_status 127.0.0.1:19000 failover-target~s2.default.primary-to-alpha.external HEALTHY 1
 }
 
-
 @test "s1 upstream should be able to connect to s2" {
   run retry_default curl -s -f -d hello localhost:5000
   [ "$status" -eq 0 ]
-  [ "$output" = "hello" ]
+  [[ "$output" == *"hello"* ]]
 }
 
 @test "s1 upstream made 1 connection" {
@@ -68,7 +67,6 @@ load helpers
   assert_service_has_healthy_instances s2 0 primary
 }
 
-
 @test "s1 upstream should have healthy endpoints for s2 in the failover cluster peer" {
   assert_upstream_has_endpoints_in_status 127.0.0.1:19000 failover-target~s2.default.primary.internal UNHEALTHY 1
   assert_upstream_has_endpoints_in_status 127.0.0.1:19000 failover-target~s2.default.primary-to-alpha.external HEALTHY 1
@@ -85,7 +83,7 @@ load helpers
 @test "s1 upstream should be able to connect to s2 in the failover cluster peer" {
   run retry_default curl -s -f -d hello localhost:5000
   [ "$status" -eq 0 ]
-  [ "$output" = "hello" ]
+  [[ "$output" == *"hello"* ]]
 }
 
 @test "s1 upstream made 1 connection to s2 through the cluster peer" {
@@ -105,7 +103,7 @@ load helpers
 @test "s1 upstream should be able to connect to s2 via virtual-s2" {
   run retry_default curl -s -f -d hello localhost:5001
   [ "$status" -eq 0 ]
-  [ "$output" = "hello" ]
+  [[ "$output" == *"hello"* ]]
 }
 
 @test "s1 upstream made 1 connection to s2 via virtual-s2 through the cluster peer" {

test/integration/connect/envoy/case-cfg-resolver-svc-redirect-http/verify.bats

@@ -37,10 +37,9 @@ load helpers
 @test "s1 upstream should be able to connect to its upstream simply" {
   run retry_default curl -s -f -d hello localhost:5000
   [ "$status" -eq 0 ]
-  [ "$output" = "hello" ]
+  [[ "$output" == *"hello"* ]]
 }
 
 @test "s1 upstream should be able to connect to s3 via upstream s2" {
   assert_expected_fortio_name s3
 }
-

test/integration/connect/envoy/case-cfg-resolver-svc-redirect-tcp/verify.bats

@@ -37,10 +37,9 @@ load helpers
 @test "s1 upstream should be able to connect to its upstream simply" {
   run retry_default curl -s -f -d hello localhost:5000
   [ "$status" -eq 0 ]
-  [ "$output" = "hello" ]
+  [[ "$output" == *"hello"* ]]
 }
 
 @test "s1 upstream should be able to connect to s3 via upstream s2" {
   assert_expected_fortio_name s3
 }
-

test/integration/connect/envoy/case-cross-peer-control-plane-mgw/primary/verify.bats

@@ -57,7 +57,7 @@ load helpers
 @test "s1 upstream should be able to connect to s2" {
   run retry_default curl -s -f -d hello localhost:5000
   [ "$status" -eq 0 ]
-  [ "$output" = "hello" ]
+  [[ "$output" == *"hello"* ]]
 }
 
 @test "s1 upstream made 1 connection to s2" {

test/integration/connect/envoy/case-cross-peers-http-router/primary/verify.bats

@@ -57,7 +57,7 @@ load helpers
 @test "s1 upstream should be able to connect to s2 with http/1.1" {
   run retry_default curl --http1.1 -s -f -d hello localhost:5000
   [ "$status" -eq 0 ]
-  [ "$output" = "hello" ]
+  [[ "$output" == *"hello"* ]]
 }
 
 @test "s1 upstream should be able to connect to s2 via s2" {

test/integration/connect/envoy/case-cross-peers-http/primary/verify.bats

@@ -49,7 +49,7 @@ load helpers
 @test "s1 upstream should be able to connect to s2 with http/1.1" {
   run retry_default curl --http1.1 -s -f -d hello localhost:5000
   [ "$status" -eq 0 ]
-  [ "$output" = "hello" ]
+  [[ "$output" == *"hello"* ]]
 }
 
 @test "s1 upstream made 1 connection to s2" {

test/integration/connect/envoy/case-cross-peers-resolver-redirect-tcp/primary/verify.bats

@@ -49,7 +49,7 @@ load helpers
 @test "s1 upstream should be able to connect to s2" {
   run retry_default curl -s -f -d hello localhost:5000
   [ "$status" -eq 0 ]
-  [ "$output" = "hello" ]
+  [[ "$output" == *"hello"* ]]
 }
 
 @test "s1 upstream should be able to connect to s3 via s2 due to redirect" {

test/integration/connect/envoy/case-cross-peers/primary/verify.bats

@@ -49,7 +49,7 @@ load helpers
 @test "s1 upstream should be able to connect to s2" {
   run retry_default curl -s -f -d hello localhost:5000
   [ "$status" -eq 0 ]
-  [ "$output" = "hello" ]
+  [[ "$output" == *"hello"* ]]
 }
 
 @test "s1 upstream made 1 connection to s2" {

test/integration/connect/envoy/case-expose-checks/verify.bats

@@ -33,7 +33,7 @@ load helpers
 @test "s1 upstream should be able to connect to s2" {
   run retry_default curl -s -f -d hello localhost:5000
   [ "$status" -eq 0 ]
-  [ "$output" = "hello" ]
+  [[ "$output" == *"hello"* ]]
 }
 
 @test "s2 exposes checks on a new listener" {

test/integration/connect/envoy/case-gateways-local/primary/verify.bats

@@ -24,17 +24,17 @@ load helpers
 }
 
 @test "gateway-primary should have healthy endpoints for secondary" {
-   assert_upstream_has_endpoints_in_status 127.0.0.1:19002 secondary HEALTHY 1
+  assert_upstream_has_endpoints_in_status 127.0.0.1:19002 secondary HEALTHY 1
 }
 
 @test "gateway-secondary should have healthy endpoints for s2" {
-   assert_upstream_has_endpoints_in_status consul-secondary-client:19003 s2 HEALTHY 1
+  assert_upstream_has_endpoints_in_status consul-secondary-client:19003 s2 HEALTHY 1
 }
 
 @test "s1 upstream should be able to connect to s2" {
   run retry_default curl -s -f -d hello localhost:5000
   [ "$status" -eq 0 ]
-  [ "$output" = "hello" ]
+  [[ "$output" == *"hello"* ]]
 }
 
 @test "s1 upstream made 1 connection" {

test/integration/connect/envoy/case-gateways-remote/primary/verify.bats

@@ -22,7 +22,7 @@ load helpers
 @test "s1 upstream should be able to connect to s2" {
   run retry_default curl -s -f -d hello localhost:5000
   [ "$status" -eq 0 ]
-  [ "$output" = "hello" ]
+  [[ "$output" == *"hello"* ]]
 }
 
 @test "s1 upstream made 1 connection" {

test/integration/connect/envoy/case-http/verify.bats

@@ -30,13 +30,13 @@ load helpers
 @test "s1 upstream should be able to connect to s2 with http/1.1" {
   run retry_default curl --http1.1 -s -f -d hello localhost:5000
   [ "$status" -eq 0 ]
-  [ "$output" = "hello" ]
+  [[ "$output" == *"hello"* ]]
 }
 
 @test "s1 proxy should have been configured with http connection managers" {
   LISTEN_FILTERS=$(get_envoy_listener_filters localhost:19000)
-  PUB=$(echo "$LISTEN_FILTERS" | grep -E "^public_listener:" | cut -f 2 -d ' ' )
-  UPS=$(echo "$LISTEN_FILTERS" | grep -E "^(default\/default\/)?s2:" | cut -f 2 -d ' ' )
+  PUB=$(echo "$LISTEN_FILTERS" | grep -E "^public_listener:" | cut -f 2 -d ' ')
+  UPS=$(echo "$LISTEN_FILTERS" | grep -E "^(default\/default\/)?s2:" | cut -f 2 -d ' ')
 
   echo "LISTEN_FILTERS = $LISTEN_FILTERS"
   echo "PUB = $PUB"
@@ -48,7 +48,7 @@ load helpers
 
 @test "s2 proxy should have been configured with an http connection manager" {
   LISTEN_FILTERS=$(get_envoy_listener_filters localhost:19001)
-  PUB=$(echo "$LISTEN_FILTERS" | grep -E "^public_listener:" | cut -f 2 -d ' ' )
+  PUB=$(echo "$LISTEN_FILTERS" | grep -E "^public_listener:" | cut -f 2 -d ' ')
 
   echo "LISTEN_FILTERS = $LISTEN_FILTERS"
   echo "PUB = $PUB"
@@ -58,8 +58,8 @@ load helpers
 
 @test "s1 proxy should have been configured with http rbac filters" {
   HTTP_FILTERS=$(get_envoy_http_filters localhost:19000)
-  PUB=$(echo "$HTTP_FILTERS" | grep -E "^public_listener:" | cut -f 2 -d ' ' )
-  UPS=$(echo "$HTTP_FILTERS" | grep -E "^(default\/default\/)?s2:" | cut -f 2 -d ' ' )
+  PUB=$(echo "$HTTP_FILTERS" | grep -E "^public_listener:" | cut -f 2 -d ' ')
+  UPS=$(echo "$HTTP_FILTERS" | grep -E "^(default\/default\/)?s2:" | cut -f 2 -d ' ')
 
   echo "HTTP_FILTERS = $HTTP_FILTERS"
   echo "PUB = $PUB"
@@ -71,7 +71,7 @@ load helpers
 
 @test "s2 proxy should have been configured with http rbac filters" {
   HTTP_FILTERS=$(get_envoy_http_filters localhost:19001)
-  PUB=$(echo "$HTTP_FILTERS" | grep -E "^public_listener:" | cut -f 2 -d ' ' )
+  PUB=$(echo "$HTTP_FILTERS" | grep -E "^public_listener:" | cut -f 2 -d ' ')
 
   echo "HTTP_FILTERS = $HTTP_FILTERS"
   echo "PUB = $PUB"

test/integration/connect/envoy/case-ingress-gateway-simple/verify.bats

@@ -42,5 +42,5 @@ load helpers
 @test "ingress should be able to connect to s1 via configured port" {
   run retry_default curl -s -f -d hello localhost:9999
   [ "$status" -eq 0 ]
-  [ "$output" = "hello" ]
+  [[ "$output" == *"hello"* ]]
 }

test/integration/connect/envoy/case-ingress-gateway-tls/verify.bats

@@ -19,7 +19,7 @@ load helpers
 }
 
 @test "ingress-gateway should have healthy endpoints for s1" {
-   assert_upstream_has_endpoints_in_status 127.0.0.1:20000 s1 HEALTHY 1
+  assert_upstream_has_endpoints_in_status 127.0.0.1:20000 s1 HEALTHY 1
 }
 
 @test "should be able to connect to s1 through the TLS-enabled ingress port" {
@@ -30,7 +30,7 @@ load helpers
     --resolve s1.ingress.consul:9998:127.0.0.1 \
     https://s1.ingress.consul:9998
   [ "$status" -eq 0 ]
-  [ "$output" = "hello" ]
+  [[ "$output" == *"hello"* ]]
 }
 
 @test "should be able to connect to s1 through the TLS-enabled ingress port using the custom host" {
@@ -39,5 +39,5 @@ load helpers
     --resolve test.example.com:9999:127.0.0.1 \
     https://test.example.com:9999
   [ "$status" -eq 0 ]
-  [ "$output" = "hello" ]
+  [[ "$output" == *"hello"* ]]
 }

test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/primary/verify.bats

@@ -19,21 +19,21 @@ load helpers
 }
 
 @test "gateway-primary should have healthy endpoints for secondary" {
-   assert_upstream_has_endpoints_in_status 127.0.0.1:19002 secondary HEALTHY 1
+  assert_upstream_has_endpoints_in_status 127.0.0.1:19002 secondary HEALTHY 1
 }
 
 @test "gateway-secondary should have healthy endpoints for s1" {
-   assert_upstream_has_endpoints_in_status consul-secondary-client:19003 s1 HEALTHY 1
+  assert_upstream_has_endpoints_in_status consul-secondary-client:19003 s1 HEALTHY 1
 }
 
 @test "gateway-secondary should have healthy endpoints for s2" {
-   assert_upstream_has_endpoints_in_status consul-secondary-client:19003 s2 HEALTHY 1
+  assert_upstream_has_endpoints_in_status consul-secondary-client:19003 s2 HEALTHY 1
 }
 
 @test "ingress should be able to connect to s1" {
   run retry_default curl -s -f -d hello localhost:10000
   [ "$status" -eq 0 ]
-  [ "$output" = "hello" ]
+  [[ "$output" == *"hello"* ]]
 }
 
 @test "ingress made 1 connection to s1" {
@@ -47,7 +47,7 @@ load helpers
 @test "ingress should be able to connect to s2" {
   run retry_default curl -s -f -d hello localhost:9999
   [ "$status" -eq 0 ]
-  [ "$output" = "hello" ]
+  [[ "$output" == *"hello"* ]]
 }
 
 @test "ingress made 1 connection to s2" {

test/integration/connect/envoy/case-multidc-rsa-ca/primary/verify.bats

@@ -21,7 +21,7 @@ load helpers
 @test "s1 upstream should be able to connect to s2" {
   run retry_default curl -s -f -d hello localhost:5000
   [ "$status" -eq 0 ]
-  [ "$output" = "hello" ]
+  [[ "$output" == *"hello"* ]]
 }
 
 @test "s1 upstream made 1 connection" {

test/integration/connect/envoy/case-prometheus/verify.bats

@@ -30,7 +30,7 @@ load helpers
 @test "s1 upstream should be able to connect to s2 with http/1.1" {
   run retry_default curl --http1.1 -s -f -d hello localhost:5000
   [ "$status" -eq 0 ]
-  [ "$output" = "hello" ]
+  [[ "$output" == *"hello"* ]]
 }
 
 @test "s1 proxy should be exposing metrics to prometheus" {

test/integration/connect/envoy/case-stats-proxy/verify.bats

@@ -30,7 +30,7 @@ load helpers
 @test "s1 upstream should be able to connect to s2 with http/1.1" {
   run retry_default curl --http1.1 -s -f -d hello localhost:5000
   [ "$status" -eq 0 ]
-  [ "$output" = "hello" ]
+  [[ "$output" == *"hello"* ]]
 }
 
 @test "s1 proxy should be exposing the /stats prefix" {
@@ -47,16 +47,16 @@ load helpers
 
   # Response should include the http public listener.
   retry_default \
-     must_match_in_stats_proxy_response localhost:1239 \
+    must_match_in_stats_proxy_response localhost:1239 \
     'stats' 'http.public_listener'
 
   # /stats/prometheus should also be reachable and labelling the local cluster.
   retry_default \
-     must_match_in_stats_proxy_response localhost:1239 \
+    must_match_in_stats_proxy_response localhost:1239 \
     'stats/prometheus' '[\{,]consul_source_service="s1"[,}]'
 
   # /stats/prometheus should also be reachable and exposing metrics.
   retry_default \
-     must_match_in_stats_proxy_response localhost:1239 \
+    must_match_in_stats_proxy_response localhost:1239 \
     'stats/prometheus' 'envoy_http_downstream_rq_active'
 }

test/integration/connect/envoy/case-terminating-gateway-hostnames/verify.bats

@@ -15,7 +15,7 @@ load helpers
 }
 
 @test "terminating-gateway should have healthy endpoints for s4" {
-   assert_upstream_has_endpoints_in_status 127.0.0.1:20000 s4 HEALTHY 1
+  assert_upstream_has_endpoints_in_status 127.0.0.1:20000 s4 HEALTHY 1
 }
 
 @test "s1 upstream should have healthy endpoints for s4" {
@@ -25,7 +25,7 @@ load helpers
 @test "s1 upstream should be able to connect to s4" {
   run retry_default curl -s -f -d hello localhost:5000
   [ "$status" -eq 0 ]
-  [ "$output" = "hello" ]
+  [[ "$output" == *"hello"* ]]
 }
 
 @test "terminating-gateway is used for the upstream connection" {

test/integration/connect/envoy/case-terminating-gateway-simple/verify.bats

@@ -15,7 +15,7 @@ load helpers
 }
 
 @test "terminating-gateway should have healthy endpoints for s2" {
-   assert_upstream_has_endpoints_in_status 127.0.0.1:20000 s2 HEALTHY 1
+  assert_upstream_has_endpoints_in_status 127.0.0.1:20000 s2 HEALTHY 1
 }
 
 @test "s1 upstream should have healthy endpoints for s2" {
@@ -25,7 +25,7 @@ load helpers
 @test "s1 upstream should be able to connect to s2" {
   run retry_default curl -s -f -d hello localhost:5000
   [ "$status" -eq 0 ]
-  [ "$output" = "hello" ]
+  [[ "$output" == *"hello"* ]]
 }
 
 @test "terminating-gateway is used for the upstream connection" {

test/integration/connect/envoy/helpers.bash

@@ -11,18 +11,14 @@ function retry {
   shift
 
   local errtrace=0
-  if grep -q "errtrace" <<< "$SHELLOPTS"
-  then
+  if grep -q "errtrace" <<<"$SHELLOPTS"; then
     errtrace=1
     set +E
   fi
 
-  for ((i=1;i<=$max;i++))
-  do
-    if "$@"
-    then
-      if test $errtrace -eq 1
-      then
+  for ((i = 1; i <= $max; i++)); do
+    if "$@"; then
+      if test $errtrace -eq 1; then
         set -E
       fi
       return 0
@@ -32,8 +28,7 @@ function retry {
     fi
   done
 
-  if test $errtrace -eq 1
-  then
+  if test $errtrace -eq 1; then
     set -E
   fi
   return 1
@@ -52,22 +47,22 @@ function retry_long {
 }
 
 function is_set {
-   # Arguments:
-   #   $1 - string value to check its truthiness
-   #
-   # Return:
-   #   0 - is truthy (backwards I know but allows syntax like `if is_set <var>` to work)
-   #   1 - is not truthy
+  # Arguments:
+  #   $1 - string value to check its truthiness
+  #
+  # Return:
+  #   0 - is truthy (backwards I know but allows syntax like `if is_set <var>` to work)
+  #   1 - is not truthy
 
-   local val=$(tr '[:upper:]' '[:lower:]' <<< "$1")
-   case $val in
-      1 | t | true | y | yes)
-         return 0
-         ;;
-      *)
-         return 1
-         ;;
-   esac
+  local val=$(tr '[:upper:]' '[:lower:]' <<<"$1")
+  case $val in
+  1 | t | true | y | yes)
+    return 0
+    ;;
+  *)
+    return 1
+    ;;
+  esac
 }
 
 function get_cert {
@@ -79,7 +74,7 @@ function get_cert {
     SNI_FLAG="-servername $SERVER_NAME"
   fi
   CERT=$(openssl s_client -connect $HOSTPORT $SNI_FLAG -showcerts </dev/null)
-  openssl x509 -noout -text <<< "$CERT"
+  openssl x509 -noout -text <<<"$CERT"
 }
 
 function assert_proxy_presents_cert_uri {
@@ -140,7 +135,7 @@ function assert_envoy_version {
   # Envoy 1.8.0 returns a plain text line like
   # envoy 5d25f466c3410c0dfa735d7d4358beb76b2da507/1.8.0/Clean/DEBUG live 3 3 0
   # Later versions return JSON.
-  if (echo $output | grep '^envoy') ; then
+  if (echo $output | grep '^envoy'); then
     VERSION=$(echo $output | cut -d ' ' -f 2)
   else
     VERSION=$(echo $output | jq -r '.version')
@@ -153,11 +148,11 @@ function assert_envoy_version {
 
   # 1.20.2, 1.19.3 and 1.18.6 are special snowflakes in that the version for
   # the release is reported with a '-dev' suffix (eg 1.20.2-dev).
-  if [ "$ENVOY_VERSION" = "1.20.2" ] ; then
+  if [ "$ENVOY_VERSION" = "1.20.2" ]; then
     ENVOY_VERSION="1.20.2-dev"
-  elif [ "$ENVOY_VERSION" = "1.19.3" ] ; then
+  elif [ "$ENVOY_VERSION" = "1.19.3" ]; then
     ENVOY_VERSION="1.19.3-dev"
-  elif [ "$ENVOY_VERSION" = "1.18.6" ] ; then
+  elif [ "$ENVOY_VERSION" = "1.18.6" ]; then
     ENVOY_VERSION="1.18.6-dev"
   fi
 
@@ -298,10 +293,10 @@ function snapshot_envoy_admin {
   local OUTDIR="${LOG_DIR}/envoy-snapshots/${DC}/${ENVOY_NAME}"
 
   mkdir -p "${OUTDIR}"
-  docker_wget "$DC" "http://${HOSTPORT}/config_dump" -q -O - > "${OUTDIR}/config_dump.json"
-  docker_wget "$DC" "http://${HOSTPORT}/clusters?format=json" -q -O - > "${OUTDIR}/clusters.json"
-  docker_wget "$DC" "http://${HOSTPORT}/stats" -q -O - > "${OUTDIR}/stats.txt"
-  docker_wget "$DC" "http://${HOSTPORT}/stats/prometheus" -q -O - > "${OUTDIR}/stats_prometheus.txt"
+  docker_wget "$DC" "http://${HOSTPORT}/config_dump" -q -O - >"${OUTDIR}/config_dump.json"
+  docker_wget "$DC" "http://${HOSTPORT}/clusters?format=json" -q -O - >"${OUTDIR}/clusters.json"
+  docker_wget "$DC" "http://${HOSTPORT}/stats" -q -O - >"${OUTDIR}/stats.txt"
+  docker_wget "$DC" "http://${HOSTPORT}/stats/prometheus" -q -O - >"${OUTDIR}/stats_prometheus.txt"
 }
 
 function reset_envoy_metrics {
@@ -365,22 +360,19 @@ function assert_envoy_metric {
 
   METRICS=$(get_envoy_metrics $HOSTPORT "$METRIC")
 
-  if [ -z "${METRICS}" ]
-  then
+  if [ -z "${METRICS}" ]; then
     echo "Metric not found" 1>&2
     return 1
   fi
 
-  GOT_COUNT=$(awk -F: '{print $2}' <<< "$METRICS" | head -n 1 | tr -d ' ')
+  GOT_COUNT=$(awk -F: '{print $2}' <<<"$METRICS" | head -n 1 | tr -d ' ')
 
-  if [ -z "$GOT_COUNT" ]
-  then
+  if [ -z "$GOT_COUNT" ]; then
     echo "Couldn't parse metric count" 1>&2
     return 1
   fi
 
-  if [ $EXPECT_COUNT -ne $GOT_COUNT ]
-  then
+  if [ $EXPECT_COUNT -ne $GOT_COUNT ]; then
     echo "$METRIC - expected count: $EXPECT_COUNT, actual count: $GOT_COUNT" 1>&2
     return 1
   fi
@@ -394,22 +386,19 @@ function assert_envoy_metric_at_least {
 
   METRICS=$(get_envoy_metrics $HOSTPORT "$METRIC")
 
-  if [ -z "${METRICS}" ]
-  then
+  if [ -z "${METRICS}" ]; then
     echo "Metric not found" 1>&2
     return 1
   fi
 
-  GOT_COUNT=$(awk -F: '{print $2}' <<< "$METRICS" | head -n 1 | tr -d ' ')
+  GOT_COUNT=$(awk -F: '{print $2}' <<<"$METRICS" | head -n 1 | tr -d ' ')
 
-  if [ -z "$GOT_COUNT" ]
-  then
+  if [ -z "$GOT_COUNT" ]; then
     echo "Couldn't parse metric count" 1>&2
     return 1
   fi
 
-  if [ $EXPECT_COUNT -gt $GOT_COUNT ]
-  then
+  if [ $EXPECT_COUNT -gt $GOT_COUNT ]; then
     echo "$METRIC - expected >= count: $EXPECT_COUNT, actual count: $GOT_COUNT" 1>&2
     return 1
   fi
@@ -423,22 +412,19 @@ function assert_envoy_aggregate_metric_at_least {
 
   METRICS=$(get_envoy_metrics $HOSTPORT "$METRIC")
 
-  if [ -z "${METRICS}" ]
-  then
+  if [ -z "${METRICS}" ]; then
     echo "Metric not found" 1>&2
     return 1
   fi
 
-  GOT_COUNT=$(awk '{ sum += $2 } END { print sum }' <<< "$METRICS")
+  GOT_COUNT=$(awk '{ sum += $2 } END { print sum }' <<<"$METRICS")
 
-  if [ -z "$GOT_COUNT" ]
-  then
+  if [ -z "$GOT_COUNT" ]; then
     echo "Couldn't parse metric count" 1>&2
     return 1
   fi
 
-  if [ $EXPECT_COUNT -gt $GOT_COUNT ]
-  then
+  if [ $EXPECT_COUNT -gt $GOT_COUNT ]; then
     echo "$METRIC - expected >= count: $EXPECT_COUNT, actual count: $GOT_COUNT" 1>&2
     return 1
   fi
@@ -541,7 +527,7 @@ function docker_consul_for_proxy_bootstrap {
   local DC=$1
   shift 1
 
-  docker run -i --rm --network container:envoy_consul-${DC}_1 consul:local "$@" 2> /dev/null
+  docker run -i --rm --network container:envoy_consul-${DC}_1 consul:local "$@" 2>/dev/null
 }
 
 function docker_wget {
@@ -557,8 +543,7 @@ function docker_curl {
 }
 
 function docker_exec {
-  if ! docker exec -i "$@"
-  then
+  if ! docker exec -i "$@"; then
     echo "Failed to execute: docker exec -i $@" 1>&2
     return 1
   fi
@@ -582,7 +567,7 @@ function must_match_in_statsd_logs {
 
   run cat /workdir/${DC}/statsd/statsd.log
   echo "$output"
-  COUNT=$( echo "$output" | grep -Ec $1 )
+  COUNT=$(echo "$output" | grep -Ec $1)
 
   echo "COUNT of '$1' matches: $COUNT"
 
@@ -592,7 +577,7 @@ function must_match_in_statsd_logs {
 
 function must_match_in_prometheus_response {
   run curl -f -s $1/metrics
-  COUNT=$( echo "$output" | grep -Ec $2 )
+  COUNT=$(echo "$output" | grep -Ec $2)
 
   echo "OUTPUT head -n 10"
   echo "$output" | head -n 10
@@ -604,7 +589,7 @@ function must_match_in_prometheus_response {
 
 function must_match_in_stats_proxy_response {
   run curl -f -s $1/$2
-  COUNT=$( echo "$output" | grep -Ec $3 )
+  COUNT=$(echo "$output" | grep -Ec $3)
 
   echo "OUTPUT head -n 10"
   echo "$output" | head -n 10
@@ -638,7 +623,7 @@ function must_pass_tcp_connection {
   echo "OUTPUT $output"
 
   [ "$status" == "0" ]
-  [ "$output" = "hello" ]
+  [[ "$output" == *"hello"* ]]
 }
 
 # must_fail_http_connection see must_fail_tcp_connection but this expects Envoy
@@ -669,17 +654,17 @@ function must_pass_http_request {
     extra_args="-H x-test-debug:${DEBUG_HEADER_VALUE}"
   fi
   case "$METHOD" in
-    GET)
-      ;;
-    DELETE)
-      extra_args="$extra_args -X${METHOD}"
-      ;;
-    PUT|POST)
-      extra_args="$extra_args -d'{}' -X${METHOD}"
-      ;;
-    *)
-      return 1
-      ;;
+  GET) ;;
+
+  DELETE)
+    extra_args="$extra_args -X${METHOD}"
+    ;;
+  PUT | POST)
+    extra_args="$extra_args -d'{}' -X${METHOD}"
+    ;;
+  *)
+    return 1
+    ;;
   esac
 
   run curl --no-keepalive -v -s -f $extra_args "$URL"
@@ -696,23 +681,23 @@ function must_fail_http_request {
 
   local extra_args
   if [[ -n "${DEBUG_HEADER_VALUE}" ]]; then
-      extra_args="-H x-test-debug:${DEBUG_HEADER_VALUE}"
+    extra_args="-H x-test-debug:${DEBUG_HEADER_VALUE}"
   fi
   case "$METHOD" in
-    HEAD)
-      extra_args="$extra_args -I"
-      ;;
-    GET)
-      ;;
-    DELETE)
-      extra_args="$extra_args -X${METHOD}"
-      ;;
-    PUT|POST)
-      extra_args="$extra_args -d'{}' -X${METHOD}"
-      ;;
-    *)
-      return 1
-      ;;
+  HEAD)
+    extra_args="$extra_args -I"
+    ;;
+  GET) ;;
+
+  DELETE)
+    extra_args="$extra_args -X${METHOD}"
+    ;;
+  PUT | POST)
+    extra_args="$extra_args -d'{}' -X${METHOD}"
+    ;;
+  *)
+    return 1
+    ;;
   esac
 
   # Attempt to curl through upstream
@@ -731,8 +716,7 @@ function gen_envoy_bootstrap {
   EXTRA_ENVOY_BS_ARGS="${5-}"
 
   PROXY_ID="$SERVICE"
-  if ! is_set "$IS_GW"
-  then
+  if ! is_set "$IS_GW"; then
     PROXY_ID="$SERVICE-sidecar-proxy"
   fi
 
@@ -742,7 +726,7 @@ function gen_envoy_bootstrap {
     -admin-bind 0.0.0.0:$ADMIN_PORT ${EXTRA_ENVOY_BS_ARGS} 2>&1); then
 
     # All OK, write config to file
-    echo "$output" > workdir/${DC}/envoy/$SERVICE-bootstrap.json
+    echo "$output" >workdir/${DC}/envoy/$SERVICE-bootstrap.json
   else
     status=$?
     # Command failed, instead of swallowing error (printed on stdout by docker
@@ -794,7 +778,7 @@ function setup_upsert_l4_intention {
   local ACTION=$3
 
   retry_default docker_curl primary -sL -XPUT "http://127.0.0.1:8500/v1/connect/intentions/exact?source=${SOURCE}&destination=${DESTINATION}" \
-      -d"{\"Action\": \"${ACTION}\"}" >/dev/null
+    -d"{\"Action\": \"${ACTION}\"}" >/dev/null
 }
 
 function upsert_l4_intention {
@@ -803,7 +787,7 @@ function upsert_l4_intention {
   local ACTION=$3
 
   retry_default curl -sL -XPUT "http://127.0.0.1:8500/v1/connect/intentions/exact?source=${SOURCE}&destination=${DESTINATION}" \
-      -d"{\"Action\": \"${ACTION}\"}" >/dev/null
+    -d"{\"Action\": \"${ACTION}\"}" >/dev/null
 }
 
 function get_ca_root {
@@ -822,15 +806,16 @@ function set_ttl_check_state {
   local DC=${3:-primary}
 
   case "$CHECK_STATE" in
-    pass)
-      ;;
-    warn)
-      ;;
-    fail)
-      ;;
-    *)
-      echo "invalid ttl check state '${CHECK_STATE}'" >&2
-      return 1
+  pass) ;;
+
+  warn) ;;
+
+  fail) ;;
+
+  *)
+    echo "invalid ttl check state '${CHECK_STATE}'" >&2
+    return 1
+    ;;
   esac
 
   retry_default docker_curl "${DC}" -sL -XPUT "http://localhost:8500/v1/agent/check/warn/${CHECK_ID}"
@@ -843,7 +828,7 @@ function get_upstream_fortio_name {
   local DEBUG_HEADER_VALUE="${4:-""}"
   local extra_args
   if [[ -n "${DEBUG_HEADER_VALUE}" ]]; then
-      extra_args="-H x-test-debug:${DEBUG_HEADER_VALUE}"
+    extra_args="-H x-test-debug:${DEBUG_HEADER_VALUE}"
   fi
   # split proto if https:// is at the front of the host since the --resolve
   # string needs just a bare host.
@@ -857,7 +842,7 @@ function get_upstream_fortio_name {
   # We use --resolve instead of setting a Host header since we need the right
   # name to be sent for SNI in some cases too.
   run retry_default curl -v -s -f --resolve "${HOST}:${PORT}:127.0.0.1" $extra_args \
-      "${PROTO}${HOST}:${PORT}${PREFIX}/debug?env=dump"
+    "${PROTO}${HOST}:${PORT}${PREFIX}/debug?env=dump"
 
   # Useful Debugging but breaks the expectation that the value output is just
   # the grep output when things don't fail
@@ -893,7 +878,7 @@ function assert_expected_fortio_name_pattern {
   GOT=$(get_upstream_fortio_name ${HOST} ${PORT} "${URL_PREFIX}" "${DEBUG_HEADER_VALUE}")
 
   if [[ "$GOT" =~ $EXPECT_NAME_PATTERN ]]; then
-      :
+    :
   else
     echo "expected name pattern: $EXPECT_NAME_PATTERN, actual name: $GOT" 1>&2
     return 1
@@ -907,10 +892,10 @@ function get_upstream_fortio_host_header {
   local DEBUG_HEADER_VALUE="${4:-""}"
   local extra_args
   if [[ -n "${DEBUG_HEADER_VALUE}" ]]; then
-      extra_args="-H x-test-debug:${DEBUG_HEADER_VALUE}"
+    extra_args="-H x-test-debug:${DEBUG_HEADER_VALUE}"
   fi
   run retry_default curl -v -s -f -H"Host: ${HOST}" $extra_args \
-      "localhost:${PORT}${PREFIX}/debug"
+    "localhost:${PORT}${PREFIX}/debug"
   [ "$status" == 0 ]
   echo "$output" | grep -E "^Host: "
 }
@@ -959,13 +944,13 @@ function register_lambdas {
   local DC=${1:-primary}
   # register lambdas to the catalog
   for f in $(find workdir/${DC}/register -type f -name 'lambda_*.json'); do
-    retry_default curl -sL -XPUT -d @${f} "http://localhost:8500/v1/catalog/register" >/dev/null && \
+    retry_default curl -sL -XPUT -d @${f} "http://localhost:8500/v1/catalog/register" >/dev/null &&
       echo "Registered Lambda: $(jq -r .Service.Service $f)"
   done
   # write service-defaults config entries for lambdas
   for f in $(find workdir/${DC}/register -type f -name 'service_defaults_*.json'); do
     varsub ${f} AWS_LAMBDA_REGION AWS_LAMBDA_ARN
-    retry_default curl -sL -XPUT -d @${f} "http://localhost:8500/v1/config" >/dev/null && \
+    retry_default curl -sL -XPUT -d @${f} "http://localhost:8500/v1/config" >/dev/null &&
       echo "Wrote config: $(jq -r '.Kind + " / " + .Name' $f)"
   done
 }
@@ -992,7 +977,8 @@ function assert_lambda_envoy_dynamic_http_filter_exists {
 }
 
 function varsub {
-  local file=$1 ; shift
+  local file=$1
+  shift
   for v in "$@"; do
     sed -i "s/\${$v}/${!v}/g" $file
   done