From ecc8a2d6f7db10bb92530be5d53548fb67de320c Mon Sep 17 00:00:00 2001 From: Chris Piraino Date: Tue, 21 Apr 2020 09:29:48 -0500 Subject: [PATCH] Allow ingress gateways to route through mesh gateways - Adds integration test for mesh gateways local + remote modes with ingress - ingress golden files updated for mesh gateway endpoints --- agent/xds/endpoints.go | 2 +- ...ver-through-local-gateway-triggered.golden | 34 ++++++++++ ...er-through-remote-gateway-triggered.golden | 34 ++++++++++ ...ver-through-local-gateway-triggered.golden | 34 ++++++++++ ...er-through-remote-gateway-triggered.golden | 34 ++++++++++ .../bind.hcl | 2 + .../capture.sh | 6 ++ .../primary/config_entries.hcl | 63 +++++++++++++++++++ .../primary/gateway.hcl | 5 ++ .../primary/ingress.hcl | 4 ++ .../primary/s1.hcl | 1 + .../primary/s2.hcl | 1 + .../primary/setup.sh | 12 ++++ .../primary/verify.bats | 59 +++++++++++++++++ .../secondary/gateway.hcl | 5 ++ .../secondary/join.hcl | 1 + .../secondary/setup.sh | 8 +++ .../secondary/verify.bats | 47 ++++++++++++++ .../vars.sh | 4 ++ 19 files changed, 355 insertions(+), 1 deletion(-) create mode 100644 test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/bind.hcl create mode 100644 test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/capture.sh create mode 100644 test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/primary/config_entries.hcl create mode 100644 test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/primary/gateway.hcl create mode 100644 test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/primary/ingress.hcl create mode 100644 test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/primary/s1.hcl create mode 100644 test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/primary/s2.hcl create mode 100644 test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/primary/setup.sh create mode 100644 test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/primary/verify.bats create mode 100644 test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/secondary/gateway.hcl create mode 100644 test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/secondary/join.hcl create mode 100644 test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/secondary/setup.sh create mode 100644 test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/secondary/verify.bats create mode 100644 test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/vars.sh diff --git a/agent/xds/endpoints.go b/agent/xds/endpoints.go index f571a20759..7b8eb91205 100644 --- a/agent/xds/endpoints.go +++ b/agent/xds/endpoints.go @@ -242,7 +242,7 @@ func (s *Server) endpointsFromSnapshotIngressGateway(cfgSnap *proxycfg.ConfigSna cfgSnap.IngressGateway.DiscoveryChain[id], cfgSnap.Datacenter, cfgSnap.IngressGateway.WatchedUpstreamEndpoints[id], - nil, + cfgSnap.IngressGateway.WatchedGatewayEndpoints[id], ) resources = append(resources, es...) } diff --git a/agent/xds/testdata/endpoints/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered.golden b/agent/xds/testdata/endpoints/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered.golden index b11569ce9e..5b9210ee92 100644 --- a/agent/xds/testdata/endpoints/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered.golden +++ b/agent/xds/testdata/endpoints/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered.golden @@ -1,6 +1,40 @@ { "versionInfo": "00000001", "resources": [ + { + "@type": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment", + "clusterName": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "10.10.1.1", + "portValue": 8443 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + }, + { + "endpoint": { + "address": { + "socketAddress": { + "address": "10.10.1.2", + "portValue": 8443 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + } ], "typeUrl": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment", "nonce": "00000001" diff --git a/agent/xds/testdata/endpoints/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered.golden b/agent/xds/testdata/endpoints/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered.golden index b11569ce9e..26e985cacc 100644 --- a/agent/xds/testdata/endpoints/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered.golden +++ b/agent/xds/testdata/endpoints/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered.golden @@ -1,6 +1,40 @@ { "versionInfo": "00000001", "resources": [ + { + "@type": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment", + "clusterName": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "198.38.1.1", + "portValue": 443 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + }, + { + "endpoint": { + "address": { + "socketAddress": { + "address": "198.38.1.2", + "portValue": 443 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + } ], "typeUrl": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment", "nonce": "00000001" diff --git a/agent/xds/testdata/endpoints/ingress-with-tcp-chain-failover-through-local-gateway-triggered.golden b/agent/xds/testdata/endpoints/ingress-with-tcp-chain-failover-through-local-gateway-triggered.golden index b11569ce9e..5b9210ee92 100644 --- a/agent/xds/testdata/endpoints/ingress-with-tcp-chain-failover-through-local-gateway-triggered.golden +++ b/agent/xds/testdata/endpoints/ingress-with-tcp-chain-failover-through-local-gateway-triggered.golden @@ -1,6 +1,40 @@ { "versionInfo": "00000001", "resources": [ + { + "@type": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment", + "clusterName": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "10.10.1.1", + "portValue": 8443 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + }, + { + "endpoint": { + "address": { + "socketAddress": { + "address": "10.10.1.2", + "portValue": 8443 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + } ], "typeUrl": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment", "nonce": "00000001" diff --git a/agent/xds/testdata/endpoints/ingress-with-tcp-chain-failover-through-remote-gateway-triggered.golden b/agent/xds/testdata/endpoints/ingress-with-tcp-chain-failover-through-remote-gateway-triggered.golden index b11569ce9e..11f798f292 100644 --- a/agent/xds/testdata/endpoints/ingress-with-tcp-chain-failover-through-remote-gateway-triggered.golden +++ b/agent/xds/testdata/endpoints/ingress-with-tcp-chain-failover-through-remote-gateway-triggered.golden @@ -1,6 +1,40 @@ { "versionInfo": "00000001", "resources": [ + { + "@type": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment", + "clusterName": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "198.18.1.1", + "portValue": 443 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + }, + { + "endpoint": { + "address": { + "socketAddress": { + "address": "198.18.1.2", + "portValue": 443 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + } ], "typeUrl": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment", "nonce": "00000001" diff --git a/test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/bind.hcl b/test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/bind.hcl new file mode 100644 index 0000000000..f54393f03e --- /dev/null +++ b/test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/bind.hcl @@ -0,0 +1,2 @@ +bind_addr = "0.0.0.0" +advertise_addr = "{{ GetInterfaceIP \"eth0\" }}" \ No newline at end of file diff --git a/test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/capture.sh b/test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/capture.sh new file mode 100644 index 0000000000..8f838e6e3a --- /dev/null +++ b/test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/capture.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +snapshot_envoy_admin localhost:20000 ingress-gateway primary || true +snapshot_envoy_admin localhost:19001 s2 secondary || true +snapshot_envoy_admin localhost:19002 mesh-gateway primary || true +snapshot_envoy_admin localhost:19003 mesh-gateway secondary || true diff --git a/test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/primary/config_entries.hcl b/test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/primary/config_entries.hcl new file mode 100644 index 0000000000..cec14adcbf --- /dev/null +++ b/test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/primary/config_entries.hcl @@ -0,0 +1,63 @@ +enable_central_service_config = true + +config_entries { + bootstrap { + kind = "ingress-gateway" + name = "ingress-gateway" + + listeners = [ + { + protocol = "tcp" + port = 9999 + services = [ + { + name = "s2" + } + ] + }, + { + protocol = "tcp" + port = 10000 + services = [ + { + name = "s1" + } + ] + } + ] + } + + bootstrap { + kind = "proxy-defaults" + name = "global" + mesh_gateway { + mode = "local" + } + } + + bootstrap { + kind = "service-resolver" + name = "s2" + redirect { + service = "s2" + datacenter = "secondary" + } + } + + bootstrap { + kind = "service-defaults" + name = "s1" + mesh_gateway { + mode = "remote" + } + } + + bootstrap { + kind = "service-resolver" + name = "s1" + redirect { + service = "s1" + datacenter = "secondary" + } + } +} diff --git a/test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/primary/gateway.hcl b/test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/primary/gateway.hcl new file mode 100644 index 0000000000..7bdbfd6dc1 --- /dev/null +++ b/test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/primary/gateway.hcl @@ -0,0 +1,5 @@ +services { + name = "mesh-gateway" + kind = "mesh-gateway" + port = 4431 +} \ No newline at end of file diff --git a/test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/primary/ingress.hcl b/test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/primary/ingress.hcl new file mode 100644 index 0000000000..781ef1851b --- /dev/null +++ b/test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/primary/ingress.hcl @@ -0,0 +1,4 @@ +services { + name = "ingress-gateway" + kind = "ingress-gateway" +} diff --git a/test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/primary/s1.hcl b/test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/primary/s1.hcl new file mode 100644 index 0000000000..1a239d879f --- /dev/null +++ b/test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/primary/s1.hcl @@ -0,0 +1 @@ +# We don't want an s1 service in the primary dc diff --git a/test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/primary/s2.hcl b/test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/primary/s2.hcl new file mode 100644 index 0000000000..77164e722b --- /dev/null +++ b/test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/primary/s2.hcl @@ -0,0 +1 @@ +# We don't want an s2 service in the primary dc \ No newline at end of file diff --git a/test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/primary/setup.sh b/test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/primary/setup.sh new file mode 100644 index 0000000000..f5531627ab --- /dev/null +++ b/test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/primary/setup.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +set -eEuo pipefail + +# wait for bootstrap to apply config entries +wait_for_config_entry ingress-gateway ingress-gateway +wait_for_config_entry proxy-defaults global + +gen_envoy_bootstrap mesh-gateway 19002 primary true +gen_envoy_bootstrap ingress-gateway 20000 primary true +retry_default docker_consul primary curl -s "http://localhost:8500/v1/catalog/service/consul?dc=secondary" >/dev/null + diff --git a/test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/primary/verify.bats b/test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/primary/verify.bats new file mode 100644 index 0000000000..a45cdb889b --- /dev/null +++ b/test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/primary/verify.bats @@ -0,0 +1,59 @@ +#!/usr/bin/env bats + +load helpers + +@test "gateway-primary proxy admin is up on :19002" { + retry_default curl -f -s localhost:19002/stats -o /dev/null +} + +@test "ingress-primary proxy admin is up on :20000" { + retry_default curl -f -s localhost:20000/stats -o /dev/null +} + +@test "ingress should have healthy endpoints for s1" { + assert_upstream_has_endpoints_in_status 127.0.0.1:20000 s1.default.secondary HEALTHY 1 +} + +@test "ingress should have healthy endpoints for s2" { + assert_upstream_has_endpoints_in_status 127.0.0.1:20000 s2.default.secondary HEALTHY 1 +} + +@test "gateway-primary should have healthy endpoints for secondary" { + assert_upstream_has_endpoints_in_status 127.0.0.1:19002 secondary HEALTHY 1 +} + +@test "gateway-secondary should have healthy endpoints for s1" { + assert_upstream_has_endpoints_in_status consul-secondary:19003 s1 HEALTHY 1 +} + +@test "gateway-secondary should have healthy endpoints for s2" { + assert_upstream_has_endpoints_in_status consul-secondary:19003 s2 HEALTHY 1 +} + +@test "ingress should be able to connect to s1" { + run retry_default curl -s -f -d hello localhost:10000 + [ "$status" -eq 0 ] + [ "$output" = "hello" ] +} + +@test "ingress made 1 connection to s1" { + assert_envoy_metric_at_least 127.0.0.1:20000 "cluster.s1.default.secondary.*cx_total" 1 +} + +@test "gateway-primary is not used for the upstream connection to s1" { + assert_envoy_metric 127.0.0.1:19002 "cluster.secondary.*cx_total" 0 +} + +@test "ingress should be able to connect to s2" { + run retry_default curl -s -f -d hello localhost:9999 + [ "$status" -eq 0 ] + [ "$output" = "hello" ] +} + +@test "ingress made 1 connection to s2" { + assert_envoy_metric_at_least 127.0.0.1:20000 "cluster.s2.default.secondary.*cx_total" 1 +} + +@test "gateway-primary is used for the upstream connection to s2" { + assert_envoy_metric_at_least 127.0.0.1:19002 "cluster.secondary.*cx_total" 1 +} diff --git a/test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/secondary/gateway.hcl b/test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/secondary/gateway.hcl new file mode 100644 index 0000000000..55f03ac050 --- /dev/null +++ b/test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/secondary/gateway.hcl @@ -0,0 +1,5 @@ +services { + name = "mesh-gateway" + kind = "mesh-gateway" + port = 4432 +} \ No newline at end of file diff --git a/test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/secondary/join.hcl b/test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/secondary/join.hcl new file mode 100644 index 0000000000..fb1307d62a --- /dev/null +++ b/test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/secondary/join.hcl @@ -0,0 +1 @@ +retry_join_wan = ["consul-primary"] \ No newline at end of file diff --git a/test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/secondary/setup.sh b/test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/secondary/setup.sh new file mode 100644 index 0000000000..8bd31bb039 --- /dev/null +++ b/test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/secondary/setup.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +set -eEuo pipefail + +gen_envoy_bootstrap s1 19001 secondary +gen_envoy_bootstrap s2 19002 secondary +gen_envoy_bootstrap mesh-gateway 19003 secondary true +retry_default docker_consul secondary curl -s "http://localhost:8500/v1/catalog/service/consul?dc=primary" >/dev/null diff --git a/test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/secondary/verify.bats b/test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/secondary/verify.bats new file mode 100644 index 0000000000..7daf0d7aca --- /dev/null +++ b/test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/secondary/verify.bats @@ -0,0 +1,47 @@ +#!/usr/bin/env bats + +load helpers + +@test "s1 proxy is running correct version" { + assert_envoy_version 19001 +} + +@test "s2 proxy is running correct version" { + assert_envoy_version 19002 +} + +@test "s1 proxy admin is up on :19001" { + retry_default curl -f -s localhost:19001/stats -o /dev/null +} + +@test "s2 proxy admin is up on :19002" { + retry_default curl -f -s localhost:19002/stats -o /dev/null +} + +@test "gateway-secondary proxy admin is up on :19003" { + retry_default curl -f -s localhost:19003/stats -o /dev/null +} + +@test "s1 proxy listener should be up and have right cert" { + assert_proxy_presents_cert_uri localhost:21000 s1 secondary +} + +@test "s2 proxy listener should be up and have right cert" { + assert_proxy_presents_cert_uri localhost:21001 s2 secondary +} + +@test "s1 proxy should be healthy" { + assert_service_has_healthy_instances s1 1 secondary +} + +@test "s2 proxy should be healthy" { + assert_service_has_healthy_instances s2 1 secondary +} + +@test "gateway-secondary is used for the upstream connection for s1" { + assert_envoy_metric_at_least 127.0.0.1:19003 "cluster.s1.default.secondary.*cx_total" 1 +} + +@test "gateway-secondary is used for the upstream connection for s2" { + assert_envoy_metric_at_least 127.0.0.1:19003 "cluster.s2.default.secondary.*cx_total" 1 +} diff --git a/test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/vars.sh b/test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/vars.sh new file mode 100644 index 0000000000..eab6bb3ee0 --- /dev/null +++ b/test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/vars.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +export REQUIRED_SERVICES="gateway-primary s1-secondary s1-sidecar-proxy-secondary s2-secondary s2-sidecar-proxy-secondary gateway-secondary ingress-gateway-primary" +export REQUIRE_SECONDARY=1