Allow ingress gateways to route through mesh gateways

- Adds integration test for mesh gateways local + remote modes with ingress
- ingress golden files updated for mesh gateway endpoints

agent/xds/endpoints.go

@@ -242,7 +242,7 @@ func (s *Server) endpointsFromSnapshotIngressGateway(cfgSnap *proxycfg.ConfigSna
 			cfgSnap.IngressGateway.DiscoveryChain[id],
 			cfgSnap.Datacenter,
 			cfgSnap.IngressGateway.WatchedUpstreamEndpoints[id],
-			nil,
+			cfgSnap.IngressGateway.WatchedGatewayEndpoints[id],
 		)
 		resources = append(resources, es...)
 	}

agent/xds/testdata/endpoints/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered.golden

@@ -1,6 +1,40 @@
 {
   "versionInfo": "00000001",
   "resources": [
+    {
+      "@type": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment",
+      "clusterName": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
+      "endpoints": [
+        {
+          "lbEndpoints": [
+            {
+              "endpoint": {
+                "address": {
+                  "socketAddress": {
+                    "address": "10.10.1.1",
+                    "portValue": 8443
+                  }
+                }
+              },
+              "healthStatus": "HEALTHY",
+              "loadBalancingWeight": 1
+            },
+            {
+              "endpoint": {
+                "address": {
+                  "socketAddress": {
+                    "address": "10.10.1.2",
+                    "portValue": 8443
+                  }
+                }
+              },
+              "healthStatus": "HEALTHY",
+              "loadBalancingWeight": 1
+            }
+          ]
+        }
+      ]
+    }
   ],
   "typeUrl": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment",
   "nonce": "00000001"

agent/xds/testdata/endpoints/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered.golden

@@ -1,6 +1,40 @@
 {
   "versionInfo": "00000001",
   "resources": [
+    {
+      "@type": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment",
+      "clusterName": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
+      "endpoints": [
+        {
+          "lbEndpoints": [
+            {
+              "endpoint": {
+                "address": {
+                  "socketAddress": {
+                    "address": "198.38.1.1",
+                    "portValue": 443
+                  }
+                }
+              },
+              "healthStatus": "HEALTHY",
+              "loadBalancingWeight": 1
+            },
+            {
+              "endpoint": {
+                "address": {
+                  "socketAddress": {
+                    "address": "198.38.1.2",
+                    "portValue": 443
+                  }
+                }
+              },
+              "healthStatus": "HEALTHY",
+              "loadBalancingWeight": 1
+            }
+          ]
+        }
+      ]
+    }
   ],
   "typeUrl": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment",
   "nonce": "00000001"

agent/xds/testdata/endpoints/ingress-with-tcp-chain-failover-through-local-gateway-triggered.golden

@@ -1,6 +1,40 @@
 {
   "versionInfo": "00000001",
   "resources": [
+    {
+      "@type": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment",
+      "clusterName": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
+      "endpoints": [
+        {
+          "lbEndpoints": [
+            {
+              "endpoint": {
+                "address": {
+                  "socketAddress": {
+                    "address": "10.10.1.1",
+                    "portValue": 8443
+                  }
+                }
+              },
+              "healthStatus": "HEALTHY",
+              "loadBalancingWeight": 1
+            },
+            {
+              "endpoint": {
+                "address": {
+                  "socketAddress": {
+                    "address": "10.10.1.2",
+                    "portValue": 8443
+                  }
+                }
+              },
+              "healthStatus": "HEALTHY",
+              "loadBalancingWeight": 1
+            }
+          ]
+        }
+      ]
+    }
   ],
   "typeUrl": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment",
   "nonce": "00000001"

agent/xds/testdata/endpoints/ingress-with-tcp-chain-failover-through-remote-gateway-triggered.golden

@@ -1,6 +1,40 @@
 {
   "versionInfo": "00000001",
   "resources": [
+    {
+      "@type": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment",
+      "clusterName": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
+      "endpoints": [
+        {
+          "lbEndpoints": [
+            {
+              "endpoint": {
+                "address": {
+                  "socketAddress": {
+                    "address": "198.18.1.1",
+                    "portValue": 443
+                  }
+                }
+              },
+              "healthStatus": "HEALTHY",
+              "loadBalancingWeight": 1
+            },
+            {
+              "endpoint": {
+                "address": {
+                  "socketAddress": {
+                    "address": "198.18.1.2",
+                    "portValue": 443
+                  }
+                }
+              },
+              "healthStatus": "HEALTHY",
+              "loadBalancingWeight": 1
+            }
+          ]
+        }
+      ]
+    }
   ],
   "typeUrl": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment",
   "nonce": "00000001"

test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/bind.hcl

@@ -0,0 +1,2 @@
+bind_addr = "0.0.0.0"
+advertise_addr = "{{ GetInterfaceIP \"eth0\" }}"

test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/capture.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+snapshot_envoy_admin localhost:20000 ingress-gateway primary || true
+snapshot_envoy_admin localhost:19001 s2 secondary || true
+snapshot_envoy_admin localhost:19002 mesh-gateway primary || true
+snapshot_envoy_admin localhost:19003 mesh-gateway secondary || true

test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/primary/config_entries.hcl

@@ -0,0 +1,63 @@
+enable_central_service_config = true
+
+config_entries {
+  bootstrap {
+    kind = "ingress-gateway"
+    name = "ingress-gateway"
+
+    listeners = [
+      {
+        protocol = "tcp"
+        port = 9999
+        services = [
+          {
+            name = "s2"
+          }
+        ]
+      },
+      {
+        protocol = "tcp"
+        port = 10000
+        services = [
+          {
+            name = "s1"
+          }
+        ]
+      }
+    ]
+  }
+
+  bootstrap {
+    kind = "proxy-defaults"
+    name = "global"
+    mesh_gateway {
+      mode = "local"
+    }
+  }
+
+  bootstrap {
+    kind = "service-resolver"
+    name = "s2"
+    redirect {
+      service = "s2"
+      datacenter = "secondary"
+    }
+  }
+
+  bootstrap {
+    kind = "service-defaults"
+    name = "s1"
+    mesh_gateway {
+      mode = "remote"
+    }
+  }
+
+  bootstrap {
+    kind = "service-resolver"
+    name = "s1"
+    redirect {
+      service = "s1"
+      datacenter = "secondary"
+    }
+  }
+}

test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/primary/gateway.hcl

@@ -0,0 +1,5 @@
+services {
+  name = "mesh-gateway"
+  kind = "mesh-gateway"
+  port = 4431
+}

test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/primary/ingress.hcl

@@ -0,0 +1,4 @@
+services {
+  name = "ingress-gateway"
+  kind = "ingress-gateway"
+}

test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/primary/s1.hcl

@@ -0,0 +1 @@
+# We don't want an s1 service in the primary dc

test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/primary/s2.hcl

@@ -0,0 +1 @@
+# We don't want an s2 service in the primary dc

test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/primary/setup.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -eEuo pipefail
+
+# wait for bootstrap to apply config entries
+wait_for_config_entry ingress-gateway ingress-gateway
+wait_for_config_entry proxy-defaults global
+
+gen_envoy_bootstrap mesh-gateway 19002 primary true
+gen_envoy_bootstrap ingress-gateway 20000 primary true
+retry_default docker_consul primary curl -s "http://localhost:8500/v1/catalog/service/consul?dc=secondary" >/dev/null
+

test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/primary/verify.bats

@@ -0,0 +1,59 @@
+#!/usr/bin/env bats
+
+load helpers
+
+@test "gateway-primary proxy admin is up on :19002" {
+  retry_default curl -f -s localhost:19002/stats -o /dev/null
+}
+
+@test "ingress-primary proxy admin is up on :20000" {
+  retry_default curl -f -s localhost:20000/stats -o /dev/null
+}
+
+@test "ingress should have healthy endpoints for s1" {
+  assert_upstream_has_endpoints_in_status 127.0.0.1:20000 s1.default.secondary HEALTHY 1
+}
+
+@test "ingress should have healthy endpoints for s2" {
+  assert_upstream_has_endpoints_in_status 127.0.0.1:20000 s2.default.secondary HEALTHY 1
+}
+
+@test "gateway-primary should have healthy endpoints for secondary" {
+   assert_upstream_has_endpoints_in_status 127.0.0.1:19002 secondary HEALTHY 1
+}
+
+@test "gateway-secondary should have healthy endpoints for s1" {
+   assert_upstream_has_endpoints_in_status consul-secondary:19003 s1 HEALTHY 1
+}
+
+@test "gateway-secondary should have healthy endpoints for s2" {
+   assert_upstream_has_endpoints_in_status consul-secondary:19003 s2 HEALTHY 1
+}
+
+@test "ingress should be able to connect to s1" {
+  run retry_default curl -s -f -d hello localhost:10000
+  [ "$status" -eq 0 ]
+  [ "$output" = "hello" ]
+}
+
+@test "ingress made 1 connection to s1" {
+  assert_envoy_metric_at_least 127.0.0.1:20000 "cluster.s1.default.secondary.*cx_total" 1
+}
+
+@test "gateway-primary is not used for the upstream connection to s1" {
+  assert_envoy_metric 127.0.0.1:19002 "cluster.secondary.*cx_total" 0
+}
+
+@test "ingress should be able to connect to s2" {
+  run retry_default curl -s -f -d hello localhost:9999
+  [ "$status" -eq 0 ]
+  [ "$output" = "hello" ]
+}
+
+@test "ingress made 1 connection to s2" {
+  assert_envoy_metric_at_least 127.0.0.1:20000 "cluster.s2.default.secondary.*cx_total" 1
+}
+
+@test "gateway-primary is used for the upstream connection to s2" {
+  assert_envoy_metric_at_least 127.0.0.1:19002 "cluster.secondary.*cx_total" 1
+}

test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/secondary/gateway.hcl

@@ -0,0 +1,5 @@
+services {
+  name = "mesh-gateway"
+  kind = "mesh-gateway"
+  port = 4432
+}

test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/secondary/join.hcl

@@ -0,0 +1 @@
+retry_join_wan = ["consul-primary"]

test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/secondary/setup.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -eEuo pipefail
+
+gen_envoy_bootstrap s1 19001 secondary
+gen_envoy_bootstrap s2 19002 secondary
+gen_envoy_bootstrap mesh-gateway 19003 secondary true
+retry_default docker_consul secondary curl -s  "http://localhost:8500/v1/catalog/service/consul?dc=primary" >/dev/null

test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/secondary/verify.bats

@@ -0,0 +1,47 @@
+#!/usr/bin/env bats
+
+load helpers
+
+@test "s1 proxy is running correct version" {
+  assert_envoy_version 19001
+}
+
+@test "s2 proxy is running correct version" {
+  assert_envoy_version 19002
+}
+
+@test "s1 proxy admin is up on :19001" {
+  retry_default curl -f -s localhost:19001/stats -o /dev/null
+}
+
+@test "s2 proxy admin is up on :19002" {
+  retry_default curl -f -s localhost:19002/stats -o /dev/null
+}
+
+@test "gateway-secondary proxy admin is up on :19003" {
+  retry_default curl -f -s localhost:19003/stats -o /dev/null
+}
+
+@test "s1 proxy listener should be up and have right cert" {
+  assert_proxy_presents_cert_uri localhost:21000 s1 secondary
+}
+
+@test "s2 proxy listener should be up and have right cert" {
+  assert_proxy_presents_cert_uri localhost:21001 s2 secondary
+}
+
+@test "s1 proxy should be healthy" {
+  assert_service_has_healthy_instances s1 1 secondary
+}
+
+@test "s2 proxy should be healthy" {
+  assert_service_has_healthy_instances s2 1 secondary
+}
+
+@test "gateway-secondary is used for the upstream connection for s1" {
+  assert_envoy_metric_at_least 127.0.0.1:19003 "cluster.s1.default.secondary.*cx_total" 1
+}
+
+@test "gateway-secondary is used for the upstream connection for s2" {
+  assert_envoy_metric_at_least 127.0.0.1:19003 "cluster.s2.default.secondary.*cx_total" 1
+}

test/integration/connect/envoy/case-ingress-mesh-gateways-resolver/vars.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+export REQUIRED_SERVICES="gateway-primary s1-secondary s1-sidecar-proxy-secondary s2-secondary s2-sidecar-proxy-secondary gateway-secondary ingress-gateway-primary"
+export REQUIRE_SECONDARY=1