diff --git a/.changelog/16661.txt b/.changelog/16661.txt new file mode 100644 index 0000000000..4133650211 --- /dev/null +++ b/.changelog/16661.txt @@ -0,0 +1,3 @@ +```release-note:bug +gateways: Fixes a bug API gateways using HTTP listeners were taking upwards of 15 seconds to get configured over xDS. +``` diff --git a/agent/consul/discoverychain/gateway.go b/agent/consul/discoverychain/gateway.go index 35a8992d80..e834b7a1d4 100644 --- a/agent/consul/discoverychain/gateway.go +++ b/agent/consul/discoverychain/gateway.go @@ -128,6 +128,29 @@ func (l *GatewayChainSynthesizer) Synthesize(chains ...*structs.CompiledDiscover return nil, nil, err } + node := compiled.Nodes[compiled.StartNode] + if node.IsRouter() { + resolverPrefix := structs.DiscoveryGraphNodeTypeResolver + ":" + node.Name + + // clean out the clusters that will get added for the router + for name := range compiled.Nodes { + if strings.HasPrefix(name, resolverPrefix) { + delete(compiled.Nodes, name) + } + } + + // clean out the route rules that'll get added for the router + filtered := []*structs.DiscoveryRoute{} + for _, route := range node.Routes { + if strings.HasPrefix(route.NextNode, resolverPrefix) { + continue + } + filtered = append(filtered, route) + } + node.Routes = filtered + } + compiled.Nodes[compiled.StartNode] = node + // fix up the nodes for the terminal targets to either be a splitter or resolver if there is no splitter present for name, node := range compiled.Nodes { switch node.Type { diff --git a/agent/consul/discoverychain/gateway_test.go b/agent/consul/discoverychain/gateway_test.go index 71e66b0512..57d236afdc 100644 --- a/agent/consul/discoverychain/gateway_test.go +++ b/agent/consul/discoverychain/gateway_test.go @@ -47,7 +47,7 @@ func TestGatewayChainSynthesizer_AddHTTPRoute(t *testing.T) { route structs.HTTPRouteConfigEntry expectedMatchesByHostname map[string][]hostnameMatch }{ - "no hostanames": { + "no hostnames": { route: structs.HTTPRouteConfigEntry{ Kind: structs.HTTPRoute, Name: "route", @@ -539,15 +539,6 @@ func TestGatewayChainSynthesizer_Synthesize(t *testing.T) { Protocol: "http", StartNode: "router:gateway-suffix-9b9265b.default.default", Nodes: map[string]*structs.DiscoveryGraphNode{ - "resolver:gateway-suffix-9b9265b.default.default.dc1": { - Type: "resolver", - Name: "gateway-suffix-9b9265b.default.default.dc1", - Resolver: &structs.DiscoveryResolver{ - Target: "gateway-suffix-9b9265b.default.default.dc1", - Default: true, - ConnectTimeout: 5000000000, - }, - }, "router:gateway-suffix-9b9265b.default.default": { Type: "router", Name: "gateway-suffix-9b9265b.default.default", @@ -569,20 +560,6 @@ func TestGatewayChainSynthesizer_Synthesize(t *testing.T) { }, }, NextNode: "resolver:foo.default.default.dc1", - }, { - Definition: &structs.ServiceRoute{ - Match: &structs.ServiceRouteMatch{ - HTTP: &structs.ServiceRouteHTTPMatch{ - PathPrefix: "/", - }, - }, - Destination: &structs.ServiceRouteDestination{ - Service: "gateway-suffix-9b9265b", - Partition: "default", - Namespace: "default", - }, - }, - NextNode: "resolver:gateway-suffix-9b9265b.default.default.dc1", }}, }, "resolver:foo.default.default.dc1": { @@ -704,15 +681,6 @@ func TestGatewayChainSynthesizer_ComplexChain(t *testing.T) { Protocol: "http", StartNode: "router:gateway-suffix-9b9265b.default.default", Nodes: map[string]*structs.DiscoveryGraphNode{ - "resolver:gateway-suffix-9b9265b.default.default.dc1": { - Type: "resolver", - Name: "gateway-suffix-9b9265b.default.default.dc1", - Resolver: &structs.DiscoveryResolver{ - Target: "gateway-suffix-9b9265b.default.default.dc1", - Default: true, - ConnectTimeout: 5000000000, - }, - }, "resolver:service-one.default.default.dc1": { Type: "resolver", Name: "service-one.default.default.dc1", @@ -770,20 +738,6 @@ func TestGatewayChainSynthesizer_ComplexChain(t *testing.T) { }, }, NextNode: "splitter:splitter-one.default.default", - }, { - Definition: &structs.ServiceRoute{ - Match: &structs.ServiceRouteMatch{ - HTTP: &structs.ServiceRouteHTTPMatch{ - PathPrefix: "/", - }, - }, - Destination: &structs.ServiceRouteDestination{ - Service: "gateway-suffix-9b9265b", - Partition: "default", - Namespace: "default", - }, - }, - NextNode: "resolver:gateway-suffix-9b9265b.default.default.dc1", }}, }, "splitter:splitter-one.default.default": { diff --git a/agent/xds/resources_test.go b/agent/xds/resources_test.go index d5009f6368..24bee76606 100644 --- a/agent/xds/resources_test.go +++ b/agent/xds/resources_test.go @@ -11,6 +11,8 @@ import ( envoy_route_v3 "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" envoy_tls_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" + "github.com/hashicorp/consul/agent/connect" + "github.com/hashicorp/consul/agent/consul/discoverychain" "github.com/hashicorp/consul/agent/xds/testcommon" "github.com/hashicorp/consul/envoyextensions/xdscommon" @@ -175,7 +177,7 @@ func TestAllResourcesFromSnapshot(t *testing.T) { tests = append(tests, getMeshGatewayPeeringGoldenTestCases()...) tests = append(tests, getTrafficControlPeeringGoldenTestCases()...) tests = append(tests, getEnterpriseGoldenTestCases()...) - tests = append(tests, getAPIGatewayGoldenTestCases()...) + tests = append(tests, getAPIGatewayGoldenTestCases(t)...) latestEnvoyVersion := xdscommon.EnvoyVersions[0] for _, envoyVersion := range xdscommon.EnvoyVersions { @@ -314,7 +316,13 @@ AAJAMaoXmoYVdgXV+CPuBb2M4XCpuzLu3bcA2PXm5ipSyIgntMKwXV7r -----END CERTIFICATE-----` ) -func getAPIGatewayGoldenTestCases() []goldenTestCase { +func getAPIGatewayGoldenTestCases(t *testing.T) []goldenTestCase { + t.Helper() + + service := structs.NewServiceName("service", nil) + serviceUID := proxycfg.NewUpstreamIDFromServiceName(service) + serviceChain := discoverychain.TestCompileConfigEntries(t, "service", "default", "default", "dc1", connect.TestClusterID+".consul", nil) + return []goldenTestCase{ { name: "api-gateway-with-tcp-route-and-inline-certificate", @@ -362,5 +370,48 @@ func getAPIGatewayGoldenTestCases() []goldenTestCase { }}, nil) }, }, + { + name: "api-gateway-with-http-route-and-inline-certificate", + create: func(t testinf.T) *proxycfg.ConfigSnapshot { + return proxycfg.TestConfigSnapshotAPIGateway(t, "default", nil, func(entry *structs.APIGatewayConfigEntry, bound *structs.BoundAPIGatewayConfigEntry) { + entry.Listeners = []structs.APIGatewayListener{ + { + Name: "listener", + Protocol: structs.ListenerProtocolHTTP, + Port: 8080, + }, + } + bound.Listeners = []structs.BoundAPIGatewayListener{ + { + Name: "listener", + Routes: []structs.ResourceReference{{ + Kind: structs.HTTPRoute, + Name: "route", + }}, + }, + } + }, []structs.BoundRoute{ + &structs.HTTPRouteConfigEntry{ + Kind: structs.HTTPRoute, + Name: "route", + Rules: []structs.HTTPRouteRule{{ + Services: []structs.HTTPService{{ + Name: "service", + }}, + }}, + }, + }, nil, []proxycfg.UpdateEvent{{ + CorrelationID: "discovery-chain:" + serviceUID.String(), + Result: &structs.DiscoveryChainResponse{ + Chain: serviceChain, + }, + }, { + CorrelationID: "upstream-target:" + serviceChain.ID() + ":" + serviceUID.String(), + Result: &structs.IndexedCheckServiceNodes{ + Nodes: proxycfg.TestUpstreamNodes(t, "service"), + }, + }}) + }, + }, } } diff --git a/agent/xds/testdata/clusters/api-gateway-with-http-route-and-inline-certificate.latest.golden b/agent/xds/testdata/clusters/api-gateway-with-http-route-and-inline-certificate.latest.golden new file mode 100644 index 0000000000..e20479dfd1 --- /dev/null +++ b/agent/xds/testdata/clusters/api-gateway-with-http-route-and-inline-certificate.latest.golden @@ -0,0 +1,55 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "name": "service.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "altStatName": "service.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "type": "EDS", + "edsClusterConfig": { + "edsConfig": { + "ads": {}, + "resourceApiVersion": "V3" + } + }, + "connectTimeout": "5s", + "circuitBreakers": {}, + "outlierDetection": {}, + "commonLbConfig": { + "healthyPanicThreshold": {} + }, + "transportSocket": { + "name": "tls", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext", + "commonTlsContext": { + "tlsParams": {}, + "tlsCertificates": [ + { + "certificateChain": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n" + }, + "privateKey": { + "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n" + } + } + ], + "validationContext": { + "trustedCa": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n" + }, + "matchSubjectAltNames": [ + { + "exact": "spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/service" + } + ] + } + }, + "sni": "service.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + } + } + } + ], + "typeUrl": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/endpoints/api-gateway-with-http-route-and-inline-certificate.latest.golden b/agent/xds/testdata/endpoints/api-gateway-with-http-route-and-inline-certificate.latest.golden new file mode 100644 index 0000000000..18adccd10c --- /dev/null +++ b/agent/xds/testdata/endpoints/api-gateway-with-http-route-and-inline-certificate.latest.golden @@ -0,0 +1,41 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", + "clusterName": "service.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "10.10.1.1", + "portValue": 8080 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + }, + { + "endpoint": { + "address": { + "socketAddress": { + "address": "10.10.1.2", + "portValue": 8080 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + } + ], + "typeUrl": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/listeners/api-gateway-with-http-route-and-inline-certificate.latest.golden b/agent/xds/testdata/listeners/api-gateway-with-http-route-and-inline-certificate.latest.golden new file mode 100644 index 0000000000..e9bee988de --- /dev/null +++ b/agent/xds/testdata/listeners/api-gateway-with-http-route-and-inline-certificate.latest.golden @@ -0,0 +1,49 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", + "name": "http:1.2.3.4:8080", + "address": { + "socketAddress": { + "address": "1.2.3.4", + "portValue": 8080 + } + }, + "filterChains": [ + { + "filters": [ + { + "name": "envoy.filters.network.http_connection_manager", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager", + "statPrefix": "ingress_upstream_8080", + "rds": { + "configSource": { + "ads": {}, + "resourceApiVersion": "V3" + }, + "routeConfigName": "8080" + }, + "httpFilters": [ + { + "name": "envoy.filters.http.router", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router" + } + } + ], + "tracing": { + "randomSampling": {} + } + } + } + ] + } + ], + "trafficDirection": "OUTBOUND" + } + ], + "typeUrl": "type.googleapis.com/envoy.config.listener.v3.Listener", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/routes/api-gateway-with-http-route-and-inline-certificate.latest.golden b/agent/xds/testdata/routes/api-gateway-with-http-route-and-inline-certificate.latest.golden new file mode 100644 index 0000000000..6abc6f2946 --- /dev/null +++ b/agent/xds/testdata/routes/api-gateway-with-http-route-and-inline-certificate.latest.golden @@ -0,0 +1,31 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + "name": "8080", + "virtualHosts": [ + { + "name": "api-gateway-listener-9b9265b", + "domains": [ + "*", + "*:8080" + ], + "routes": [ + { + "match": { + "prefix": "/" + }, + "route": { + "cluster": "service.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + } + } + ] + } + ], + "validateClusters": true + } + ], + "typeUrl": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/secrets/api-gateway-with-http-route-and-inline-certificate.latest.golden b/agent/xds/testdata/secrets/api-gateway-with-http-route-and-inline-certificate.latest.golden new file mode 100644 index 0000000000..95612291de --- /dev/null +++ b/agent/xds/testdata/secrets/api-gateway-with-http-route-and-inline-certificate.latest.golden @@ -0,0 +1,5 @@ +{ + "versionInfo": "00000001", + "typeUrl": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret", + "nonce": "00000001" +} \ No newline at end of file diff --git a/test/integration/consul-container/test/gateways/gateway_endpoint_test.go b/test/integration/consul-container/test/gateways/gateway_endpoint_test.go index d79bf7c65e..da642f1834 100644 --- a/test/integration/consul-container/test/gateways/gateway_endpoint_test.go +++ b/test/integration/consul-container/test/gateways/gateway_endpoint_test.go @@ -9,28 +9,30 @@ import ( "testing" "time" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/hashicorp/consul/api" - "github.com/hashicorp/consul/sdk/testutil/retry" libassert "github.com/hashicorp/consul/test/integration/consul-container/libs/assert" libcluster "github.com/hashicorp/consul/test/integration/consul-container/libs/cluster" libservice "github.com/hashicorp/consul/test/integration/consul-container/libs/service" libtopology "github.com/hashicorp/consul/test/integration/consul-container/libs/topology" - "github.com/hashicorp/consul/test/integration/consul-container/libs/utils" "github.com/hashicorp/go-cleanhttp" ) // Creates a gateway service and tests to see if it is routable func TestAPIGatewayCreate(t *testing.T) { - t.Skip() if testing.Short() { t.Skip("too slow for testing.Short") } t.Parallel() + + gatewayName := randomName("gateway", 16) + routeName := randomName("route", 16) + serviceName := randomName("service", 16) listenerPortOne := 6000 + serviceHTTPPort := 6001 + serviceGRPCPort := 6002 clusterConfig := &libtopology.ClusterConfig{ NumServers: 1, @@ -41,17 +43,28 @@ func TestAPIGatewayCreate(t *testing.T) { InjectGossipEncryption: true, AllowHTTPAnyway: true, }, - Ports: []int{listenerPortOne}, - ApplyDefaultProxySettings: true, + Ports: []int{ + listenerPortOne, + serviceHTTPPort, + serviceGRPCPort, + }, } cluster, _, _ := libtopology.NewCluster(t, clusterConfig) client := cluster.APIClient(0) + namespace := getNamespace() + if namespace != "" { + ns := &api.Namespace{Name: namespace} + _, _, err := client.Namespaces().Create(ns, nil) + require.NoError(t, err) + } + // add api gateway config apiGateway := &api.APIGatewayConfigEntry{ - Kind: api.APIGateway, - Name: "api-gateway", + Kind: api.APIGateway, + Namespace: namespace, + Name: gatewayName, Listeners: []api.APIGatewayListener{ { Name: "listener", @@ -63,32 +76,48 @@ func TestAPIGatewayCreate(t *testing.T) { require.NoError(t, cluster.ConfigEntryWrite(apiGateway)) + _, _, err := libservice.CreateAndRegisterStaticServerAndSidecar(cluster.Agents[0], &libservice.ServiceOpts{ + ID: serviceName, + Name: serviceName, + Namespace: namespace, + HTTPPort: serviceHTTPPort, + GRPCPort: serviceGRPCPort, + }) + require.NoError(t, err) + tcpRoute := &api.TCPRouteConfigEntry{ - Kind: api.TCPRoute, - Name: "api-gateway-route", + Kind: api.TCPRoute, + Name: routeName, + Namespace: namespace, Parents: []api.ResourceReference{ { - Kind: api.APIGateway, - Name: "api-gateway", + Kind: api.APIGateway, + Namespace: namespace, + Name: gatewayName, }, }, Services: []api.TCPService{ { - Name: libservice.StaticServerServiceName, + Namespace: namespace, + Name: serviceName, }, }, } require.NoError(t, cluster.ConfigEntryWrite(tcpRoute)) - // Create a client proxy instance with the server as an upstream - _, gatewayService := createServices(t, cluster, listenerPortOne) + // Create a gateway + gatewayService, err := libservice.NewGatewayService(context.Background(), libservice.GatewayConfig{ + Kind: "api", + Namespace: namespace, + Name: gatewayName, + }, cluster.Agents[0], listenerPortOne) + require.NoError(t, err) // make sure the gateway/route come online // make sure config entries have been properly created - namespace := getNamespace() - checkGatewayConfigEntry(t, client, "api-gateway", namespace) - checkTCPRouteConfigEntry(t, client, "api-gateway-route", namespace) + checkGatewayConfigEntry(t, client, gatewayName, namespace) + checkTCPRouteConfigEntry(t, client, routeName, namespace) port, err := gatewayService.GetPort(listenerPortOne) require.NoError(t, err) @@ -112,72 +141,36 @@ func conditionStatusIsValue(typeName string, statusValue string, conditions []ap return false } -// TODO this code is just copy pasted from elsewhere, it is likely we will need to modify it some -func createCluster(t *testing.T, ports ...int) *libcluster.Cluster { - opts := libcluster.BuildOptions{ - InjectAutoEncryption: true, - InjectGossipEncryption: true, - AllowHTTPAnyway: true, - } - ctx := libcluster.NewBuildContext(t, opts) - - conf := libcluster.NewConfigBuilder(ctx). - ToAgentConfig(t) - t.Logf("Cluster config:\n%s", conf.JSON) - - configs := []libcluster.Config{*conf} - - cluster, err := libcluster.New(t, configs, ports...) - require.NoError(t, err) - - node := cluster.Agents[0] - client := node.GetClient() - - libcluster.WaitForLeader(t, cluster, client) - libcluster.WaitForMembers(t, client, 1) - - // Default Proxy Settings - ok, err := utils.ApplyDefaultProxySettings(client) - require.NoError(t, err) - require.True(t, ok) - - require.NoError(t, err) - - return cluster -} - -func createGatewayConfigEntry(gatewayName, protocol, namespace string, listenerPort int) *api.APIGatewayConfigEntry { - return &api.APIGatewayConfigEntry{ - Kind: api.APIGateway, - Name: gatewayName, - Listeners: []api.APIGatewayListener{ - { - Name: "listener", - Port: listenerPort, - Protocol: protocol, - }, - }, - Namespace: namespace, - } -} - func checkGatewayConfigEntry(t *testing.T, client *api.Client, gatewayName string, namespace string) { + t.Helper() + require.Eventually(t, func() bool { entry, _, err := client.ConfigEntries().Get(api.APIGateway, gatewayName, &api.QueryOptions{Namespace: namespace}) - require.NoError(t, err) - if entry == nil { + if err != nil { + t.Log("error constructing request", err) return false } + if entry == nil { + t.Log("returned entry is nil") + return false + } + apiEntry := entry.(*api.APIGatewayConfigEntry) return isAccepted(apiEntry.Status.Conditions) }, time.Second*10, time.Second*1) } func checkHTTPRouteConfigEntry(t *testing.T, client *api.Client, routeName string, namespace string) { + t.Helper() + require.Eventually(t, func() bool { entry, _, err := client.ConfigEntries().Get(api.HTTPRoute, routeName, &api.QueryOptions{Namespace: namespace}) - require.NoError(t, err) + if err != nil { + t.Log("error constructing request", err) + return false + } if entry == nil { + t.Log("returned entry is nil") return false } @@ -187,10 +180,16 @@ func checkHTTPRouteConfigEntry(t *testing.T, client *api.Client, routeName strin } func checkTCPRouteConfigEntry(t *testing.T, client *api.Client, routeName string, namespace string) { + t.Helper() + require.Eventually(t, func() bool { entry, _, err := client.ConfigEntries().Get(api.TCPRoute, routeName, &api.QueryOptions{Namespace: namespace}) - require.NoError(t, err) + if err != nil { + t.Log("error constructing request", err) + return false + } if entry == nil { + t.Log("returned entry is nil") return false } @@ -199,43 +198,6 @@ func checkTCPRouteConfigEntry(t *testing.T, client *api.Client, routeName string }, time.Second*10, time.Second*1) } -func createService(t *testing.T, cluster *libcluster.Cluster, serviceOpts *libservice.ServiceOpts, containerArgs []string) libservice.Service { - node := cluster.Agents[0] - client := node.GetClient() - // Create a service and proxy instance - service, _, err := libservice.CreateAndRegisterStaticServerAndSidecar(node, serviceOpts, containerArgs...) - require.NoError(t, err) - - libassert.CatalogServiceExists(t, client, serviceOpts.Name+"-sidecar-proxy", &api.QueryOptions{Namespace: serviceOpts.Namespace}) - libassert.CatalogServiceExists(t, client, serviceOpts.Name, &api.QueryOptions{Namespace: serviceOpts.Namespace}) - - return service -} - -func createServices(t *testing.T, cluster *libcluster.Cluster, ports ...int) (libservice.Service, libservice.Service) { - node := cluster.Agents[0] - client := node.GetClient() - // Create a service and proxy instance - serviceOpts := &libservice.ServiceOpts{ - Name: libservice.StaticServerServiceName, - ID: "static-server", - HTTPPort: 8080, - GRPCPort: 8079, - } - - clientConnectProxy := createService(t, cluster, serviceOpts, nil) - gwCfg := libservice.GatewayConfig{ - Name: "api-gateway", - Kind: "api", - } - - gatewayService, err := libservice.NewGatewayService(context.Background(), gwCfg, cluster.Agents[0], ports...) - require.NoError(t, err) - libassert.CatalogServiceExists(t, client, "api-gateway", nil) - - return clientConnectProxy, gatewayService -} - type checkOptions struct { debug bool statusCode int @@ -244,26 +206,23 @@ type checkOptions struct { // checkRoute, customized version of libassert.RouteEchos to allow for headers/distinguishing between the server instances func checkRoute(t *testing.T, port int, path string, headers map[string]string, expected checkOptions) { - ip := "localhost" + t.Helper() + if expected.testName != "" { t.Log("running " + expected.testName) } - const phrase = "hello" - - failer := func() *retry.Timer { - return &retry.Timer{Timeout: time.Second * 60, Wait: time.Second * 60} - } client := cleanhttp.DefaultClient() - path = strings.TrimPrefix(path, "/") - url := fmt.Sprintf("http://%s:%d/%s", ip, port, path) + url := fmt.Sprintf("http://localhost:%d/%s", port, path) - retry.RunWith(failer(), t, func(r *retry.R) { - t.Logf("making call to %s", url) - reader := strings.NewReader(phrase) + require.Eventually(t, func() bool { + reader := strings.NewReader("hello") req, err := http.NewRequest("POST", url, reader) - require.NoError(t, err) + if err != nil { + t.Log("error constructing request", err) + return false + } headers["content-type"] = "text/plain" for k, v := range headers { @@ -273,39 +232,41 @@ func checkRoute(t *testing.T, port int, path string, headers map[string]string, req.Host = v } } + res, err := client.Do(req) if err != nil { - t.Log(err) - r.Fatal("could not make call to service ", url) + t.Log("error sending request", err) + return false } defer res.Body.Close() body, err := io.ReadAll(res.Body) if err != nil { - r.Fatal("could not read response body ", url) + t.Log("error reading response body", err) + return false } - assert.Equal(t, expected.statusCode, res.StatusCode) if expected.statusCode != res.StatusCode { - r.Fatal("unexpected response code returned") + t.Logf("bad status code - expected: %d, actual: %d", expected.statusCode, res.StatusCode) + return false + } + if expected.debug { + if !strings.Contains(string(body), "debug") { + t.Log("body does not contain 'debug'") + return false + } + } + if !strings.Contains(string(body), "hello") { + t.Log("body does not contain 'hello'") + return false } - // if debug is expected, debug should be in the response body - assert.Equal(t, expected.debug, strings.Contains(string(body), "debug")) - if expected.statusCode != res.StatusCode { - r.Fatal("unexpected response body returned") - } - - if !strings.Contains(string(body), phrase) { - r.Fatal("received an incorrect response ", string(body)) - } - }) + return true + }, time.Second*30, time.Second*1) } func checkRouteError(t *testing.T, ip string, port int, path string, headers map[string]string, expected string) { - failer := func() *retry.Timer { - return &retry.Timer{Timeout: time.Second * 60, Wait: time.Second * 60} - } + t.Helper() client := cleanhttp.DefaultClient() url := fmt.Sprintf("http://%s:%d", ip, port) @@ -314,11 +275,12 @@ func checkRouteError(t *testing.T, ip string, port int, path string, headers map url += "/" + path } - retry.RunWith(failer(), t, func(r *retry.R) { - t.Logf("making call to %s", url) + require.Eventually(t, func() bool { req, err := http.NewRequest("GET", url, nil) - assert.NoError(t, err) - + if err != nil { + t.Log("error constructing request", err) + return false + } for k, v := range headers { req.Header.Set(k, v) @@ -327,10 +289,16 @@ func checkRouteError(t *testing.T, ip string, port int, path string, headers map } } _, err = client.Do(req) - assert.Error(t, err) - - if expected != "" { - assert.ErrorContains(t, err, expected) + if err == nil { + t.Log("client request should have errored, but didn't") + return false } - }) + if expected != "" { + if !strings.Contains(err.Error(), expected) { + t.Logf("expected %q to contain %q", err.Error(), expected) + return false + } + } + return true + }, time.Second*30, time.Second*1) } diff --git a/test/integration/consul-container/test/gateways/http_route_test.go b/test/integration/consul-container/test/gateways/http_route_test.go index bf2745b206..a1a6c5a0e7 100644 --- a/test/integration/consul-container/test/gateways/http_route_test.go +++ b/test/integration/consul-container/test/gateways/http_route_test.go @@ -36,10 +36,25 @@ func TestHTTPRouteFlattening(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } + t.Parallel() // infrastructure set up - listenerPort := 6000 + listenerPort := 6004 + serviceOneHTTPPort := 6005 + serviceOneGRPCPort := 6006 + serviceTwoHTTPPort := 6007 + serviceTwoGRPCPort := 6008 + + serviceOneName := randomName("service", 16) + serviceTwoName := randomName("service", 16) + serviceOneResponseCode := 200 + serviceTwoResponseCode := 418 + gatewayName := randomName("gw", 16) + routeOneName := randomName("route", 16) + routeTwoName := randomName("route", 16) + path1 := "/" + path2 := "/v2" clusterConfig := &libtopology.ClusterConfig{ NumServers: 1, @@ -50,7 +65,13 @@ func TestHTTPRouteFlattening(t *testing.T) { InjectGossipEncryption: true, AllowHTTPAnyway: true, }, - Ports: []int{listenerPort}, + Ports: []int{ + listenerPort, + serviceOneHTTPPort, + serviceOneGRPCPort, + serviceTwoHTTPPort, + serviceTwoGRPCPort, + }, ApplyDefaultProxySettings: true, } @@ -64,40 +85,34 @@ func TestHTTPRouteFlattening(t *testing.T) { require.NoError(t, err) } - service1ResponseCode := 200 - service2ResponseCode := 418 - serviceOne := createService(t, cluster, &libservice.ServiceOpts{ - Name: "service1", - ID: "service1", - HTTPPort: 8080, - GRPCPort: 8079, + _, _, err := libservice.CreateAndRegisterStaticServerAndSidecar(cluster.Agents[0], &libservice.ServiceOpts{ + ID: serviceOneName, + Name: serviceOneName, Namespace: namespace, - }, []string{ - // customizes response code so we can distinguish between which service is responding - "-echo-server-default-params", fmt.Sprintf("status=%d", service1ResponseCode), - }) - serviceTwo := createService(t, cluster, &libservice.ServiceOpts{ - Name: "service2", - ID: "service2", - HTTPPort: 8081, - GRPCPort: 8082, - Namespace: namespace, - }, []string{ - "-echo-server-default-params", fmt.Sprintf("status=%d", service2ResponseCode), + HTTPPort: serviceOneHTTPPort, + GRPCPort: serviceOneGRPCPort, }, + // customizes response code so we can distinguish between which service is responding + "-echo-server-default-params", fmt.Sprintf("status=%d", serviceOneResponseCode), ) + require.NoError(t, err) - gatewayName := randomName("gw", 16) - routeOneName := randomName("route", 16) - routeTwoName := randomName("route", 16) - path1 := "/" - path2 := "/v2" + _, _, err = libservice.CreateAndRegisterStaticServerAndSidecar(cluster.Agents[0], &libservice.ServiceOpts{ + ID: serviceTwoName, + Name: serviceTwoName, + Namespace: namespace, + HTTPPort: serviceTwoHTTPPort, + GRPCPort: serviceTwoGRPCPort, + }, + // customizes response code so we can distinguish between which service is responding + "-echo-server-default-params", fmt.Sprintf("status=%d", serviceTwoResponseCode), + ) + require.NoError(t, err) // write config entries proxyDefaults := &api.ProxyConfigEntry{ - Kind: api.ProxyDefaults, - Name: api.ProxyConfigGlobal, - Namespace: "", // proxy-defaults can only be set in the default namespace + Kind: api.ProxyDefaults, + Name: api.ProxyConfigGlobal, Config: map[string]interface{}{ "protocol": "http", }, @@ -119,8 +134,9 @@ func TestHTTPRouteFlattening(t *testing.T) { } routeOne := &api.HTTPRouteConfigEntry{ - Kind: api.HTTPRoute, - Name: routeOneName, + Kind: api.HTTPRoute, + Name: routeOneName, + Namespace: namespace, Parents: []api.ResourceReference{ { Kind: api.APIGateway, @@ -132,12 +148,11 @@ func TestHTTPRouteFlattening(t *testing.T) { "test.foo", "test.example", }, - Namespace: namespace, Rules: []api.HTTPRouteRule{ { Services: []api.HTTPService{ { - Name: serviceOne.GetServiceName(), + Name: serviceOneName, Namespace: namespace, }, }, @@ -154,8 +169,9 @@ func TestHTTPRouteFlattening(t *testing.T) { } routeTwo := &api.HTTPRouteConfigEntry{ - Kind: api.HTTPRoute, - Name: routeTwoName, + Kind: api.HTTPRoute, + Name: routeTwoName, + Namespace: namespace, Parents: []api.ResourceReference{ { Kind: api.APIGateway, @@ -166,12 +182,11 @@ func TestHTTPRouteFlattening(t *testing.T) { Hostnames: []string{ "test.foo", }, - Namespace: namespace, Rules: []api.HTTPRouteRule{ { Services: []api.HTTPService{ { - Name: serviceTwo.GetServiceName(), + Name: serviceTwoName, Namespace: namespace, }, }, @@ -210,6 +225,7 @@ func TestHTTPRouteFlattening(t *testing.T) { // make sure config entries have been properly created checkGatewayConfigEntry(t, client, gatewayName, namespace) + t.Log("checking route one") checkHTTPRouteConfigEntry(t, client, routeOneName, namespace) checkHTTPRouteConfigEntry(t, client, routeTwoName, namespace) @@ -222,31 +238,31 @@ func TestHTTPRouteFlattening(t *testing.T) { checkRoute(t, gatewayPort, "/v2", map[string]string{ "Host": "test.foo", "x-v2": "v2", - }, checkOptions{statusCode: service2ResponseCode, testName: "service2 header and path"}) + }, checkOptions{statusCode: serviceTwoResponseCode, testName: "service2 header and path"}) checkRoute(t, gatewayPort, "/v2", map[string]string{ "Host": "test.foo", - }, checkOptions{statusCode: service2ResponseCode, testName: "service2 just path match"}) + }, checkOptions{statusCode: serviceTwoResponseCode, testName: "service2 just path match"}) // //v1 path with the header checkRoute(t, gatewayPort, "/check", map[string]string{ "Host": "test.foo", "x-v2": "v2", - }, checkOptions{statusCode: service2ResponseCode, testName: "service2 just header match"}) + }, checkOptions{statusCode: serviceTwoResponseCode, testName: "service2 just header match"}) checkRoute(t, gatewayPort, "/v2/path/value", map[string]string{ "Host": "test.foo", "x-v2": "v2", - }, checkOptions{statusCode: service2ResponseCode, testName: "service2 v2 with path"}) + }, checkOptions{statusCode: serviceTwoResponseCode, testName: "service2 v2 with path"}) // hit service 1 by hitting root path checkRoute(t, gatewayPort, "", map[string]string{ "Host": "test.foo", - }, checkOptions{debug: false, statusCode: service1ResponseCode, testName: "service1 root prefix"}) + }, checkOptions{debug: false, statusCode: serviceOneResponseCode, testName: "service1 root prefix"}) // hit service 1 by hitting v2 path with v1 hostname checkRoute(t, gatewayPort, "/v2", map[string]string{ "Host": "test.example", - }, checkOptions{debug: false, statusCode: service1ResponseCode, testName: "service1, v2 path with v2 hostname"}) + }, checkOptions{debug: false, statusCode: serviceOneResponseCode, testName: "service1, v2 path with v2 hostname"}) } func TestHTTPRoutePathRewrite(t *testing.T) { @@ -257,14 +273,46 @@ func TestHTTPRoutePathRewrite(t *testing.T) { t.Parallel() // infrastructure set up - listenerPort := 6001 + listenerPort := 6009 + fooHTTPPort := 6010 + fooGRPCPort := 6011 + barHTTPPort := 6012 + barGRPCPort := 6013 + + fooName := randomName("foo", 16) + barName := randomName("bar", 16) + gatewayName := randomName("gw", 16) + invalidRouteName := randomName("route", 16) + validRouteName := randomName("route", 16) + // create cluster - cluster := createCluster(t, listenerPort) - client := cluster.Agents[0].GetClient() + clusterConfig := &libtopology.ClusterConfig{ + NumServers: 1, + NumClients: 1, + BuildOpts: &libcluster.BuildOptions{ + Datacenter: "dc1", + InjectAutoEncryption: true, + InjectGossipEncryption: true, + AllowHTTPAnyway: true, + }, + Ports: []int{ + listenerPort, + fooHTTPPort, + fooGRPCPort, + barHTTPPort, + barGRPCPort, + }, + ApplyDefaultProxySettings: true, + } + + cluster, _, _ := libtopology.NewCluster(t, clusterConfig) + client := cluster.APIClient(0) + fooStatusCode := 400 barStatusCode := 201 fooPath := "/v1/foo" barPath := "/v1/bar" + namespace := getNamespace() if namespace != "" { ns := &api.Namespace{Name: namespace} @@ -272,33 +320,32 @@ func TestHTTPRoutePathRewrite(t *testing.T) { require.NoError(t, err) } - fooService := createService(t, cluster, &libservice.ServiceOpts{ - Name: "foo", - ID: "foo", - HTTPPort: 8080, - GRPCPort: 8081, + _, _, err := libservice.CreateAndRegisterStaticServerAndSidecar(cluster.Agents[0], &libservice.ServiceOpts{ + ID: fooName, + Name: fooName, Namespace: namespace, - }, []string{ + HTTPPort: fooHTTPPort, + GRPCPort: fooGRPCPort, + }, // customizes response code so we can distinguish between which service is responding "-echo-debug-path", fooPath, "-echo-server-default-params", fmt.Sprintf("status=%d", fooStatusCode), - }) - barService := createService(t, cluster, &libservice.ServiceOpts{ - Name: "bar", - ID: "bar", - // TODO we can potentially get conflicts if these ports are the same - HTTPPort: 8079, - GRPCPort: 8078, + ) + require.NoError(t, err) + + _, _, err = libservice.CreateAndRegisterStaticServerAndSidecar(cluster.Agents[0], &libservice.ServiceOpts{ + ID: barName, + Name: barName, Namespace: namespace, - }, []string{ + HTTPPort: barHTTPPort, + GRPCPort: barGRPCPort, + }, + // customizes response code so we can distinguish between which service is responding "-echo-debug-path", barPath, "-echo-server-default-params", fmt.Sprintf("status=%d", barStatusCode), - }, ) + require.NoError(t, err) - gatewayName := randomName("gw", 16) - invalidRouteName := randomName("route", 16) - validRouteName := randomName("route", 16) fooUnrewritten := "/foo" barUnrewritten := "/bar" @@ -314,7 +361,18 @@ func TestHTTPRoutePathRewrite(t *testing.T) { require.NoError(t, cluster.ConfigEntryWrite(proxyDefaults)) - apiGateway := createGatewayConfigEntry(gatewayName, "http", namespace, listenerPort) + apiGateway := &api.APIGatewayConfigEntry{ + Kind: api.APIGateway, + Name: gatewayName, + Listeners: []api.APIGatewayListener{ + { + Name: "listener", + Port: listenerPort, + Protocol: "http", + }, + }, + Namespace: namespace, + } fooRoute := &api.HTTPRouteConfigEntry{ Kind: api.HTTPRoute, @@ -339,7 +397,7 @@ func TestHTTPRoutePathRewrite(t *testing.T) { }, Services: []api.HTTPService{ { - Name: fooService.GetServiceName(), + Name: fooName, Namespace: namespace, }, }, @@ -378,7 +436,7 @@ func TestHTTPRoutePathRewrite(t *testing.T) { }, Services: []api.HTTPService{ { - Name: barService.GetServiceName(), + Name: barName, Namespace: namespace, }, }, @@ -445,17 +503,43 @@ func TestHTTPRouteParentRefChange(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } + t.Parallel() // infrastructure set up address := "localhost" - listenerOnePort := 6000 - listenerTwoPort := 6001 + listenerOnePort := 6014 + listenerTwoPort := 6015 + serviceHTTPPort := 6016 + serviceGRPCPort := 6017 - // create cluster and service - cluster := createCluster(t, listenerOnePort, listenerTwoPort) - client := cluster.Agents[0].GetClient() + serviceName := randomName("service", 16) + gatewayOneName := randomName("gw1", 16) + gatewayTwoName := randomName("gw2", 16) + routeName := randomName("route", 16) + + // create cluster + clusterConfig := &libtopology.ClusterConfig{ + NumServers: 1, + NumClients: 1, + BuildOpts: &libcluster.BuildOptions{ + Datacenter: "dc1", + InjectAutoEncryption: true, + InjectGossipEncryption: true, + AllowHTTPAnyway: true, + }, + Ports: []int{ + listenerOnePort, + listenerTwoPort, + serviceHTTPPort, + serviceGRPCPort, + }, + ApplyDefaultProxySettings: true, + } + + cluster, _, _ := libtopology.NewCluster(t, clusterConfig) + client := cluster.APIClient(0) // getNamespace() should always return an empty string in Consul OSS namespace := getNamespace() @@ -465,23 +549,19 @@ func TestHTTPRouteParentRefChange(t *testing.T) { require.NoError(t, err) } - service := createService(t, cluster, &libservice.ServiceOpts{ - Name: "service", - ID: "service", - HTTPPort: 8080, - GRPCPort: 8079, + _, _, err := libservice.CreateAndRegisterStaticServerAndSidecar(cluster.Agents[0], &libservice.ServiceOpts{ + ID: serviceName, + Name: serviceName, Namespace: namespace, - }, []string{}) - - gatewayOneName := randomName("gw1", 16) - gatewayTwoName := randomName("gw2", 16) - routeName := randomName("route", 16) + HTTPPort: serviceHTTPPort, + GRPCPort: serviceGRPCPort, + }) + require.NoError(t, err) // write config entries proxyDefaults := &api.ProxyConfigEntry{ - Kind: api.ProxyDefaults, - Name: api.ProxyConfigGlobal, - Namespace: "", // proxy-defaults can only be set in the default namespace + Kind: api.ProxyDefaults, + Name: api.ProxyConfigGlobal, Config: map[string]interface{}{ "protocol": "http", }, @@ -504,16 +584,7 @@ func TestHTTPRouteParentRefChange(t *testing.T) { Namespace: namespace, } require.NoError(t, cluster.ConfigEntryWrite(gatewayOne)) - require.Eventually(t, func() bool { - entry, _, err := client.ConfigEntries().Get(api.APIGateway, gatewayOneName, &api.QueryOptions{Namespace: namespace}) - assert.NoError(t, err) - if entry == nil { - return false - } - apiEntry := entry.(*api.APIGatewayConfigEntry) - t.Log(entry) - return isAccepted(apiEntry.Status.Conditions) - }, time.Second*10, time.Second*1) + checkGatewayConfigEntry(t, client, gatewayOneName, namespace) // create gateway service gwOneCfg := libservice.GatewayConfig{ @@ -539,19 +610,8 @@ func TestHTTPRouteParentRefChange(t *testing.T) { }, Namespace: namespace, } - require.NoError(t, cluster.ConfigEntryWrite(gatewayTwo)) - - require.Eventually(t, func() bool { - entry, _, err := client.ConfigEntries().Get(api.APIGateway, gatewayTwoName, &api.QueryOptions{Namespace: namespace}) - assert.NoError(t, err) - if entry == nil { - return false - } - apiEntry := entry.(*api.APIGatewayConfigEntry) - t.Log(entry) - return isAccepted(apiEntry.Status.Conditions) - }, time.Second*10, time.Second*1) + checkGatewayConfigEntry(t, client, gatewayTwoName, namespace) // create gateway service gwTwoCfg := libservice.GatewayConfig{ @@ -583,7 +643,7 @@ func TestHTTPRouteParentRefChange(t *testing.T) { { Services: []api.HTTPService{ { - Name: service.GetServiceName(), + Name: serviceName, Namespace: namespace, }, },