diff --git a/test/integration/consul-container/libs/assert/service.go b/test/integration/consul-container/libs/assert/service.go index 7c6b41461f..1110cfe09d 100644 --- a/test/integration/consul-container/libs/assert/service.go +++ b/test/integration/consul-container/libs/assert/service.go @@ -3,6 +3,8 @@ package assert import ( "fmt" "io" + "net/http" + "regexp" "strings" "testing" "time" @@ -13,6 +15,7 @@ import ( "github.com/hashicorp/consul/api" "github.com/hashicorp/consul/sdk/testutil/retry" libservice "github.com/hashicorp/consul/test/integration/consul-container/libs/service" + "github.com/stretchr/testify/assert" ) const ( @@ -89,6 +92,30 @@ func ServiceLogContains(t *testing.T, service libservice.Service, target string) return strings.Contains(logs, target) } +// AssertFortioName asserts that the fortio service replying at urlbase/debug +// has a `FORTIO_NAME` env variable set. This validates that the client is sending +// traffic to the right envoy proxy. +// +// It retries with timeout defaultHTTPTimeout and wait defaultHTTPWait. +func AssertFortioName(t *testing.T, urlbase string, name string) { + t.Helper() + var fortioNameRE = regexp.MustCompile(("\nFORTIO_NAME=(.+)\n")) + var body []byte + retry.RunWith(&retry.Timer{Timeout: defaultHTTPTimeout, Wait: defaultHTTPWait}, t, func(r *retry.R) { + resp, err := http.Get(fmt.Sprintf("%s/debug?env=dump", urlbase)) + if err != nil { + r.Error(err) + return + } + defer resp.Body.Close() + body, err = io.ReadAll(resp.Body) + require.NoError(t, err) + }) + m := fortioNameRE.FindStringSubmatch(string(body)) + require.GreaterOrEqual(t, len(m), 2) + assert.Equal(t, name, m[1]) +} + // AssertContainerState validates service container status func AssertContainerState(t *testing.T, service libservice.Service, state string) { containerStatus, err := service.GetStatus() diff --git a/test/integration/consul-container/libs/cluster/container.go b/test/integration/consul-container/libs/cluster/container.go index a1de996830..c9ce7792b6 100644 --- a/test/integration/consul-container/libs/cluster/container.go +++ b/test/integration/consul-container/libs/cluster/container.go @@ -514,8 +514,13 @@ func newContainerRequest(config Config, opts containerOpts) (podRequest, consulR "8443/tcp", // Envoy Gateway Listener - "8079/tcp", // Envoy App Listener - "8080/tcp", // Envoy App Listener + "8079/tcp", // Envoy App Listener - grpc port used by static-server + "8078/tcp", // Envoy App Listener - grpc port used by static-server-v1 + "8077/tcp", // Envoy App Listener - grpc port used by static-server-v2 + + "8080/tcp", // Envoy App Listener - http port used by static-server + "8081/tcp", // Envoy App Listener - http port used by static-server-v1 + "8082/tcp", // Envoy App Listener - http port used by static-server-v2 "9998/tcp", // Envoy App Listener "9999/tcp", // Envoy App Listener }, diff --git a/test/integration/consul-container/libs/service/helpers.go b/test/integration/consul-container/libs/service/helpers.go index 703a8b130f..197990f5e4 100644 --- a/test/integration/consul-container/libs/service/helpers.go +++ b/test/integration/consul-container/libs/service/helpers.go @@ -16,9 +16,11 @@ const ( ) type ServiceOpts struct { - Name string - ID string - Meta map[string]string + Name string + ID string + Meta map[string]string + HTTPPort int + GRPCPort int } func CreateAndRegisterStaticServerAndSidecar(node libcluster.Agent, serviceOpts *ServiceOpts) (Service, Service, error) { @@ -32,7 +34,7 @@ func CreateAndRegisterStaticServerAndSidecar(node libcluster.Agent, serviceOpts req := &api.AgentServiceRegistration{ Name: serviceOpts.Name, ID: serviceOpts.ID, - Port: 8080, + Port: serviceOpts.HTTPPort, Connect: &api.AgentServiceConnect{ SidecarService: &api.AgentServiceRegistration{ Proxy: &api.AgentServiceConnectProxyConfig{}, @@ -40,7 +42,7 @@ func CreateAndRegisterStaticServerAndSidecar(node libcluster.Agent, serviceOpts }, Check: &api.AgentServiceCheck{ Name: "Static Server Listening", - TCP: fmt.Sprintf("127.0.0.1:%d", 8080), + TCP: fmt.Sprintf("127.0.0.1:%d", serviceOpts.HTTPPort), Interval: "10s", Status: api.HealthPassing, }, @@ -52,7 +54,7 @@ func CreateAndRegisterStaticServerAndSidecar(node libcluster.Agent, serviceOpts } // Create a service and proxy instance - serverService, err := NewExampleService(context.Background(), StaticServerServiceName, 8080, 8079, node) + serverService, err := NewExampleService(context.Background(), serviceOpts.ID, serviceOpts.HTTPPort, serviceOpts.GRPCPort, node) if err != nil { return nil, nil, err } @@ -60,7 +62,7 @@ func CreateAndRegisterStaticServerAndSidecar(node libcluster.Agent, serviceOpts _ = serverService.Terminate() }) - serverConnectProxy, err := NewConnectService(context.Background(), fmt.Sprintf("%s-sidecar", StaticServerServiceName), serviceOpts.ID, 8080, node) // bindPort not used + serverConnectProxy, err := NewConnectService(context.Background(), fmt.Sprintf("%s-sidecar", StaticServerServiceName), serviceOpts.ID, serviceOpts.HTTPPort, node) // bindPort not used if err != nil { return nil, nil, err } diff --git a/test/integration/consul-container/libs/topology/peering_topology.go b/test/integration/consul-container/libs/topology/peering_topology.go index 917f03ed36..c808a347f4 100644 --- a/test/integration/consul-container/libs/topology/peering_topology.go +++ b/test/integration/consul-container/libs/topology/peering_topology.go @@ -68,9 +68,11 @@ func BasicPeeringTwoClustersSetup( var err error // Create a service and proxy instance serviceOpts := libservice.ServiceOpts{ - Name: libservice.StaticServerServiceName, - ID: "static-server", - Meta: map[string]string{"version": ""}, + Name: libservice.StaticServerServiceName, + ID: "static-server", + Meta: map[string]string{"version": ""}, + HTTPPort: 8080, + GRPCPort: 8079, } serverSidecarService, _, err := libservice.CreateAndRegisterStaticServerAndSidecar(clientNode, &serviceOpts) require.NoError(t, err) @@ -107,6 +109,7 @@ func BasicPeeringTwoClustersSetup( libassert.AssertUpstreamEndpointStatus(t, adminPort, fmt.Sprintf("static-server.default.%s.external", DialingPeerName), "HEALTHY", 1) _, port := clientSidecarService.GetAddr() libassert.HTTPServiceEchoes(t, "localhost", port, "") + libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), "static-server") return &BuiltCluster{ Cluster: acceptingCluster, diff --git a/test/integration/consul-container/test/basic/connect_service_test.go b/test/integration/consul-container/test/basic/connect_service_test.go index 81d9446f37..90a80c84c6 100644 --- a/test/integration/consul-container/test/basic/connect_service_test.go +++ b/test/integration/consul-container/test/basic/connect_service_test.go @@ -1,6 +1,7 @@ package basic import ( + "fmt" "testing" "github.com/stretchr/testify/require" @@ -33,6 +34,7 @@ func TestBasicConnectService(t *testing.T) { libassert.AssertContainerState(t, clientService, "running") libassert.HTTPServiceEchoes(t, "localhost", port, "") + libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), "static-server") } func createCluster(t *testing.T) *libcluster.Cluster { @@ -72,8 +74,10 @@ func createServices(t *testing.T, cluster *libcluster.Cluster) libservice.Servic client := node.GetClient() // Create a service and proxy instance serviceOpts := &libservice.ServiceOpts{ - Name: libservice.StaticServerServiceName, - ID: "static-server", + Name: libservice.StaticServerServiceName, + ID: "static-server", + HTTPPort: 8080, + GRPCPort: 8079, } // Create a service and proxy instance diff --git a/test/integration/consul-container/test/observability/access_logs_test.go b/test/integration/consul-container/test/observability/access_logs_test.go index b8bed2c623..dcedb0de55 100644 --- a/test/integration/consul-container/test/observability/access_logs_test.go +++ b/test/integration/consul-container/test/observability/access_logs_test.go @@ -70,6 +70,7 @@ func TestAccessLogs(t *testing.T) { // Validate Custom JSON require.Eventually(t, func() bool { libassert.HTTPServiceEchoes(t, "localhost", port, "banana") + libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), "static-server") client := libassert.ServiceLogContains(t, clientService, "\"banana_path\":\"/banana\"") server := libassert.ServiceLogContains(t, serverService, "\"banana_path\":\"/banana\"") return client && server @@ -111,6 +112,7 @@ func TestAccessLogs(t *testing.T) { _, port = clientService.GetAddr() require.Eventually(t, func() bool { libassert.HTTPServiceEchoes(t, "localhost", port, "orange") + libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), "static-server") client := libassert.ServiceLogContains(t, clientService, "Orange you glad I didn't say banana: /orange, -") server := libassert.ServiceLogContains(t, serverService, "Orange you glad I didn't say banana: /orange, -") return client && server @@ -137,8 +139,10 @@ func createServices(t *testing.T, cluster *libcluster.Cluster) (libservice.Servi // Create a service and proxy instance serviceOpts := &libservice.ServiceOpts{ - Name: libservice.StaticServerServiceName, - ID: "static-server", + Name: libservice.StaticServerServiceName, + ID: "static-server", + HTTPPort: 8080, + GRPCPort: 8079, } // Create a service and proxy instance diff --git a/test/integration/consul-container/test/peering/rotate_server_and_ca_then_fail_test.go b/test/integration/consul-container/test/peering/rotate_server_and_ca_then_fail_test.go index a0d8655b1b..223effa449 100644 --- a/test/integration/consul-container/test/peering/rotate_server_and_ca_then_fail_test.go +++ b/test/integration/consul-container/test/peering/rotate_server_and_ca_then_fail_test.go @@ -3,6 +3,7 @@ package peering import ( "context" "encoding/pem" + "fmt" "testing" "time" @@ -93,6 +94,7 @@ func TestPeering_RotateServerAndCAThenFail_(t *testing.T) { _, port := clientSidecarService.GetAddr() libassert.HTTPServiceEchoes(t, "localhost", port, "") + libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), "static-server") } testutil.RunStep(t, "rotate exporting cluster's root CA", func(t *testing.T) { @@ -142,6 +144,7 @@ func TestPeering_RotateServerAndCAThenFail_(t *testing.T) { // Connectivity should still be contained _, port := clientSidecarService.GetAddr() libassert.HTTPServiceEchoes(t, "localhost", port, "") + libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), "static-server") verifySidecarHasTwoRootCAs(t, clientSidecarService) }) @@ -163,6 +166,7 @@ func TestPeering_RotateServerAndCAThenFail_(t *testing.T) { _, port := clientSidecarService.GetAddr() libassert.HTTPServiceEchoes(t, "localhost", port, "") + libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), "static-server") }) } diff --git a/test/integration/consul-container/test/upgrade/peer_control_plane_mgw_test.go b/test/integration/consul-container/test/upgrade/peer_control_plane_mgw_test.go index 3d65a179fb..f4112b6f6b 100644 --- a/test/integration/consul-container/test/upgrade/peer_control_plane_mgw_test.go +++ b/test/integration/consul-container/test/upgrade/peer_control_plane_mgw_test.go @@ -110,6 +110,7 @@ func TestPeering_Upgrade_ControlPlane_MGW(t *testing.T) { require.NoError(t, clientSidecarService.Restart()) libassert.AssertUpstreamEndpointStatus(t, adminPort, fmt.Sprintf("static-server.default.%s.external", libtopology.DialingPeerName), "HEALTHY", 1) libassert.HTTPServiceEchoes(t, "localhost", port, "") + libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), "static-server") } for _, tc := range tcs { diff --git a/test/integration/consul-container/test/upgrade/peers_http_test.go b/test/integration/consul-container/test/upgrade/peers_http_test.go index 059f5be3e6..3bf8636314 100644 --- a/test/integration/consul-container/test/upgrade/peers_http_test.go +++ b/test/integration/consul-container/test/upgrade/peers_http_test.go @@ -80,6 +80,7 @@ func TestPeering_UpgradeToTarget_fromLatest(t *testing.T) { require.NoError(t, clientSidecarService.Restart()) libassert.AssertUpstreamEndpointStatus(t, adminPort, fmt.Sprintf("static-server.default.%s.external", libtopology.DialingPeerName), "HEALTHY", 1) libassert.HTTPServiceEchoes(t, "localhost", port, "") + libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), "static-server") } for _, tc := range tcs { diff --git a/test/integration/consul-container/test/upgrade/traffic_management_default_subset_test.go b/test/integration/consul-container/test/upgrade/traffic_management_default_subset_test.go index df170ee40c..5ff574e77e 100644 --- a/test/integration/consul-container/test/upgrade/traffic_management_default_subset_test.go +++ b/test/integration/consul-container/test/upgrade/traffic_management_default_subset_test.go @@ -76,28 +76,26 @@ func TestTrafficManagement_ServiceWithSubsets(t *testing.T) { err := cluster.ConfigEntryWrite(serviceResolver) require.NoError(t, err) - serverService, serverServiceV1, serverServiceV2, clientService := createService(t, cluster) + serverConnectProxy, serverConnectProxyV1, serverConnectProxyV2, clientConnectProxy := createService(t, cluster) - _, port := clientService.GetAddr() - _, adminPort := clientService.GetAdminAddr() - _, serverAdminPort := serverService.GetAdminAddr() - _, serverAdminPortV1 := serverServiceV1.GetAdminAddr() - _, serverAdminPortV2 := serverServiceV2.GetAdminAddr() + _, port := clientConnectProxy.GetAddr() + _, adminPort := clientConnectProxy.GetAdminAddr() + _, serverAdminPort := serverConnectProxy.GetAdminAddr() + _, serverAdminPortV1 := serverConnectProxyV1.GetAdminAddr() + _, serverAdminPortV2 := serverConnectProxyV2.GetAdminAddr() // validate client and proxy is up and running - libassert.AssertContainerState(t, clientService, "running") - - // TO-DO: static-client upstream should be able to connect to static-server-v2 via upstream s2 + libassert.AssertContainerState(t, clientConnectProxy, "running") libassert.HTTPServiceEchoes(t, "localhost", port, "") libassert.AssertUpstreamEndpointStatus(t, adminPort, "v2.static-server.default", "HEALTHY", 1) // Upgrade cluster, restart sidecars then begin service traffic validation require.NoError(t, cluster.StandardUpgrade(t, context.Background(), tc.targetVersion)) - require.NoError(t, clientService.Restart()) - require.NoError(t, serverService.Restart()) - require.NoError(t, serverServiceV1.Restart()) - require.NoError(t, serverServiceV2.Restart()) + require.NoError(t, clientConnectProxy.Restart()) + require.NoError(t, serverConnectProxy.Restart()) + require.NoError(t, serverConnectProxyV1.Restart()) + require.NoError(t, serverConnectProxyV2.Restart()) // POST upgrade validation; repeat client & proxy validation libassert.HTTPServiceEchoes(t, "localhost", port, "") @@ -129,7 +127,8 @@ func TestTrafficManagement_ServiceWithSubsets(t *testing.T) { libassert.AssertEnvoyPresentsCertURI(t, serverAdminPortV1, "static-server") libassert.AssertEnvoyPresentsCertURI(t, serverAdminPortV2, "static-server") - // TO-DO: restart envoy sidecar and validate traffic management + // static-client upstream should connect to static-server-v2 because the default subset value is to v2 set in the service resolver + libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), "static-server-v2") } for _, tc := range tcs { @@ -146,35 +145,41 @@ func createService(t *testing.T, cluster *libcluster.Cluster) (libservice.Servic client := node.GetClient() serviceOpts := &libservice.ServiceOpts{ - Name: libservice.StaticServerServiceName, - ID: "static-server", + Name: libservice.StaticServerServiceName, + ID: "static-server", + HTTPPort: 8080, + GRPCPort: 8079, } - _, serverService, err := libservice.CreateAndRegisterStaticServerAndSidecar(node, serviceOpts) + _, serverConnectProxy, err := libservice.CreateAndRegisterStaticServerAndSidecar(node, serviceOpts) libassert.CatalogServiceExists(t, client, "static-server") require.NoError(t, err) serviceOptsV1 := &libservice.ServiceOpts{ - Name: libservice.StaticServerServiceName, - ID: "static-server-v1", - Meta: map[string]string{"version": "v1"}, + Name: libservice.StaticServerServiceName, + ID: "static-server-v1", + Meta: map[string]string{"version": "v1"}, + HTTPPort: 8081, + GRPCPort: 8078, } - _, serverServiceV1, err := libservice.CreateAndRegisterStaticServerAndSidecar(node, serviceOptsV1) + _, serverConnectProxyV1, err := libservice.CreateAndRegisterStaticServerAndSidecar(node, serviceOptsV1) libassert.CatalogServiceExists(t, client, "static-server") require.NoError(t, err) serviceOptsV2 := &libservice.ServiceOpts{ - Name: libservice.StaticServerServiceName, - ID: "static-server-v2", - Meta: map[string]string{"version": "v2"}, + Name: libservice.StaticServerServiceName, + ID: "static-server-v2", + Meta: map[string]string{"version": "v2"}, + HTTPPort: 8082, + GRPCPort: 8077, } - _, serverServiceV2, err := libservice.CreateAndRegisterStaticServerAndSidecar(node, serviceOptsV2) + _, serverConnectProxyV2, err := libservice.CreateAndRegisterStaticServerAndSidecar(node, serviceOptsV2) libassert.CatalogServiceExists(t, client, "static-server") require.NoError(t, err) // Create a client proxy instance with the server as an upstream - clientService, err := libservice.CreateAndRegisterStaticClientSidecar(node, "", false) + clientConnectProxy, err := libservice.CreateAndRegisterStaticClientSidecar(node, "", false) require.NoError(t, err) libassert.CatalogServiceExists(t, client, fmt.Sprintf("%s-sidecar-proxy", libservice.StaticClientServiceName)) - return serverService, serverServiceV1, serverServiceV2, clientService + return serverConnectProxy, serverConnectProxyV1, serverConnectProxyV2, clientConnectProxy }