From 6f0df20d101bf9b095ac8f63bc2def42b5a3446d Mon Sep 17 00:00:00 2001 From: John Murret Date: Tue, 26 Sep 2023 13:21:47 -0700 Subject: [PATCH] NET-5657 - consul-containers test for explicit upstreams (#18952) * Explicit container test * remove static resources * fix passing serviceBindPorts * WIP * fix explicit upstream test * use my image in CI until dataplane is fixed. * gofmt * fixing reference to v2beta1 in test-containers * WIP * remove bad references * add missing license headers * allow access internal/resource/resourcetest * fix check-allowed-imports to append array items * use preview image for dataplane * revert some inadverntent comment updates in peering_topology * add building local consul-dataplane image to compatibility-tests CI * fix substitution in CI * change upstreams to destinations based on incoming change * fixing use of upstreams in resource update * remove commented out lines and enable envoy concurrency on dataplane. * changes to addess PR feedback * small fixes --------- Co-authored-by: Eric --- .github/workflows/test-integrations.yml | 3 + Makefile | 5 +- .../scripts/check-allowed-imports.sh | 2 +- .../assets/Dockerfile-consul-dataplane | 8 + .../libs/cluster/dataplane.go | 120 ++++++++++ .../multiport/explicit_destination_test.go | 222 ++++++++++++++++++ 6 files changed, 358 insertions(+), 2 deletions(-) create mode 100644 test/integration/consul-container/assets/Dockerfile-consul-dataplane create mode 100644 test/integration/consul-container/libs/cluster/dataplane.go create mode 100644 test/integration/consul-container/test/multiport/explicit_destination_test.go diff --git a/.github/workflows/test-integrations.yml b/.github/workflows/test-integrations.yml index 40e75ac3ed..9dd7d3a3a3 100644 --- a/.github/workflows/test-integrations.yml +++ b/.github/workflows/test-integrations.yml @@ -384,6 +384,7 @@ jobs: contents: read env: ENVOY_VERSION: "1.25.4" + CONSUL_DATAPLANE_IMAGE: "docker.io/hashicorppreview/consul-dataplane:1.3-dev" steps: - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 # NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos. @@ -415,6 +416,8 @@ jobs: - name: Retry Build consul-envoy:target-version image if: steps.buildConsulEnvoyImage.outcome == 'failure' run: docker build -t consul-envoy:target-version --build-arg CONSUL_IMAGE=${{ env.CONSUL_LATEST_IMAGE_NAME }}:local --build-arg ENVOY_VERSION=${{ env.ENVOY_VERSION }} -f ./test/integration/consul-container/assets/Dockerfile-consul-envoy ./test/integration/consul-container/assets + - name: Build consul-dataplane:local image + run: docker build -t consul-dataplane:local --build-arg CONSUL_DATAPLANE_IMAGE=${{ env.CONSUL_DATAPLANE_IMAGE }} -f ./test/integration/consul-container/assets/Dockerfile-consul-dataplane ./test/integration/consul-container/assets - name: Configure GH workaround for ipv6 loopback if: ${{ !endsWith(github.repository, '-enterprise') }} run: | diff --git a/Makefile b/Makefile index 73d381cff5..386088e7b6 100644 --- a/Makefile +++ b/Makefile @@ -66,6 +66,7 @@ UI_BUILD_TAG?=consul-build-ui BUILD_CONTAINER_NAME?=consul-builder CONSUL_IMAGE_VERSION?=latest ENVOY_VERSION?='1.25.4' +CONSUL_DATAPLANE_IMAGE := $(or $(CONSUL_DATAPLANE_IMAGE),"docker.io/hashicorppreview/consul-dataplane:1.3-dev") CONSUL_VERSION?=$(shell cat version/VERSION) @@ -263,7 +264,8 @@ lint-container-test-deps: ## Check that the test-container module only imports a @cd test/integration/consul-container && \ $(CURDIR)/build-support/scripts/check-allowed-imports.sh \ github.com/hashicorp/consul \ - internal/catalog/catalogtest + "internal/catalog/catalogtest" \ + "internal/resource/resourcetest" ##@ Testing @@ -347,6 +349,7 @@ test-compat-integ-setup: dev-docker @docker run --rm -t $(CONSUL_COMPAT_TEST_IMAGE):local consul version @# 'consul-envoy:target-version' is needed by compatibility integ test @docker build -t consul-envoy:target-version --build-arg CONSUL_IMAGE=$(CONSUL_COMPAT_TEST_IMAGE):local --build-arg ENVOY_VERSION=${ENVOY_VERSION} -f ./test/integration/consul-container/assets/Dockerfile-consul-envoy ./test/integration/consul-container/assets + @docker build -t consul-dataplane:local --build-arg CONSUL_DATAPLANE_IMAGE=${CONSUL_DATAPLANE_IMAGE} -f ./test/integration/consul-container/assets/Dockerfile-consul-dataplane ./test/integration/consul-container/assets .PHONY: test-compat-integ test-compat-integ: test-compat-integ-setup ## Test compat integ diff --git a/build-support/scripts/check-allowed-imports.sh b/build-support/scripts/check-allowed-imports.sh index efba156c79..02cd8ffdee 100755 --- a/build-support/scripts/check-allowed-imports.sh +++ b/build-support/scripts/check-allowed-imports.sh @@ -46,7 +46,7 @@ function main { then module_root="$1" else - allowed_packages+="$1" + allowed_packages+=("$1") fi shift esac diff --git a/test/integration/consul-container/assets/Dockerfile-consul-dataplane b/test/integration/consul-container/assets/Dockerfile-consul-dataplane new file mode 100644 index 0000000000..508ac1e96b --- /dev/null +++ b/test/integration/consul-container/assets/Dockerfile-consul-dataplane @@ -0,0 +1,8 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +ARG CONSUL_DATAPLANE_IMAGE + +FROM ${CONSUL_DATAPLANE_IMAGE} as consuldataplane +COPY --from=busybox:uclibc /bin/sh /bin/sh +COPY --from=ghcr.io/tarampampam/curl:latest /bin/curl /bin/curl \ No newline at end of file diff --git a/test/integration/consul-container/libs/cluster/dataplane.go b/test/integration/consul-container/libs/cluster/dataplane.go new file mode 100644 index 0000000000..3e43c290cb --- /dev/null +++ b/test/integration/consul-container/libs/cluster/dataplane.go @@ -0,0 +1,120 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package cluster + +import ( + "context" + "fmt" + "github.com/hashicorp/consul/test/integration/consul-container/libs/utils" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" + "strconv" + "time" +) + +type ConsulDataplaneContainer struct { + ctx context.Context + container testcontainers.Container + ip string + appPort []int + serviceName string + externalAdminPort int + internalAdminPort int +} + +func (g ConsulDataplaneContainer) GetAddr() (string, int) { + return g.ip, g.appPort[0] +} + +// GetAdminAddr returns the external admin port +func (g ConsulDataplaneContainer) GetAdminAddr() (string, int) { + return "localhost", g.externalAdminPort +} + +func (c ConsulDataplaneContainer) Terminate() error { + return TerminateContainer(c.ctx, c.container, true) +} + +func (g ConsulDataplaneContainer) GetStatus() (string, error) { + state, err := g.container.State(g.ctx) + return state.Status, err +} + +func NewConsulDataplane(ctx context.Context, proxyID string, serverAddresses string, grpcPort int, serviceBindPorts []int, + node Agent, containerArgs ...string) (*ConsulDataplaneContainer, error) { + namePrefix := fmt.Sprintf("%s-consul-dataplane-%s", node.GetDatacenter(), proxyID) + containerName := utils.RandName(namePrefix) + + internalAdminPort, err := node.ClaimAdminPort() + if err != nil { + return nil, err + } + + pod := node.GetPod() + if pod == nil { + return nil, fmt.Errorf("node Pod is required") + } + + var ( + appPortStrs []string + adminPortStr = strconv.Itoa(internalAdminPort) + ) + + for _, port := range serviceBindPorts { + appPortStrs = append(appPortStrs, strconv.Itoa(port)) + } + + // expose the app ports and the envoy adminPortStr on the agent container + exposedPorts := make([]string, len(appPortStrs)) + copy(exposedPorts, appPortStrs) + exposedPorts = append(exposedPorts, adminPortStr) + + command := []string{ + "-addresses", serverAddresses, + fmt.Sprintf("-grpc-port=%d", grpcPort), + fmt.Sprintf("-proxy-id=%s", proxyID), + "-proxy-namespace=default", + "-proxy-partition=default", + "-log-level=info", + "-log-json=false", + "-envoy-concurrency=2", + "-tls-disabled", + fmt.Sprintf("-envoy-admin-bind-port=%d", internalAdminPort), + } + + command = append(command, containerArgs...) + + req := testcontainers.ContainerRequest{ + Image: "consul-dataplane:local", + WaitingFor: wait.ForLog("").WithStartupTimeout(60 * time.Second), + AutoRemove: false, + Name: containerName, + Cmd: command, + Env: map[string]string{}, + } + + info, err := LaunchContainerOnNode(ctx, node, req, exposedPorts) + if err != nil { + return nil, err + } + out := &ConsulDataplaneContainer{ + ctx: ctx, + container: info.Container, + ip: info.IP, + serviceName: containerName, + externalAdminPort: info.MappedPorts[adminPortStr].Int(), + internalAdminPort: internalAdminPort, + } + + for _, port := range appPortStrs { + out.appPort = append(out.appPort, info.MappedPorts[port].Int()) + } + + fmt.Printf("NewConsulDataplane: proxyID %s, mapped App Port %d, service bind port %v\n", + proxyID, out.appPort, serviceBindPorts) + fmt.Printf("NewConsulDataplane: proxyID %s, , mapped admin port %d, admin port %d\n", + proxyID, out.externalAdminPort, internalAdminPort) + + return out, nil +} diff --git a/test/integration/consul-container/test/multiport/explicit_destination_test.go b/test/integration/consul-container/test/multiport/explicit_destination_test.go new file mode 100644 index 0000000000..d8b02d0654 --- /dev/null +++ b/test/integration/consul-container/test/multiport/explicit_destination_test.go @@ -0,0 +1,222 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package multiport + +import ( + "context" + "fmt" + pbcatalog "github.com/hashicorp/consul/proto-public/pbcatalog/v2beta1" + pbmesh "github.com/hashicorp/consul/proto-public/pbmesh/v2beta1" + libassert "github.com/hashicorp/consul/test/integration/consul-container/libs/assert" + "github.com/stretchr/testify/require" + "testing" + + rtest "github.com/hashicorp/consul/internal/resource/resourcetest" + "github.com/hashicorp/consul/proto-public/pbresource" + libcluster "github.com/hashicorp/consul/test/integration/consul-container/libs/cluster" + libservice "github.com/hashicorp/consul/test/integration/consul-container/libs/service" + "github.com/hashicorp/consul/test/integration/consul-container/libs/topology" + "github.com/hashicorp/consul/test/integration/consul-container/libs/utils" +) + +// TestMultiportService_Explicit makes sure two services in the same datacenter have connectivity +// with transparent proxy enabled. +// +// Steps: +// - Create a single server cluster. +// - Create the example static-server and sidecar containers, then register them both with Consul +// - Create an example static-client sidecar, then register both the service and sidecar with Consul +// - Make sure a request from static-client to the virtual address (.virtual.consul) returns a +// response from the upstream. +func TestMultiportService_Explicit(t *testing.T) { + t.Parallel() + + cluster := createCluster(t) + followers, err := cluster.Followers() + require.NoError(t, err) + client := pbresource.NewResourceServiceClient(followers[0].GetGRPCConn()) + resourceClient := rtest.NewClient(client) + + serverIP := cluster.Agents[1].GetIP() + clientIP := cluster.Agents[2].GetIP() + + serverService := createServerServicesAndWorkloads(t, resourceClient, serverIP) + createClientResources(t, resourceClient, serverService, clientIP) + + _, clientDataplane := createServices(t, cluster) + + _, port := clientDataplane.GetAddr() + + assertDataplaneContainerState(t, clientDataplane, "running") + libassert.HTTPServiceEchoes(t, "localhost", port, "") + libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), "static-server", "") +} + +// createServices creates the static-client and static-server services with +// transparent proxy enabled. It returns a Service for the static-client. +func createServices(t *testing.T, cluster *libcluster.Cluster) (*libcluster.ConsulDataplaneContainer, *libcluster.ConsulDataplaneContainer) { + n1 := cluster.Agents[1] + + // Create a service and dataplane + serverDataplane, err := createServiceAndDataplane(t, n1, "static-server-workload", "static-server", 8080, 8079, []int{}) + require.NoError(t, err) + + n2 := cluster.Agents[2] + // Create a service and dataplane + clientDataplane, err := createServiceAndDataplane(t, n2, "static-client-workload", "static-client", 8080, 8079, []int{libcluster.ServiceUpstreamLocalBindPort}) + require.NoError(t, err) + + return serverDataplane, clientDataplane +} + +func createServiceAndDataplane(t *testing.T, node libcluster.Agent, proxyID, serviceName string, httpPort, grpcPort int, serviceBindPorts []int) (*libcluster.ConsulDataplaneContainer, error) { + // Do some trickery to ensure that partial completion is correctly torn + // down, but successful execution is not. + var deferClean utils.ResettableDefer + defer deferClean.Execute() + + // Create a service and proxy instance + svc, err := libservice.NewExampleService(context.Background(), serviceName, httpPort, grpcPort, node) + if err != nil { + return nil, err + } + deferClean.Add(func() { + _ = svc.Terminate() + }) + + // Create Consul Dataplane + dp, err := libcluster.NewConsulDataplane(context.Background(), proxyID, "0.0.0.0", 8502, serviceBindPorts, node) + require.NoError(t, err) + deferClean.Add(func() { + _ = dp.Terminate() + }) + + // disable cleanup functions now that we have an object with a Terminate() function + deferClean.Reset() + + return dp, nil +} + +func createServerServicesAndWorkloads(t *testing.T, resourceClient *rtest.Client, ipAddress string) *pbresource.Resource { + serverService := rtest.ResourceID(&pbresource.ID{ + Name: "static-server-service", + Type: pbcatalog.ServiceType, + }).WithData(t, &pbcatalog.Service{ + Workloads: &pbcatalog.WorkloadSelector{Prefixes: []string{"static-server"}}, + Ports: []*pbcatalog.ServicePort{ + {TargetPort: "tcp", Protocol: pbcatalog.Protocol_PROTOCOL_TCP}, + {TargetPort: "mesh", Protocol: pbcatalog.Protocol_PROTOCOL_MESH}, + }, + }).Write(t, resourceClient) + + workloadPortMap := map[string]*pbcatalog.WorkloadPort{ + "tcp": { + Port: 8080, Protocol: pbcatalog.Protocol_PROTOCOL_TCP, + }, + "mesh": { + Port: 20000, Protocol: pbcatalog.Protocol_PROTOCOL_MESH, + }, + } + + rtest.ResourceID(&pbresource.ID{ + Name: "static-server-workload", + Type: pbcatalog.WorkloadType, + }). + WithData(t, &pbcatalog.Workload{ + Addresses: []*pbcatalog.WorkloadAddress{ + {Host: ipAddress}, + }, + Ports: workloadPortMap, + Identity: "static-server-identity", + }). + Write(t, resourceClient) + return serverService +} + +func createClientResources(t *testing.T, resourceClient *rtest.Client, staticServerResource *pbresource.Resource, ipAddress string) { + rtest.ResourceID(&pbresource.ID{ + Name: "static-client-service", + Type: pbcatalog.ServiceType, + }).WithData(t, &pbcatalog.Service{ + Workloads: &pbcatalog.WorkloadSelector{Prefixes: []string{"static-client"}}, + Ports: []*pbcatalog.ServicePort{ + {TargetPort: "tcp", Protocol: pbcatalog.Protocol_PROTOCOL_TCP}, + {TargetPort: "mesh", Protocol: pbcatalog.Protocol_PROTOCOL_MESH}, + }, + }).Write(t, resourceClient) + + workloadPortMap := map[string]*pbcatalog.WorkloadPort{ + "tcp": { + Port: 8080, Protocol: pbcatalog.Protocol_PROTOCOL_TCP, + }, + "mesh": { + Port: 20000, Protocol: pbcatalog.Protocol_PROTOCOL_MESH, + }, + } + + rtest.ResourceID(&pbresource.ID{ + Name: "static-client-workload", + Type: pbcatalog.WorkloadType, + }). + WithData(t, &pbcatalog.Workload{ + Addresses: []*pbcatalog.WorkloadAddress{ + {Host: ipAddress}, + }, + Ports: workloadPortMap, + Identity: "static-client-identity", + }). + Write(t, resourceClient) + + destId := staticServerResource.GetId() + destRef := &pbresource.Reference{ + Type: destId.Type, + Tenancy: destId.Tenancy, + Name: destId.Name, + Section: "", + } + rtest.ResourceID(&pbresource.ID{ + Name: "static-client-upstreams", + Type: pbmesh.DestinationsType, + }). + WithData(t, &pbmesh.Destinations{ + Destinations: []*pbmesh.Destination{ + { + DestinationRef: destRef, + DestinationPort: "tcp", + ListenAddr: &pbmesh.Destination_IpPort{ + IpPort: &pbmesh.IPPortAddress{ + Ip: "0.0.0.0", + Port: libcluster.ServiceUpstreamLocalBindPort, + }, + }, + }, + }, + Workloads: &pbcatalog.WorkloadSelector{ + Prefixes: []string{"static-client"}, + }, + }). + Write(t, resourceClient) +} + +func createCluster(t *testing.T) *libcluster.Cluster { + cluster, _, _ := topology.NewCluster(t, &topology.ClusterConfig{ + NumServers: 3, + BuildOpts: &libcluster.BuildOptions{ + Datacenter: "dc1", + InjectAutoEncryption: true, + InjectGossipEncryption: true, + AllowHTTPAnyway: true, + }, + Cmd: `-hcl=experiments=["resource-apis"] log_level="TRACE"`, + }) + + return cluster +} + +// assertDataplaneContainerState validates service container status +func assertDataplaneContainerState(t *testing.T, dataplane *libcluster.ConsulDataplaneContainer, state string) { + containerStatus, err := dataplane.GetStatus() + require.NoError(t, err) + require.Equal(t, containerStatus, state, fmt.Sprintf("Expected: %s. Got %s", state, containerStatus)) +}