switch all client nodes in dc2 to dataplane [NET-4299] (#18608)

test-integ/peering_commontopo/ac5_2_pq_failover_test.go

@@ -86,7 +86,10 @@ func (s *ac5_2PQFailoverSuite) setupDC(ct *commonTopo, clu, peerClu *topology.Cl
 		Service: NewFortioServiceWithDefaults(
 			clu.Datacenter,
 			serverSID,
-			nil,
+			func(s *topology.Service) {
+				s.EnvoyAdminPort = 0
+				s.DisableServiceMesh = true
+			},
 		),
 		Exports: []api.ServiceConsumer{{Peer: peer}},
 	}
@@ -149,7 +152,10 @@ func (s *ac5_2PQFailoverSuite) setupDC3(ct *commonTopo, clu, peer1, peer2 *topol
 		Service: NewFortioServiceWithDefaults(
 			clu.Datacenter,
 			serverSID,
-			nil,
+			func(s *topology.Service) {
+				s.EnvoyAdminPort = 0
+				s.DisableServiceMesh = true
+			},
 		),
 		Exports: func() []api.ServiceConsumer {
 			var consumers []api.ServiceConsumer

test-integ/peering_commontopo/ac7_1_rotate_gw_test.go

@@ -141,8 +141,12 @@ func (s *suiteRotateGW) setup(t *testing.T, ct *commonTopo) {
 
 	// add a second mesh gateway "new"
 	s.newMGWNodeName = fmt.Sprintf("new-%s-default-mgw", clu.Name)
+	nodeKind := topology.NodeKindClient
+	if clu.Datacenter == agentlessDC {
+		nodeKind = topology.NodeKindDataplane
+	}
 	clu.Nodes = append(clu.Nodes, newTopologyMeshGatewaySet(
-		topology.NodeKindClient,
+		nodeKind,
 		"default",
 		s.newMGWNodeName,
 		1,

test-integ/peering_commontopo/asserter.go

@@ -41,6 +41,7 @@ type asserter struct {
 type sprawlLite interface {
 	HTTPClientForCluster(clusterName string) (*http.Client, error)
 	APIClientForNode(clusterName string, nid topology.NodeID, token string) (*api.Client, error)
+	APIClientForCluster(clusterName string, token string) (*api.Client, error)
 	Topology() *topology.Topology
 }
 
@@ -58,18 +59,12 @@ func (a *asserter) mustGetHTTPClient(t *testing.T, cluster string) *http.Client
 }
 
 func (a *asserter) mustGetAPIClient(t *testing.T, cluster string) *api.Client {
-	cl, err := a.apiClientFor(cluster)
+	clu := a.sp.Topology().Clusters[cluster]
+	cl, err := a.sp.APIClientForCluster(clu.Name, "")
 	require.NoError(t, err)
 	return cl
 }
 
-func (a *asserter) apiClientFor(cluster string) (*api.Client, error) {
-	clu := a.sp.Topology().Clusters[cluster]
-	// TODO: this always goes to the first client, but we might want to balance this
-	cl, err := a.sp.APIClientForNode(cluster, clu.FirstClient().ID(), "")
-	return cl, err
-}
-
 // httpClientFor returns a pre-configured http.Client that proxies requests
 // through the embedded squid instance in each LAN.
 //

test-integ/peering_commontopo/commontopo.go

@@ -52,6 +52,8 @@ type commonTopo struct {
 	services map[string]map[topology.ServiceID]struct{}
 }
 
+const agentlessDC = "dc2"
+
 func NewCommonTopo(t *testing.T) *commonTopo {
 	t.Helper()
 
@@ -84,11 +86,9 @@ func NewCommonTopo(t *testing.T) *commonTopo {
 	peerings = append(peerings, addPeerings(dc1, dc3)...)
 	peerings = append(peerings, addPeerings(dc2, dc3)...)
 
-	addMeshGateways(dc1, topology.NodeKindClient)
+	addMeshGateways(dc1)
-	addMeshGateways(dc2, topology.NodeKindClient)
+	addMeshGateways(dc2)
-	addMeshGateways(dc3, topology.NodeKindClient)
+	addMeshGateways(dc3)
-	// TODO: consul-topology doesn't support this yet
-	// addMeshGateways(dc2, topology.NodeKindDataplane)
 
 	setupGlobals(dc1)
 	setupGlobals(dc2)
@@ -131,7 +131,7 @@ func (ct *commonTopo) postLaunchChecks(t *testing.T) {
 	)
 
 	// check that exports line up as expected
-	for _, clu := range ct.Sprawl.Config().Clusters {
+	for _, clu := range ct.Sprawl.Topology().Clusters {
 		// expected exports per peer
 		type key struct {
 			peer      string
@@ -191,9 +191,6 @@ func LocalPeerName(clu *topology.Cluster, partition string) string {
 type serviceExt struct {
 	*topology.Service
 
-	// default NodeKindClient
-	NodeKind topology.NodeKind
-
 	Exports    []api.ServiceConsumer
 	Config     *api.ServiceConfigEntry
 	Intentions *api.ServiceIntentionsConfigEntry
@@ -227,8 +224,15 @@ func (ct *commonTopo) AddServiceNode(clu *topology.Cluster, svc serviceExt) *top
 		return n
 	}
 
+	nodeKind := topology.NodeKindClient
+	// TODO: bug in deployer somewhere; it should guard against a KindDataplane node with
+	// DisableServiceMesh services on it; dataplane is only for service-mesh
+	if !svc.DisableServiceMesh && clu.Datacenter == agentlessDC {
+		nodeKind = topology.NodeKindDataplane
+	}
+
 	node := &topology.Node{
-		Kind:      topology.NodeKindClient,
+		Kind:      nodeKind,
 		Name:      serviceHostnameString(clu.Datacenter, svc.ID),
 		Partition: svc.ID.Partition,
 		Addresses: []*topology.Address{
@@ -239,9 +243,6 @@ func (ct *commonTopo) AddServiceNode(clu *topology.Cluster, svc serviceExt) *top
 		},
 		Cluster: clusterName,
 	}
-	if svc.NodeKind != "" {
-		node.Kind = svc.NodeKind
-	}
 	clu.Nodes = append(clu.Nodes, node)
 
 	// Export if necessary
@@ -265,7 +266,7 @@ func (ct *commonTopo) AddServiceNode(clu *topology.Cluster, svc serviceExt) *top
 }
 
 func (ct *commonTopo) APIClientForCluster(t *testing.T, clu *topology.Cluster) *api.Client {
-	cl, err := ct.Sprawl.APIClientForNode(clu.Name, clu.FirstClient().ID(), "")
+	cl, err := ct.Sprawl.APIClientForCluster(clu.Name, "")
 	require.NoError(t, err)
 	return cl
 }
@@ -372,10 +373,14 @@ func setupGlobals(clu *topology.Cluster) {
 
 // addMeshGateways adds a mesh gateway for every partition in the cluster.
 // Assumes that the LAN network name is equal to datacenter name.
-func addMeshGateways(c *topology.Cluster, kind topology.NodeKind) {
+func addMeshGateways(c *topology.Cluster) {
+	nodeKind := topology.NodeKindClient
+	if c.Datacenter == agentlessDC {
+		nodeKind = topology.NodeKindDataplane
+	}
 	for _, p := range c.Partitions {
 		c.Nodes = topology.MergeSlices(c.Nodes, newTopologyMeshGatewaySet(
-			kind,
+			nodeKind,
 			p.Name,
 			fmt.Sprintf("%s-%s-mgw", c.Name, p.Name),
 			1,

testing/deployer/sprawl/catalog.go

@@ -318,6 +318,18 @@ func serviceToCatalogRegistration(
 			Address: node.LocalAddress(),
 		},
 	}
+	if svc.IsMeshGateway {
+		reg.Service.Kind = api.ServiceKindMeshGateway
+		reg.Service.Proxy = &api.AgentServiceConnectProxyConfig{
+			Config: map[string]interface{}{
+				"envoy_gateway_no_default_bind":       true,
+				"envoy_gateway_bind_tagged_addresses": true,
+			},
+			MeshGateway: api.MeshGatewayConfig{
+				Mode: api.MeshGatewayModeLocal,
+			},
+		}
+	}
 	if node.HasPublicAddress() {
 		reg.TaggedAddresses = map[string]string{
 			"lan":      node.LocalAddress(),
@@ -325,6 +337,26 @@ func serviceToCatalogRegistration(
 			"wan":      node.PublicAddress(),
 			"wan_ipv4": node.PublicAddress(),
 		}
+		// TODO: not sure what the difference is between these, but with just the
+		// top-level set, it appeared to not get set in either :/
+		reg.Service.TaggedAddresses = map[string]api.ServiceAddress{
+			"lan": {
+				Address: node.LocalAddress(),
+				Port:    svc.Port,
+			},
+			"lan_ipv4": {
+				Address: node.LocalAddress(),
+				Port:    svc.Port,
+			},
+			"wan": {
+				Address: node.PublicAddress(),
+				Port:    svc.Port,
+			},
+			"wan_ipv4": {
+				Address: node.PublicAddress(),
+				Port:    svc.Port,
+			},
+		}
 	}
 	if cluster.Enterprise {
 		reg.Partition = svc.ID.Partition

testing/deployer/sprawl/internal/build/docker.go

@@ -23,9 +23,10 @@ COPY --from=0 /bin/consul /bin/consul
 
 // FROM hashicorp/consul-dataplane:latest
 // COPY --from=busybox:uclibc /bin/sh /bin/sh
+// TODO: busybox:latest doesn't work, see https://hashicorp.slack.com/archives/C03EUN3QF1C/p1691784078972959
 const dockerfileDataplane = `
 ARG DATAPLANE_IMAGE
-FROM busybox:latest
+FROM busybox:1.34
 FROM ${DATAPLANE_IMAGE}
 COPY --from=0 /bin/busybox /bin/busybox
 USER 0:0

testing/deployer/sprawl/internal/tfgen/agent.go

@@ -13,14 +13,14 @@ import (
 	"github.com/hashicorp/consul/testing/deployer/topology"
 )
 
-func (g *Generator) generateAgentHCL(node *topology.Node) (string, error) {
+func (g *Generator) generateAgentHCL(node *topology.Node) string {
 	if !node.IsAgent() {
-		return "", fmt.Errorf("not an agent")
+		panic("generateAgentHCL only applies to agents")
 	}
 
 	cluster, ok := g.topology.Clusters[node.Cluster]
 	if !ok {
-		return "", fmt.Errorf("no such cluster: %s", node.Cluster)
+		panic(fmt.Sprintf("no such cluster: %s", node.Cluster))
 	}
 
 	var b HCLBuilder
@@ -167,7 +167,7 @@ func (g *Generator) generateAgentHCL(node *topology.Node) (string, error) {
 		}
 	}
 
-	return b.String(), nil
+	return b.String()
 }
 
 type HCLBuilder struct {

testing/deployer/sprawl/internal/tfgen/dns.go

@@ -9,7 +9,6 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
-	"text/template"
 
 	"github.com/hashicorp/consul/testing/deployer/topology"
 	"github.com/hashicorp/consul/testing/deployer/util"
@@ -179,5 +178,3 @@ server IN A %s ; Consul server
 
 	return buf.Bytes()
 }
-
-var tfCorednsT = template.Must(template.ParseFS(content, "templates/container-coredns.tf.tmpl"))

testing/deployer/sprawl/internal/tfgen/docker.go

@@ -11,6 +11,9 @@ import (
 var invalidResourceName = regexp.MustCompile(`[^a-z0-9-]+`)
 
 func DockerImageResourceName(image string) string {
+	if image == "" {
+		panic(`image must not be ""`)
+	}
 	return invalidResourceName.ReplaceAllLiteralString(image, "-")
 }
 

testing/deployer/sprawl/internal/tfgen/nodes.go

@@ -5,9 +5,6 @@ package tfgen
 
 import (
 	"fmt"
-	"sort"
-	"strconv"
-	"text/template"
 
 	"github.com/hashicorp/consul/testing/deployer/topology"
 )
@@ -22,32 +19,6 @@ type terraformPod struct {
 	DockerNetworkName string
 }
 
-type terraformConsulAgent struct {
-	terraformPod
-	ImageResource     string
-	HCL               string
-	EnterpriseLicense string
-	Env               []string
-}
-
-type terraformMeshGatewayService struct {
-	terraformPod
-	EnvoyImageResource string
-	Service            *topology.Service
-	Command            []string
-}
-
-type terraformService struct {
-	terraformPod
-	AppImageResource       string
-	EnvoyImageResource     string // agentful
-	DataplaneImageResource string // agentless
-	Service                *topology.Service
-	Env                    []string
-	Command                []string
-	EnvoyCommand           []string // agentful
-}
-
 func (g *Generator) generateNodeContainers(
 	step Step,
 	cluster *topology.Cluster,
@@ -82,156 +53,99 @@ func (g *Generator) generateNodeContainers(
 	}
 	pod.DockerNetworkName = net.DockerName
 
-	var (
+	containers := []Resource{}
-		containers []Resource
-	)
 
 	if node.IsAgent() {
-		agentHCL, err := g.generateAgentHCL(node)
-		if err != nil {
-			return nil, err
-		}
-
-		agent := terraformConsulAgent{
-			terraformPod:      pod,
-			ImageResource:     DockerImageResourceName(node.Images.Consul),
-			HCL:               agentHCL,
-			EnterpriseLicense: g.license,
-			Env:               node.AgentEnv,
-		}
-
 		switch {
 		case node.IsServer() && step.StartServers(),
 			!node.IsServer() && step.StartAgents():
-			containers = append(containers, Eval(tfConsulT, &agent))
+			containers = append(containers, Eval(tfConsulT, struct {
+				terraformPod
+				ImageResource     string
+				HCL               string
+				EnterpriseLicense string
+			}{
+				terraformPod:      pod,
+				ImageResource:     DockerImageResourceName(node.Images.Consul),
+				HCL:               g.generateAgentHCL(node),
+				EnterpriseLicense: g.license,
+			}))
 		}
 	}
 
+	svcContainers := []Resource{}
 	for _, svc := range node.SortedServices() {
-		if svc.IsMeshGateway {
+		token := g.sec.ReadServiceToken(node.Cluster, svc.ID)
-			if node.Kind == topology.NodeKindDataplane {
+		switch {
-				panic("NOT READY YET")
+		case svc.IsMeshGateway && !node.IsDataplane():
-			}
+			svcContainers = append(svcContainers, Eval(tfMeshGatewayT, struct {
-			gw := terraformMeshGatewayService{
+				terraformPod
+				ImageResource string
+				Enterprise    bool
+				Service       *topology.Service
+				Token         string
+			}{
 				terraformPod:  pod,
-				EnvoyImageResource: DockerImageResourceName(node.Images.EnvoyConsulImage()),
+				ImageResource: DockerImageResourceName(node.Images.EnvoyConsulImage()),
+				Enterprise:    cluster.Enterprise,
 				Service:       svc,
-				Command: []string{
+				Token:         token,
-					"consul", "connect", "envoy",
+			}))
-					"-register",
+		case svc.IsMeshGateway && node.IsDataplane():
-					"-mesh-gateway",
+			svcContainers = append(svcContainers, Eval(tfMeshGatewayDataplaneT, &struct {
-				},
+				terraformPod
-			}
+				ImageResource string
-			if token := g.sec.ReadServiceToken(node.Cluster, svc.ID); token != "" {
+				Enterprise    bool
-				gw.Command = append(gw.Command, "-token", token)
+				Service       *topology.Service
-			}
+				Token         string
-			if cluster.Enterprise {
+			}{
-				gw.Command = append(gw.Command,
-					"-partition",
-					svc.ID.Partition,
-				)
-			}
-			gw.Command = append(gw.Command,
-				"-address",
-				`{{ GetInterfaceIP \"eth0\" }}:`+strconv.Itoa(svc.Port),
-				"-wan-address",
-				`{{ GetInterfaceIP \"eth1\" }}:`+strconv.Itoa(svc.Port),
-			)
-			gw.Command = append(gw.Command,
-				"-grpc-addr", "http://127.0.0.1:8502",
-				"-admin-bind",
-				// for demo purposes
-				"0.0.0.0:"+strconv.Itoa(svc.EnvoyAdminPort),
-				"--",
-				"-l",
-				"trace",
-			)
-			if step.StartServices() {
-				containers = append(containers, Eval(tfMeshGatewayT, &gw))
-			}
-		} else {
-			tfsvc := terraformService{
 				terraformPod:  pod,
-				AppImageResource: DockerImageResourceName(svc.Image),
+				ImageResource: DockerImageResourceName(node.Images.LocalDataplaneImage()),
+				Enterprise:    cluster.Enterprise,
 				Service:       svc,
-				Command:          svc.Command,
+				Token:         token,
+			}))
+
+		case !svc.IsMeshGateway:
+			svcContainers = append(svcContainers, Eval(tfAppT, struct {
+				terraformPod
+				ImageResource string
+				Service       *topology.Service
+			}{
+				terraformPod:  pod,
+				ImageResource: DockerImageResourceName(svc.Image),
+				Service:       svc,
+			}))
+
+			if svc.DisableServiceMesh {
+				break
 			}
-			tfsvc.Env = append(tfsvc.Env, svc.Env...)
+
+			tmpl := tfAppSidecarT
+			var img string
+			if node.IsDataplane() {
+				tmpl = tfAppDataplaneT
+				img = DockerImageResourceName(node.Images.LocalDataplaneImage())
+			} else {
+				img = DockerImageResourceName(node.Images.EnvoyConsulImage())
+			}
+			svcContainers = append(svcContainers, Eval(tmpl, struct {
+				terraformPod
+				ImageResource string
+				Service       *topology.Service
+				Token         string
+				Enterprise    bool
+			}{
+				terraformPod:  pod,
+				ImageResource: img,
+				Service:       svc,
+				Token:         token,
+				Enterprise:    cluster.Enterprise,
+			}))
+		}
+
 		if step.StartServices() {
-				containers = append(containers, Eval(tfAppT, &tfsvc))
+			containers = append(containers, svcContainers...)
-			}
-
-			setenv := func(k, v string) {
-				tfsvc.Env = append(tfsvc.Env, k+"="+v)
-			}
-
-			if !svc.DisableServiceMesh {
-				if node.IsDataplane() {
-					tfsvc.DataplaneImageResource = DockerImageResourceName(node.Images.LocalDataplaneImage())
-					tfsvc.EnvoyImageResource = ""
-					tfsvc.EnvoyCommand = nil
-					// --- REQUIRED ---
-					setenv("DP_CONSUL_ADDRESSES", "server."+node.Cluster+"-consulcluster.lan")
-					setenv("DP_SERVICE_NODE_NAME", node.PodName())
-					setenv("DP_PROXY_SERVICE_ID", svc.ID.Name+"-sidecar-proxy")
-				} else {
-					tfsvc.DataplaneImageResource = ""
-					tfsvc.EnvoyImageResource = DockerImageResourceName(node.Images.EnvoyConsulImage())
-					tfsvc.EnvoyCommand = []string{
-						"consul", "connect", "envoy",
-						"-sidecar-for", svc.ID.Name,
-					}
-				}
-				if cluster.Enterprise {
-					if node.IsDataplane() {
-						setenv("DP_SERVICE_NAMESPACE", svc.ID.Namespace)
-						setenv("DP_SERVICE_PARTITION", svc.ID.Partition)
-					} else {
-						tfsvc.EnvoyCommand = append(tfsvc.EnvoyCommand,
-							"-partition",
-							svc.ID.Partition,
-							"-namespace",
-							svc.ID.Namespace,
-						)
-					}
-				}
-				if token := g.sec.ReadServiceToken(node.Cluster, svc.ID); token != "" {
-					if node.IsDataplane() {
-						setenv("DP_CREDENTIAL_TYPE", "static")
-						setenv("DP_CREDENTIAL_STATIC_TOKEN", token)
-					} else {
-						tfsvc.EnvoyCommand = append(tfsvc.EnvoyCommand, "-token", token)
-					}
-				}
-				if node.IsDataplane() {
-					setenv("DP_ENVOY_ADMIN_BIND_ADDRESS", "0.0.0.0") // for demo purposes
-					setenv("DP_ENVOY_ADMIN_BIND_PORT", "19000")
-					setenv("DP_LOG_LEVEL", "trace")
-
-					setenv("DP_CA_CERTS", "/consul/config/certs/consul-agent-ca.pem")
-					setenv("DP_CONSUL_GRPC_PORT", "8503")
-					setenv("DP_TLS_SERVER_NAME", "server."+node.Datacenter+".consul")
-				} else {
-					tfsvc.EnvoyCommand = append(tfsvc.EnvoyCommand,
-						"-grpc-addr", "http://127.0.0.1:8502",
-						"-admin-bind",
-						// for demo purposes
-						"0.0.0.0:"+strconv.Itoa(svc.EnvoyAdminPort),
-						"--",
-						"-l",
-						"trace",
-					)
-				}
-				if step.StartServices() {
-					sort.Strings(tfsvc.Env)
-
-					if node.IsDataplane() {
-						containers = append(containers, Eval(tfAppDataplaneT, &tfsvc))
-					} else {
-						containers = append(containers, Eval(tfAppSidecarT, &tfsvc))
-					}
-				}
-			}
 		}
 	}
 
@@ -243,10 +157,3 @@ func (g *Generator) generateNodeContainers(
 
 	return containers, nil
 }
-
-var tfPauseT = template.Must(template.ParseFS(content, "templates/container-pause.tf.tmpl"))
-var tfConsulT = template.Must(template.ParseFS(content, "templates/container-consul.tf.tmpl"))
-var tfMeshGatewayT = template.Must(template.ParseFS(content, "templates/container-mgw.tf.tmpl"))
-var tfAppT = template.Must(template.ParseFS(content, "templates/container-app.tf.tmpl"))
-var tfAppSidecarT = template.Must(template.ParseFS(content, "templates/container-app-sidecar.tf.tmpl"))
-var tfAppDataplaneT = template.Must(template.ParseFS(content, "templates/container-app-dataplane.tf.tmpl"))

testing/deployer/sprawl/internal/tfgen/proxy.go

@@ -7,7 +7,6 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
-	"text/template"
 
 	"github.com/hashicorp/consul/testing/deployer/topology"
 	"github.com/hashicorp/consul/testing/deployer/util"
@@ -86,5 +85,3 @@ func (g *Generator) getForwardProxyContainer(
 
 	return Eval(tfForwardProxyT, &proxy)
 }
-
-var tfForwardProxyT = template.Must(template.ParseFS(content, "templates/container-proxy.tf.tmpl"))

testing/deployer/sprawl/internal/tfgen/templates/container-app-dataplane.tf.tmpl

@@ -1,7 +1,7 @@
 resource "docker_container" "{{.Node.DockerName}}-{{.Service.ID.TFString}}-sidecar" {
 	name = "{{.Node.DockerName}}-{{.Service.ID.TFString}}-sidecar"
     network_mode = "container:${docker_container.{{.PodName}}.id}"
-    image        = docker_image.{{.DataplaneImageResource}}.latest
+    image        = docker_image.{{.ImageResource}}.latest
     restart      = "on-failure"
 
 {{- range $k, $v := .Labels }}
@@ -18,9 +18,24 @@ resource "docker_container" "{{.Node.DockerName}}-{{.Service.ID.TFString}}-sidec
   }
 
   env = [
-{{- range .Env }}
+			"DP_CONSUL_ADDRESSES=server.{{.Node.Cluster}}-consulcluster.lan",
-      "{{.}}",
+			"DP_SERVICE_NODE_NAME={{.Node.PodName}}",
-{{- end}}
+			"DP_PROXY_SERVICE_ID={{.Service.ID.Name}}-sidecar-proxy",
+    {{ if .Enterprise }}
+      "DP_SERVICE_NAMESPACE={{.Service.ID.Namespace}}",
+      "DP_SERVICE_PARTITION={{.Service.ID.Partition}}",
+    {{ end }}
+    {{ if .Token }}
+      "DP_CREDENTIAL_TYPE=static",
+      "DP_CREDENTIAL_STATIC_TOKEN={{.Token}}",
+    {{ end }}
+      // for demo purposes
+			"DP_ENVOY_ADMIN_BIND_ADDRESS=0.0.0.0",
+			"DP_ENVOY_ADMIN_BIND_PORT=19000",
+			"DP_LOG_LEVEL=trace",
+			"DP_CA_CERTS=/consul/config/certs/consul-agent-ca.pem",
+			"DP_CONSUL_GRPC_PORT=8503",
+			"DP_TLS_SERVER_NAME=server.{{.Node.Datacenter}}.consul",
   ]
 
   command = [

testing/deployer/sprawl/internal/tfgen/templates/container-app-sidecar.tf.tmpl

@@ -1,7 +1,7 @@
 resource "docker_container" "{{.Node.DockerName}}-{{.Service.ID.TFString}}-sidecar" {
 	name = "{{.Node.DockerName}}-{{.Service.ID.TFString}}-sidecar"
     network_mode = "container:${docker_container.{{.PodName}}.id}"
-    image        = docker_image.{{.EnvoyImageResource}}.latest
+    image        = docker_image.{{.ImageResource}}.latest
     restart  = "on-failure"
 
 {{- range $k, $v := .Labels }}
@@ -17,15 +17,21 @@ resource "docker_container" "{{.Node.DockerName}}-{{.Service.ID.TFString}}-sidec
     read_only      = true
   }
 
-  env = [
-{{- range .Env }}
-      "{{.}}",
-{{- end}}
-  ]
-
   command = [
-{{- range .EnvoyCommand }}
+    "consul", "connect", "envoy",
-    "{{.}}",
+    "-sidecar-for={{.Service.ID.Name}}",
-{{- end }}
+    "-grpc-addr=http://127.0.0.1:8502",
+    // for demo purposes (TODO: huh?)
+    "-admin-bind=0.0.0.0:{{.Service.EnvoyAdminPort}}",
+  {{if .Enterprise}} 
+    "-partition={{.Service.ID.Partition}}",
+    "-namespace={{.Service.ID.Namespace}}",
+  {{end}}
+  {{if .Token }}
+    "-token={{.Token}}",
+  {{end}}
+    "--",
+    "-l",
+    "trace",
   ]
 }

testing/deployer/sprawl/internal/tfgen/templates/container-app.tf.tmpl

@@ -1,7 +1,7 @@
 resource "docker_container" "{{.Node.DockerName}}-{{.Service.ID.TFString}}" {
 	name = "{{.Node.DockerName}}-{{.Service.ID.TFString}}"
     network_mode = "container:${docker_container.{{.PodName}}.id}"
-    image        = docker_image.{{.AppImageResource}}.latest
+    image        = docker_image.{{.ImageResource}}.latest
     restart  = "on-failure"
 
 {{- range $k, $v := .Labels }}
@@ -12,13 +12,13 @@ resource "docker_container" "{{.Node.DockerName}}-{{.Service.ID.TFString}}" {
 {{- end }}
 
   env = [
-{{- range .Env }}
+{{- range .Service.Env }}
       "{{.}}",
 {{- end}}
   ]
 
   command = [
-{{- range .Command }}
+{{- range .Service.Command }}
     "{{.}}",
 {{- end }}
   ]

testing/deployer/sprawl/internal/tfgen/templates/container-consul.tf.tmpl

@@ -8,9 +8,6 @@ resource "docker_container" "{{.Node.DockerName}}" {
       "CONSUL_UID=0",
       "CONSUL_GID=0",
       "CONSUL_LICENSE={{.EnterpriseLicense}}",
-{{- range .Env }}
-      "{{.}}",
-{{- end}}
   ]
 
 {{- range $k, $v := .Labels }}

testing/deployer/sprawl/internal/tfgen/templates/container-mgw-dataplane.tf.tmpl

@@ -0,0 +1,45 @@
+resource "docker_container" "{{.Node.DockerName}}-{{.Service.ID.TFString}}" {
+	name = "{{.Node.DockerName}}-{{.Service.ID.TFString}}"
+    network_mode = "container:${docker_container.{{.PodName}}.id}"
+    image        = docker_image.{{.ImageResource}}.latest
+    restart      = "on-failure"
+
+{{- range $k, $v := .Labels }}
+  labels {
+    label = "{{ $k }}"
+    value = "{{ $v }}"
+  }
+{{- end }}
+
+  volumes {
+    volume_name    = "{{.TLSVolumeName}}"
+    container_path = "/consul/config/certs"
+    read_only      = true
+  }
+
+  env = [
+			"DP_CONSUL_ADDRESSES=server.{{.Node.Cluster}}-consulcluster.lan",
+			"DP_SERVICE_NODE_NAME={{.Node.PodName}}",
+			"DP_PROXY_SERVICE_ID={{.Service.ID.Name}}",
+    {{ if .Enterprise }}
+      "DP_SERVICE_NAMESPACE={{.Service.ID.Namespace}}",
+      "DP_SERVICE_PARTITION={{.Service.ID.Partition}}",
+    {{ end }}
+    {{ if .Token }}
+      "DP_CREDENTIAL_TYPE=static",
+      "DP_CREDENTIAL_STATIC_TOKEN={{.Token}}",
+    {{ end }}
+      // for demo purposes
+			"DP_ENVOY_ADMIN_BIND_ADDRESS=0.0.0.0",
+			"DP_ENVOY_ADMIN_BIND_PORT=19000",
+			"DP_LOG_LEVEL=trace",
+			"DP_CA_CERTS=/consul/config/certs/consul-agent-ca.pem",
+			"DP_CONSUL_GRPC_PORT=8503",
+			"DP_TLS_SERVER_NAME=server.{{.Node.Datacenter}}.consul",
+  ]
+
+  command = [
+    "/usr/local/bin/consul-dataplane",
+  ]
+}
+

testing/deployer/sprawl/internal/tfgen/templates/container-mgw.tf.tmpl

@@ -1,7 +1,7 @@
 resource "docker_container" "{{.Node.DockerName}}-{{.Service.ID.TFString}}" {
     name = "{{.Node.DockerName}}-{{.Service.ID.TFString}}"
     network_mode = "container:${docker_container.{{.PodName}}.id}"
-    image        = docker_image.{{.EnvoyImageResource}}.latest
+    image        = docker_image.{{.ImageResource}}.latest
     restart  = "on-failure"
 
 {{- range $k, $v := .Labels }}
@@ -18,8 +18,22 @@ resource "docker_container" "{{.Node.DockerName}}-{{.Service.ID.TFString}}" {
   }
 
   command = [
-{{- range .Command }}
+    "consul", "connect", "envoy",
-    "{{.}}",
+    "-register",
-{{- end }}
+    "-mesh-gateway",
+    "-address={{`{{ GetInterfaceIP \"eth0\" }}`}}:{{.Service.Port}}",
+    "-wan-address={{`{{ GetInterfaceIP \"eth1\" }}`}}:{{.Service.Port}}",
+    "-grpc-addr=http://127.0.0.1:8502",
+    // for demo purposes (TODO: huh?)
+    "-admin-bind=0.0.0.0:{{.Service.EnvoyAdminPort}}",
+  {{ if .Enterprise }} 
+    "-partition={{.Service.ID.Partition}}",
+  {{end}}
+  {{ if .Token }}
+    "-token={{.Token}}",
+  {{end}}
+    "--",
+    "-l",
+    "trace",
   ]
 }

testing/deployer/sprawl/internal/tfgen/tfgen.go

@@ -5,6 +5,7 @@ package tfgen
 
 import (
 	"embed"
+	"text/template"
 )
 
 //go:embed templates/container-app-dataplane.tf.tmpl
@@ -12,7 +13,20 @@ import (
 //go:embed templates/container-app.tf.tmpl
 //go:embed templates/container-consul.tf.tmpl
 //go:embed templates/container-mgw.tf.tmpl
+//go:embed templates/container-mgw-dataplane.tf.tmpl
 //go:embed templates/container-pause.tf.tmpl
 //go:embed templates/container-proxy.tf.tmpl
 //go:embed templates/container-coredns.tf.tmpl
 var content embed.FS
+
+var (
+	tfAppDataplaneT         = template.Must(template.ParseFS(content, "templates/container-app-dataplane.tf.tmpl"))
+	tfAppSidecarT           = template.Must(template.ParseFS(content, "templates/container-app-sidecar.tf.tmpl"))
+	tfAppT                  = template.Must(template.ParseFS(content, "templates/container-app.tf.tmpl"))
+	tfConsulT               = template.Must(template.ParseFS(content, "templates/container-consul.tf.tmpl"))
+	tfMeshGatewayT          = template.Must(template.ParseFS(content, "templates/container-mgw.tf.tmpl"))
+	tfMeshGatewayDataplaneT = template.Must(template.ParseFS(content, "templates/container-mgw-dataplane.tf.tmpl"))
+	tfPauseT                = template.Must(template.ParseFS(content, "templates/container-pause.tf.tmpl"))
+	tfForwardProxyT         = template.Must(template.ParseFS(content, "templates/container-proxy.tf.tmpl"))
+	tfCorednsT              = template.Must(template.ParseFS(content, "templates/container-coredns.tf.tmpl"))
+)

testing/deployer/sprawl/sprawl.go

@@ -113,6 +113,21 @@ func (s *Sprawl) APIClientForNode(clusterName string, nid topology.NodeID, token
 	)
 }
 
+// APIClientForCluster is a convenience wrapper for APIClientForNode that returns
+// an API client for an agent node in the cluster, preferring clients, then servers
+func (s *Sprawl) APIClientForCluster(clusterName, token string) (*api.Client, error) {
+	clu := s.topology.Clusters[clusterName]
+	// TODO: this always goes to the first client, but we might want to balance this
+	firstAgent := clu.FirstClient()
+	if firstAgent == nil {
+		firstAgent = clu.FirstServer()
+	}
+	if firstAgent == nil {
+		return nil, fmt.Errorf("failed to find agent in cluster %s", clusterName)
+	}
+	return s.APIClientForNode(clusterName, firstAgent.ID(), token)
+}
+
 func copyConfig(cfg *topology.Config) (*topology.Config, error) {
 	dup, err := copystructure.Copy(cfg)
 	if err != nil {

testing/deployer/sprawl/tls.go

@@ -88,7 +88,9 @@ fi
 			"-i",
 			"--net=none",
 			"-v", cluster.TLSVolumeName + ":/data",
-			"busybox:latest",
+			// TODO: latest busted?
+			// https://hashicorp.slack.com/archives/C03EUN3QF1C/p1691784078972959
+			"busybox:1.34",
 			"sh", "-c",
 			// Need this so the permissions stick; docker seems to treat unused volumes differently.
 			`touch /data/VOLUME_PLACEHOLDER && chown -R ` + consulUserArg + ` /data`,

testing/deployer/topology/images.go

@@ -66,6 +66,7 @@ func (i Images) EnvoyConsulImage() string {
 	return "local/" + name1 + "-and-" + name2 + ":" + tag1 + "-with-" + tag2
 }
 
+// TODO: what is this for and why do we need to do this and why is it named this?
 func (i Images) ChooseNode(kind NodeKind) Images {
 	switch kind {
 	case NodeKindServer:

testing/deployer/topology/topology.go

@@ -290,6 +290,7 @@ func (c *Cluster) ServerByAddr(addr string) *Node {
 
 func (c *Cluster) FirstServer() *Node {
 	for _, node := range c.Nodes {
+		// TODO: not sure why we check that it has 8500 exposed?
 		if node.IsServer() && !node.Disabled && node.ExposedPort(8500) > 0 {
 			return node
 		}
@@ -432,9 +433,6 @@ type Node struct {
 	// the enclosing Cluster.
 	Images Images
 
-	// AgentEnv contains optional environment variables to attach to Consul agents.
-	AgentEnv []string
-
 	Disabled bool `json:",omitempty"`
 
 	Addresses []*Address