status-im
/
consul
mirror of https://github.com/status-im/consul.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

switch all client nodes in dc2 to dataplane [NET-4299] (#18608)

This commit is contained in:
Nick Irvine 2023-09-06 16:46:34 -07:00 committed by GitHub
parent 4eb2197e82
commit 373c7dc144
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 294 additions and 240 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
test-integ/peering_commontopo/ac5_2_pq_failover_test.go
View File

@ -86,7 +86,10 @@ func (s *ac5_2PQFailoverSuite) setupDC(ct *commonTopo, clu, peerClu *topology.Cl
Service: NewFortioServiceWithDefaults(
clu.Datacenter,
serverSID,
nil,
func(s *topology.Service) {
s.EnvoyAdminPort = 0
s.DisableServiceMesh = true
},
),
Exports: []api.ServiceConsumer{{Peer: peer}},
}
@ -149,7 +152,10 @@ func (s *ac5_2PQFailoverSuite) setupDC3(ct *commonTopo, clu, peer1, peer2 *topol
Service: NewFortioServiceWithDefaults(
clu.Datacenter,
serverSID,
nil,
func(s *topology.Service) {
s.EnvoyAdminPort = 0
s.DisableServiceMesh = true
},
),
Exports: func() []api.ServiceConsumer {
var consumers []api.ServiceConsumer

6
test-integ/peering_commontopo/ac7_1_rotate_gw_test.go
View File

@ -141,8 +141,12 @@ func (s *suiteRotateGW) setup(t *testing.T, ct *commonTopo) {
// add a second mesh gateway "new"
s.newMGWNodeName = fmt.Sprintf("new-%s-default-mgw", clu.Name)
nodeKind := topology.NodeKindClient
if clu.Datacenter == agentlessDC {
nodeKind = topology.NodeKindDataplane
}
clu.Nodes = append(clu.Nodes, newTopologyMeshGatewaySet(
topology.NodeKindClient,
nodeKind,
"default",
s.newMGWNodeName,
1,

11
test-integ/peering_commontopo/asserter.go
View File

@ -41,6 +41,7 @@ type asserter struct {
type sprawlLite interface {
HTTPClientForCluster(clusterName string) (*http.Client, error)
APIClientForNode(clusterName string, nid topology.NodeID, token string) (*api.Client, error)
APIClientForCluster(clusterName string, token string) (*api.Client, error)
Topology() *topology.Topology
}
@ -58,18 +59,12 @@ func (a *asserter) mustGetHTTPClient(t *testing.T, cluster string) *http.Client
}
func (a *asserter) mustGetAPIClient(t *testing.T, cluster string) *api.Client {
cl, err := a.apiClientFor(cluster)
clu := a.sp.Topology().Clusters[cluster]
cl, err := a.sp.APIClientForCluster(clu.Name, "")
require.NoError(t, err)
return cl
}
func (a *asserter) apiClientFor(cluster string) (*api.Client, error) {
clu := a.sp.Topology().Clusters[cluster]
// TODO: this always goes to the first client, but we might want to balance this
cl, err := a.sp.APIClientForNode(cluster, clu.FirstClient().ID(), "")
return cl, err
}
// httpClientFor returns a pre-configured http.Client that proxies requests
// through the embedded squid instance in each LAN.
//

37
test-integ/peering_commontopo/commontopo.go
View File

@ -52,6 +52,8 @@ type commonTopo struct {
services map[string]map[topology.ServiceID]struct{}
}
const agentlessDC = "dc2"
func NewCommonTopo(t *testing.T) *commonTopo {
t.Helper()
@ -84,11 +86,9 @@ func NewCommonTopo(t *testing.T) *commonTopo {
peerings = append(peerings, addPeerings(dc1, dc3)...)
peerings = append(peerings, addPeerings(dc2, dc3)...)
addMeshGateways(dc1, topology.NodeKindClient)
addMeshGateways(dc2, topology.NodeKindClient)
addMeshGateways(dc3, topology.NodeKindClient)
// TODO: consul-topology doesn't support this yet
// addMeshGateways(dc2, topology.NodeKindDataplane)
addMeshGateways(dc1)
addMeshGateways(dc2)
addMeshGateways(dc3)
setupGlobals(dc1)
setupGlobals(dc2)
@ -131,7 +131,7 @@ func (ct *commonTopo) postLaunchChecks(t *testing.T) {
)
// check that exports line up as expected
for _, clu := range ct.Sprawl.Config().Clusters {
for _, clu := range ct.Sprawl.Topology().Clusters {
// expected exports per peer
type key struct {
peer string
@ -191,9 +191,6 @@ func LocalPeerName(clu *topology.Cluster, partition string) string {
type serviceExt struct {
*topology.Service
// default NodeKindClient
NodeKind topology.NodeKind
Exports []api.ServiceConsumer
Config *api.ServiceConfigEntry
Intentions *api.ServiceIntentionsConfigEntry
@ -227,8 +224,15 @@ func (ct *commonTopo) AddServiceNode(clu *topology.Cluster, svc serviceExt) *top
return n
}
nodeKind := topology.NodeKindClient
// TODO: bug in deployer somewhere; it should guard against a KindDataplane node with
// DisableServiceMesh services on it; dataplane is only for service-mesh
if !svc.DisableServiceMesh && clu.Datacenter == agentlessDC {
nodeKind = topology.NodeKindDataplane
}
node := &topology.Node{
Kind: topology.NodeKindClient,
Kind: nodeKind,
Name: serviceHostnameString(clu.Datacenter, svc.ID),
Partition: svc.ID.Partition,
Addresses: []*topology.Address{
@ -239,9 +243,6 @@ func (ct *commonTopo) AddServiceNode(clu *topology.Cluster, svc serviceExt) *top
},
Cluster: clusterName,
}
if svc.NodeKind != "" {
node.Kind = svc.NodeKind
}
clu.Nodes = append(clu.Nodes, node)
// Export if necessary
@ -265,7 +266,7 @@ func (ct *commonTopo) AddServiceNode(clu *topology.Cluster, svc serviceExt) *top
}
func (ct *commonTopo) APIClientForCluster(t *testing.T, clu *topology.Cluster) *api.Client {
cl, err := ct.Sprawl.APIClientForNode(clu.Name, clu.FirstClient().ID(), "")
cl, err := ct.Sprawl.APIClientForCluster(clu.Name, "")
require.NoError(t, err)
return cl
}
@ -372,10 +373,14 @@ func setupGlobals(clu *topology.Cluster) {
// addMeshGateways adds a mesh gateway for every partition in the cluster.
// Assumes that the LAN network name is equal to datacenter name.
func addMeshGateways(c *topology.Cluster, kind topology.NodeKind) {
func addMeshGateways(c *topology.Cluster) {
nodeKind := topology.NodeKindClient
if c.Datacenter == agentlessDC {
nodeKind = topology.NodeKindDataplane
}
for _, p := range c.Partitions {
c.Nodes = topology.MergeSlices(c.Nodes, newTopologyMeshGatewaySet(
kind,
nodeKind,
p.Name,
fmt.Sprintf("%s-%s-mgw", c.Name, p.Name),
1,

32
testing/deployer/sprawl/catalog.go
View File

@ -318,6 +318,18 @@ func serviceToCatalogRegistration(
Address: node.LocalAddress(),
},
}
if svc.IsMeshGateway {
reg.Service.Kind = api.ServiceKindMeshGateway
reg.Service.Proxy = &api.AgentServiceConnectProxyConfig{
Config: map[string]interface{}{
"envoy_gateway_no_default_bind": true,
"envoy_gateway_bind_tagged_addresses": true,
},
MeshGateway: api.MeshGatewayConfig{
Mode: api.MeshGatewayModeLocal,
},
}
}
if node.HasPublicAddress() {
reg.TaggedAddresses = map[string]string{
"lan": node.LocalAddress(),
@ -325,6 +337,26 @@ func serviceToCatalogRegistration(
"wan": node.PublicAddress(),
"wan_ipv4": node.PublicAddress(),
}
// TODO: not sure what the difference is between these, but with just the
// top-level set, it appeared to not get set in either :/
reg.Service.TaggedAddresses = map[string]api.ServiceAddress{
"lan": {
Address: node.LocalAddress(),
Port: svc.Port,
},
"lan_ipv4": {
Address: node.LocalAddress(),
Port: svc.Port,
},
"wan": {
Address: node.PublicAddress(),
Port: svc.Port,
},
"wan_ipv4": {
Address: node.PublicAddress(),
Port: svc.Port,
},
}
}
if cluster.Enterprise {
reg.Partition = svc.ID.Partition

3
testing/deployer/sprawl/internal/build/docker.go
View File

@ -23,9 +23,10 @@ COPY --from=0 /bin/consul /bin/consul
// FROM hashicorp/consul-dataplane:latest
// COPY --from=busybox:uclibc /bin/sh /bin/sh
// TODO: busybox:latest doesn't work, see https://hashicorp.slack.com/archives/C03EUN3QF1C/p1691784078972959
const dockerfileDataplane = `
ARG DATAPLANE_IMAGE
FROM busybox:latest
FROM busybox:1.34
FROM ${DATAPLANE_IMAGE}
COPY --from=0 /bin/busybox /bin/busybox
USER 0:0

8
testing/deployer/sprawl/internal/tfgen/agent.go
View File

@ -13,14 +13,14 @@ import (
"github.com/hashicorp/consul/testing/deployer/topology"
)
func (g *Generator) generateAgentHCL(node *topology.Node) (string, error) {
func (g *Generator) generateAgentHCL(node *topology.Node) string {
if !node.IsAgent() {
return "", fmt.Errorf("not an agent")
panic("generateAgentHCL only applies to agents")
}
cluster, ok := g.topology.Clusters[node.Cluster]
if !ok {
return "", fmt.Errorf("no such cluster: %s", node.Cluster)
panic(fmt.Sprintf("no such cluster: %s", node.Cluster))
}
var b HCLBuilder
@ -167,7 +167,7 @@ func (g *Generator) generateAgentHCL(node *topology.Node) (string, error) {
}
}
return b.String(), nil
return b.String()
}
type HCLBuilder struct {

3
testing/deployer/sprawl/internal/tfgen/dns.go
View File

@ -9,7 +9,6 @@ import (
"os"
"path/filepath"
"strings"
"text/template"
"github.com/hashicorp/consul/testing/deployer/topology"
"github.com/hashicorp/consul/testing/deployer/util"
@ -179,5 +178,3 @@ server IN A %s ; Consul server
return buf.Bytes()
}
var tfCorednsT = template.Must(template.ParseFS(content, "templates/container-coredns.tf.tmpl"))

3
testing/deployer/sprawl/internal/tfgen/docker.go
View File

@ -11,6 +11,9 @@ import (
var invalidResourceName = regexp.MustCompile(`[^a-z0-9-]+`)
func DockerImageResourceName(image string) string {
if image == "" {
panic(`image must not be ""`)
}
return invalidResourceName.ReplaceAllLiteralString(image, "-")
}

249
testing/deployer/sprawl/internal/tfgen/nodes.go
View File

@ -5,9 +5,6 @@ package tfgen
import (
"fmt"
"sort"
"strconv"
"text/template"
"github.com/hashicorp/consul/testing/deployer/topology"
)
@ -22,32 +19,6 @@ type terraformPod struct {
DockerNetworkName string
}
type terraformConsulAgent struct {
terraformPod
ImageResource string
HCL string
EnterpriseLicense string
Env []string
}
type terraformMeshGatewayService struct {
terraformPod
EnvoyImageResource string
Service *topology.Service
Command []string
}
type terraformService struct {
terraformPod
AppImageResource string
EnvoyImageResource string // agentful
DataplaneImageResource string // agentless
Service *topology.Service
Env []string
Command []string
EnvoyCommand []string // agentful
}
func (g *Generator) generateNodeContainers(
step Step,
cluster *topology.Cluster,
@ -82,156 +53,99 @@ func (g *Generator) generateNodeContainers(
}
pod.DockerNetworkName = net.DockerName
var (
containers []Resource
)
containers := []Resource{}
if node.IsAgent() {
agentHCL, err := g.generateAgentHCL(node)
if err != nil {
return nil, err
}
agent := terraformConsulAgent{
terraformPod: pod,
ImageResource: DockerImageResourceName(node.Images.Consul),
HCL: agentHCL,
EnterpriseLicense: g.license,
Env: node.AgentEnv,
}
switch {
case node.IsServer() && step.StartServers(),
!node.IsServer() && step.StartAgents():
containers = append(containers, Eval(tfConsulT, &agent))
containers = append(containers, Eval(tfConsulT, struct {
terraformPod
ImageResource string
HCL string
EnterpriseLicense string
}{
terraformPod: pod,
ImageResource: DockerImageResourceName(node.Images.Consul),
HCL: g.generateAgentHCL(node),
EnterpriseLicense: g.license,
}))
}
}
svcContainers := []Resource{}
for _, svc := range node.SortedServices() {
if svc.IsMeshGateway {
if node.Kind == topology.NodeKindDataplane {
panic("NOT READY YET")
}
gw := terraformMeshGatewayService{
token := g.sec.ReadServiceToken(node.Cluster, svc.ID)
switch {
case svc.IsMeshGateway && !node.IsDataplane():
svcContainers = append(svcContainers, Eval(tfMeshGatewayT, struct {
terraformPod
ImageResource string
Enterprise bool
Service *topology.Service
Token string
}{
terraformPod: pod,
EnvoyImageResource: DockerImageResourceName(node.Images.EnvoyConsulImage()),
ImageResource: DockerImageResourceName(node.Images.EnvoyConsulImage()),
Enterprise: cluster.Enterprise,
Service: svc,
Command: []string{
"consul", "connect", "envoy",
"-register",
"-mesh-gateway",
},
}
if token := g.sec.ReadServiceToken(node.Cluster, svc.ID); token != "" {
gw.Command = append(gw.Command, "-token", token)
}
if cluster.Enterprise {
gw.Command = append(gw.Command,
"-partition",
svc.ID.Partition,
)
}
gw.Command = append(gw.Command,
"-address",
`{{ GetInterfaceIP \"eth0\" }}:`+strconv.Itoa(svc.Port),
"-wan-address",
`{{ GetInterfaceIP \"eth1\" }}:`+strconv.Itoa(svc.Port),
)
gw.Command = append(gw.Command,
"-grpc-addr", "http://127.0.0.1:8502",
"-admin-bind",
// for demo purposes
"0.0.0.0:"+strconv.Itoa(svc.EnvoyAdminPort),
"--",
"-l",
"trace",
)
if step.StartServices() {
containers = append(containers, Eval(tfMeshGatewayT, &gw))
}
} else {
tfsvc := terraformService{
Token: token,
}))
case svc.IsMeshGateway && node.IsDataplane():
svcContainers = append(svcContainers, Eval(tfMeshGatewayDataplaneT, &struct {
terraformPod
ImageResource string
Enterprise bool
Service *topology.Service
Token string
}{
terraformPod: pod,
AppImageResource: DockerImageResourceName(svc.Image),
ImageResource: DockerImageResourceName(node.Images.LocalDataplaneImage()),
Enterprise: cluster.Enterprise,
Service: svc,
Command: svc.Command,
Token: token,
}))
case !svc.IsMeshGateway:
svcContainers = append(svcContainers, Eval(tfAppT, struct {
terraformPod
ImageResource string
Service *topology.Service
}{
terraformPod: pod,
ImageResource: DockerImageResourceName(svc.Image),
Service: svc,
}))
if svc.DisableServiceMesh {
break
}
tfsvc.Env = append(tfsvc.Env, svc.Env...)
tmpl := tfAppSidecarT
var img string
if node.IsDataplane() {
tmpl = tfAppDataplaneT
img = DockerImageResourceName(node.Images.LocalDataplaneImage())
} else {
img = DockerImageResourceName(node.Images.EnvoyConsulImage())
}
svcContainers = append(svcContainers, Eval(tmpl, struct {
terraformPod
ImageResource string
Service *topology.Service
Token string
Enterprise bool
}{
terraformPod: pod,
ImageResource: img,
Service: svc,
Token: token,
Enterprise: cluster.Enterprise,
}))
}
if step.StartServices() {
containers = append(containers, Eval(tfAppT, &tfsvc))
}
setenv := func(k, v string) {
tfsvc.Env = append(tfsvc.Env, k+"="+v)
}
if !svc.DisableServiceMesh {
if node.IsDataplane() {
tfsvc.DataplaneImageResource = DockerImageResourceName(node.Images.LocalDataplaneImage())
tfsvc.EnvoyImageResource = ""
tfsvc.EnvoyCommand = nil
// --- REQUIRED ---
setenv("DP_CONSUL_ADDRESSES", "server."+node.Cluster+"-consulcluster.lan")
setenv("DP_SERVICE_NODE_NAME", node.PodName())
setenv("DP_PROXY_SERVICE_ID", svc.ID.Name+"-sidecar-proxy")
} else {
tfsvc.DataplaneImageResource = ""
tfsvc.EnvoyImageResource = DockerImageResourceName(node.Images.EnvoyConsulImage())
tfsvc.EnvoyCommand = []string{
"consul", "connect", "envoy",
"-sidecar-for", svc.ID.Name,
}
}
if cluster.Enterprise {
if node.IsDataplane() {
setenv("DP_SERVICE_NAMESPACE", svc.ID.Namespace)
setenv("DP_SERVICE_PARTITION", svc.ID.Partition)
} else {
tfsvc.EnvoyCommand = append(tfsvc.EnvoyCommand,
"-partition",
svc.ID.Partition,
"-namespace",
svc.ID.Namespace,
)
}
}
if token := g.sec.ReadServiceToken(node.Cluster, svc.ID); token != "" {
if node.IsDataplane() {
setenv("DP_CREDENTIAL_TYPE", "static")
setenv("DP_CREDENTIAL_STATIC_TOKEN", token)
} else {
tfsvc.EnvoyCommand = append(tfsvc.EnvoyCommand, "-token", token)
}
}
if node.IsDataplane() {
setenv("DP_ENVOY_ADMIN_BIND_ADDRESS", "0.0.0.0") // for demo purposes
setenv("DP_ENVOY_ADMIN_BIND_PORT", "19000")
setenv("DP_LOG_LEVEL", "trace")
setenv("DP_CA_CERTS", "/consul/config/certs/consul-agent-ca.pem")
setenv("DP_CONSUL_GRPC_PORT", "8503")
setenv("DP_TLS_SERVER_NAME", "server."+node.Datacenter+".consul")
} else {
tfsvc.EnvoyCommand = append(tfsvc.EnvoyCommand,
"-grpc-addr", "http://127.0.0.1:8502",
"-admin-bind",
// for demo purposes
"0.0.0.0:"+strconv.Itoa(svc.EnvoyAdminPort),
"--",
"-l",
"trace",
)
}
if step.StartServices() {
sort.Strings(tfsvc.Env)
if node.IsDataplane() {
containers = append(containers, Eval(tfAppDataplaneT, &tfsvc))
} else {
containers = append(containers, Eval(tfAppSidecarT, &tfsvc))
}
}
}
containers = append(containers, svcContainers...)
}
}
@ -243,10 +157,3 @@ func (g *Generator) generateNodeContainers(
return containers, nil
}
var tfPauseT = template.Must(template.ParseFS(content, "templates/container-pause.tf.tmpl"))
var tfConsulT = template.Must(template.ParseFS(content, "templates/container-consul.tf.tmpl"))
var tfMeshGatewayT = template.Must(template.ParseFS(content, "templates/container-mgw.tf.tmpl"))
var tfAppT = template.Must(template.ParseFS(content, "templates/container-app.tf.tmpl"))
var tfAppSidecarT = template.Must(template.ParseFS(content, "templates/container-app-sidecar.tf.tmpl"))
var tfAppDataplaneT = template.Must(template.ParseFS(content, "templates/container-app-dataplane.tf.tmpl"))

3
testing/deployer/sprawl/internal/tfgen/proxy.go
View File

@ -7,7 +7,6 @@ import (
"fmt"
"os"
"path/filepath"
"text/template"
"github.com/hashicorp/consul/testing/deployer/topology"
"github.com/hashicorp/consul/testing/deployer/util"
@ -86,5 +85,3 @@ func (g *Generator) getForwardProxyContainer(
return Eval(tfForwardProxyT, &proxy)
}
var tfForwardProxyT = template.Must(template.ParseFS(content, "templates/container-proxy.tf.tmpl"))

23
testing/deployer/sprawl/internal/tfgen/templates/container-app-dataplane.tf.tmpl
View File

@ -1,7 +1,7 @@
resource "docker_container" "{{.Node.DockerName}}-{{.Service.ID.TFString}}-sidecar" {
name = "{{.Node.DockerName}}-{{.Service.ID.TFString}}-sidecar"
network_mode = "container:${docker_container.{{.PodName}}.id}"
image = docker_image.{{.DataplaneImageResource}}.latest
image = docker_image.{{.ImageResource}}.latest
restart = "on-failure"
{{- range $k, $v := .Labels }}
@ -18,9 +18,24 @@ resource "docker_container" "{{.Node.DockerName}}-{{.Service.ID.TFString}}-sidec
}
env = [
{{- range .Env }}
"{{.}}",
{{- end}}
"DP_CONSUL_ADDRESSES=server.{{.Node.Cluster}}-consulcluster.lan",
"DP_SERVICE_NODE_NAME={{.Node.PodName}}",
"DP_PROXY_SERVICE_ID={{.Service.ID.Name}}-sidecar-proxy",
{{ if .Enterprise }}
"DP_SERVICE_NAMESPACE={{.Service.ID.Namespace}}",
"DP_SERVICE_PARTITION={{.Service.ID.Partition}}",
{{ end }}
{{ if .Token }}
"DP_CREDENTIAL_TYPE=static",
"DP_CREDENTIAL_STATIC_TOKEN={{.Token}}",
{{ end }}
// for demo purposes
"DP_ENVOY_ADMIN_BIND_ADDRESS=0.0.0.0",
"DP_ENVOY_ADMIN_BIND_PORT=19000",
"DP_LOG_LEVEL=trace",
"DP_CA_CERTS=/consul/config/certs/consul-agent-ca.pem",
"DP_CONSUL_GRPC_PORT=8503",
"DP_TLS_SERVER_NAME=server.{{.Node.Datacenter}}.consul",
]
command = [

26
testing/deployer/sprawl/internal/tfgen/templates/container-app-sidecar.tf.tmpl
View File

@ -1,7 +1,7 @@
resource "docker_container" "{{.Node.DockerName}}-{{.Service.ID.TFString}}-sidecar" {
name = "{{.Node.DockerName}}-{{.Service.ID.TFString}}-sidecar"
network_mode = "container:${docker_container.{{.PodName}}.id}"
image = docker_image.{{.EnvoyImageResource}}.latest
image = docker_image.{{.ImageResource}}.latest
restart = "on-failure"
{{- range $k, $v := .Labels }}
@ -17,15 +17,21 @@ resource "docker_container" "{{.Node.DockerName}}-{{.Service.ID.TFString}}-sidec
read_only = true
}
env = [
{{- range .Env }}
"{{.}}",
{{- end}}
]
command = [
{{- range .EnvoyCommand }}
"{{.}}",
{{- end }}
"consul", "connect", "envoy",
"-sidecar-for={{.Service.ID.Name}}",
"-grpc-addr=http://127.0.0.1:8502",
// for demo purposes (TODO: huh?)
"-admin-bind=0.0.0.0:{{.Service.EnvoyAdminPort}}",
{{if .Enterprise}}
"-partition={{.Service.ID.Partition}}",
"-namespace={{.Service.ID.Namespace}}",
{{end}}
{{if .Token }}
"-token={{.Token}}",
{{end}}
"--",
"-l",
"trace",
]
}

6
testing/deployer/sprawl/internal/tfgen/templates/container-app.tf.tmpl
View File

@ -1,7 +1,7 @@
resource "docker_container" "{{.Node.DockerName}}-{{.Service.ID.TFString}}" {
name = "{{.Node.DockerName}}-{{.Service.ID.TFString}}"
network_mode = "container:${docker_container.{{.PodName}}.id}"
image = docker_image.{{.AppImageResource}}.latest
image = docker_image.{{.ImageResource}}.latest
restart = "on-failure"
{{- range $k, $v := .Labels }}
@ -12,13 +12,13 @@ resource "docker_container" "{{.Node.DockerName}}-{{.Service.ID.TFString}}" {
{{- end }}
env = [
{{- range .Env }}
{{- range .Service.Env }}
"{{.}}",
{{- end}}
]
command = [
{{- range .Command }}
{{- range .Service.Command }}
"{{.}}",
{{- end }}
]

3
testing/deployer/sprawl/internal/tfgen/templates/container-consul.tf.tmpl
View File

@ -8,9 +8,6 @@ resource "docker_container" "{{.Node.DockerName}}" {
"CONSUL_UID=0",
"CONSUL_GID=0",
"CONSUL_LICENSE={{.EnterpriseLicense}}",
{{- range .Env }}
"{{.}}",
{{- end}}
]
{{- range $k, $v := .Labels }}

45
testing/deployer/sprawl/internal/tfgen/templates/container-mgw-dataplane.tf.tmpl Normal file
View File

@ -0,0 +1,45 @@
resource "docker_container" "{{.Node.DockerName}}-{{.Service.ID.TFString}}" {
name = "{{.Node.DockerName}}-{{.Service.ID.TFString}}"
network_mode = "container:${docker_container.{{.PodName}}.id}"
image = docker_image.{{.ImageResource}}.latest
restart = "on-failure"
{{- range $k, $v := .Labels }}
labels {
label = "{{ $k }}"
value = "{{ $v }}"
}
{{- end }}
volumes {
volume_name = "{{.TLSVolumeName}}"
container_path = "/consul/config/certs"
read_only = true
}
env = [
"DP_CONSUL_ADDRESSES=server.{{.Node.Cluster}}-consulcluster.lan",
"DP_SERVICE_NODE_NAME={{.Node.PodName}}",
"DP_PROXY_SERVICE_ID={{.Service.ID.Name}}",
{{ if .Enterprise }}
"DP_SERVICE_NAMESPACE={{.Service.ID.Namespace}}",
"DP_SERVICE_PARTITION={{.Service.ID.Partition}}",
{{ end }}
{{ if .Token }}
"DP_CREDENTIAL_TYPE=static",
"DP_CREDENTIAL_STATIC_TOKEN={{.Token}}",
{{ end }}
// for demo purposes
"DP_ENVOY_ADMIN_BIND_ADDRESS=0.0.0.0",
"DP_ENVOY_ADMIN_BIND_PORT=19000",
"DP_LOG_LEVEL=trace",
"DP_CA_CERTS=/consul/config/certs/consul-agent-ca.pem",
"DP_CONSUL_GRPC_PORT=8503",
"DP_TLS_SERVER_NAME=server.{{.Node.Datacenter}}.consul",
]
command = [
"/usr/local/bin/consul-dataplane",
]
}

22
testing/deployer/sprawl/internal/tfgen/templates/container-mgw.tf.tmpl
View File

@ -1,7 +1,7 @@
resource "docker_container" "{{.Node.DockerName}}-{{.Service.ID.TFString}}" {
name = "{{.Node.DockerName}}-{{.Service.ID.TFString}}"
network_mode = "container:${docker_container.{{.PodName}}.id}"
image = docker_image.{{.EnvoyImageResource}}.latest
image = docker_image.{{.ImageResource}}.latest
restart = "on-failure"
{{- range $k, $v := .Labels }}
@ -18,8 +18,22 @@ resource "docker_container" "{{.Node.DockerName}}-{{.Service.ID.TFString}}" {
}
command = [
{{- range .Command }}
"{{.}}",
{{- end }}
"consul", "connect", "envoy",
"-register",
"-mesh-gateway",
"-address={{`{{ GetInterfaceIP \"eth0\" }}`}}:{{.Service.Port}}",
"-wan-address={{`{{ GetInterfaceIP \"eth1\" }}`}}:{{.Service.Port}}",
"-grpc-addr=http://127.0.0.1:8502",
// for demo purposes (TODO: huh?)
"-admin-bind=0.0.0.0:{{.Service.EnvoyAdminPort}}",
{{ if .Enterprise }}
"-partition={{.Service.ID.Partition}}",
{{end}}
{{ if .Token }}
"-token={{.Token}}",
{{end}}
"--",
"-l",
"trace",
]
}

14
testing/deployer/sprawl/internal/tfgen/tfgen.go
View File

@ -5,6 +5,7 @@ package tfgen
import (
"embed"
"text/template"
)
//go:embed templates/container-app-dataplane.tf.tmpl
@ -12,7 +13,20 @@ import (
//go:embed templates/container-app.tf.tmpl
//go:embed templates/container-consul.tf.tmpl
//go:embed templates/container-mgw.tf.tmpl
//go:embed templates/container-mgw-dataplane.tf.tmpl
//go:embed templates/container-pause.tf.tmpl
//go:embed templates/container-proxy.tf.tmpl
//go:embed templates/container-coredns.tf.tmpl
var content embed.FS
var (
tfAppDataplaneT = template.Must(template.ParseFS(content, "templates/container-app-dataplane.tf.tmpl"))
tfAppSidecarT = template.Must(template.ParseFS(content, "templates/container-app-sidecar.tf.tmpl"))
tfAppT = template.Must(template.ParseFS(content, "templates/container-app.tf.tmpl"))
tfConsulT = template.Must(template.ParseFS(content, "templates/container-consul.tf.tmpl"))
tfMeshGatewayT = template.Must(template.ParseFS(content, "templates/container-mgw.tf.tmpl"))
tfMeshGatewayDataplaneT = template.Must(template.ParseFS(content, "templates/container-mgw-dataplane.tf.tmpl"))
tfPauseT = template.Must(template.ParseFS(content, "templates/container-pause.tf.tmpl"))
tfForwardProxyT = template.Must(template.ParseFS(content, "templates/container-proxy.tf.tmpl"))
tfCorednsT = template.Must(template.ParseFS(content, "templates/container-coredns.tf.tmpl"))
)

15
testing/deployer/sprawl/sprawl.go
View File

@ -113,6 +113,21 @@ func (s *Sprawl) APIClientForNode(clusterName string, nid topology.NodeID, token
)
}
// APIClientForCluster is a convenience wrapper for APIClientForNode that returns
// an API client for an agent node in the cluster, preferring clients, then servers
func (s *Sprawl) APIClientForCluster(clusterName, token string) (*api.Client, error) {
clu := s.topology.Clusters[clusterName]
// TODO: this always goes to the first client, but we might want to balance this
firstAgent := clu.FirstClient()
if firstAgent == nil {
firstAgent = clu.FirstServer()
}
if firstAgent == nil {
return nil, fmt.Errorf("failed to find agent in cluster %s", clusterName)
}
return s.APIClientForNode(clusterName, firstAgent.ID(), token)
}
func copyConfig(cfg *topology.Config) (*topology.Config, error) {
dup, err := copystructure.Copy(cfg)
if err != nil {

4
testing/deployer/sprawl/tls.go
View File

@ -88,7 +88,9 @@ fi
"-i",
"--net=none",
"-v", cluster.TLSVolumeName + ":/data",
"busybox:latest",
// TODO: latest busted?
// https://hashicorp.slack.com/archives/C03EUN3QF1C/p1691784078972959
"busybox:1.34",
"sh", "-c",
// Need this so the permissions stick; docker seems to treat unused volumes differently.
`touch /data/VOLUME_PLACEHOLDER && chown -R ` + consulUserArg + ` /data`,

1
testing/deployer/topology/images.go
View File

@ -66,6 +66,7 @@ func (i Images) EnvoyConsulImage() string {
return "local/" + name1 + "-and-" + name2 + ":" + tag1 + "-with-" + tag2
}
// TODO: what is this for and why do we need to do this and why is it named this?
func (i Images) ChooseNode(kind NodeKind) Images {
switch kind {
case NodeKindServer:

4
testing/deployer/topology/topology.go
View File

@ -290,6 +290,7 @@ func (c *Cluster) ServerByAddr(addr string) *Node {
func (c *Cluster) FirstServer() *Node {
for _, node := range c.Nodes {
// TODO: not sure why we check that it has 8500 exposed?
if node.IsServer() && !node.Disabled && node.ExposedPort(8500) > 0 {
return node
}
@ -432,9 +433,6 @@ type Node struct {
// the enclosing Cluster.
Images Images
// AgentEnv contains optional environment variables to attach to Consul agents.
AgentEnv []string
Disabled bool `json:",omitempty"`
Addresses []*Address