From 33a90edfab653fe68076a8d25f193056bba1e8fb Mon Sep 17 00:00:00 2001 From: cskh Date: Wed, 13 Dec 2023 20:15:55 -0500 Subject: [PATCH] Upgrade test(LTS): use network area to federate cluster (#19934) - Join areas - wait for members alive and validate cross area service discovery --- testing/deployer/sprawl/acl.go | 2 +- testing/deployer/sprawl/boot.go | 23 +++++++++++++++++-- .../deployer/sprawl/internal/tfgen/agent.go | 4 +++- testing/deployer/sprawl/network_area_ce.go | 14 +++++++++++ testing/deployer/topology/compile.go | 11 +++++---- testing/deployer/topology/topology.go | 19 +++++++++++++++ 6 files changed, 64 insertions(+), 9 deletions(-) create mode 100644 testing/deployer/sprawl/network_area_ce.go diff --git a/testing/deployer/sprawl/acl.go b/testing/deployer/sprawl/acl.go index 2f5cdcc160..d4098be251 100644 --- a/testing/deployer/sprawl/acl.go +++ b/testing/deployer/sprawl/acl.go @@ -171,7 +171,7 @@ node_prefix "" { policy = "write" } -operator = "read" +operator = "write" ` policy, _, err := acl.PolicyCreate( &api.ACLPolicy{ diff --git a/testing/deployer/sprawl/boot.go b/testing/deployer/sprawl/boot.go index 8f8aa6cee1..0389c77914 100644 --- a/testing/deployer/sprawl/boot.go +++ b/testing/deployer/sprawl/boot.go @@ -12,6 +12,7 @@ import ( "strings" "time" + retry "github.com/avast/retry-go" "github.com/hashicorp/consul/api" "github.com/hashicorp/go-multierror" @@ -146,6 +147,10 @@ func (s *Sprawl) launchType(firstTime bool, launchPhase LaunchPhase) (launchErr return fmt.Errorf("waitForPeeringEstablishment: %w", err) } + if err := s.waitForNetworkAreaEstablishment(); err != nil { + return fmt.Errorf("waitForNetworkAreaEstablishment: %w", err) + } + cleanupFuncs = nil // reset return nil @@ -198,7 +203,7 @@ func (s *Sprawl) assignIPAddresses() error { return fmt.Errorf("unknown network %q", addr.Network) } addr.IPAddress = net.IPByIndex(node.Index) - s.logger.Info("assign addr", "node", node.Name, "addr", addr.IPAddress, "enabled", !node.Disabled) + s.logger.Info("assign addr", "node", node.Name, "addr", addr.IPAddress, "type", addr.Type, "enabled", !node.Disabled) } } } @@ -315,7 +320,17 @@ func (s *Sprawl) createFirstTime() error { return fmt.Errorf("generator[agents]: %w", err) } for _, cluster := range s.topology.Clusters { - if err := s.waitForClientAntiEntropyOnce(cluster); err != nil { + err := retry.Do( + func() error { + if err := s.waitForClientAntiEntropyOnce(cluster); err != nil { + return fmt.Errorf("create first time - waitForClientAntiEntropyOnce[%s]: %w", cluster.Name, err) + } + return nil + }, + retry.MaxDelay(5*time.Second), + retry.Attempts(15), + ) + if err != nil { return fmt.Errorf("create first time - waitForClientAntiEntropyOnce[%s]: %w", cluster.Name, err) } } @@ -344,6 +359,10 @@ func (s *Sprawl) createFirstTime() error { if err := s.initPeerings(); err != nil { return fmt.Errorf("initPeerings: %w", err) } + + if err := s.initNetworkAreas(); err != nil { + return fmt.Errorf("initNetworkAreas: %w", err) + } return nil } diff --git a/testing/deployer/sprawl/internal/tfgen/agent.go b/testing/deployer/sprawl/internal/tfgen/agent.go index 05aed6ccdd..9245925d53 100644 --- a/testing/deployer/sprawl/internal/tfgen/agent.go +++ b/testing/deployer/sprawl/internal/tfgen/agent.go @@ -84,7 +84,9 @@ func (g *Generator) generateAgentHCL(node *topology.Node, enableV2, enableV2Tena b.add("prometheus_retention_time", "168h") }) - b.add("encrypt", g.sec.ReadGeneric(node.Cluster, secrets.GossipKey)) + if !cluster.DisableGossipEncryption { + b.add("encrypt", g.sec.ReadGeneric(node.Cluster, secrets.GossipKey)) + } { var ( diff --git a/testing/deployer/sprawl/network_area_ce.go b/testing/deployer/sprawl/network_area_ce.go new file mode 100644 index 0000000000..3b7a182a54 --- /dev/null +++ b/testing/deployer/sprawl/network_area_ce.go @@ -0,0 +1,14 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +//go:build !consulent + +package sprawl + +func (s *Sprawl) initNetworkAreas() error { + return nil +} + +func (s *Sprawl) waitForNetworkAreaEstablishment() error { + return nil +} diff --git a/testing/deployer/topology/compile.go b/testing/deployer/topology/compile.go index 2f1761e15b..0651115baa 100644 --- a/testing/deployer/topology/compile.go +++ b/testing/deployer/topology/compile.go @@ -694,11 +694,12 @@ func compile(logger hclog.Logger, raw *Config, prev *Topology) (*Topology, error } t := &Topology{ - ID: id, - Networks: networks, - Clusters: clusters, - Images: images, - Peerings: raw.Peerings, + ID: id, + Networks: networks, + Clusters: clusters, + Images: images, + Peerings: raw.Peerings, + NetworkAreas: raw.NetworkAreas, } if prev != nil { diff --git a/testing/deployer/topology/topology.go b/testing/deployer/topology/topology.go index 8e842ca32d..985fc837c4 100644 --- a/testing/deployer/topology/topology.go +++ b/testing/deployer/topology/topology.go @@ -35,6 +35,10 @@ type Topology struct { // Peerings defines the list of pairwise peerings that should be established // between clusters. Peerings []*Peering `json:",omitempty"` + + // NetworkAreas defines the list of pairwise network area that should be established + // between clusters. + NetworkAreas []*NetworkArea `json:",omitempty"` } func (t *Topology) DigestExposedProxyPort(netName string, proxyPort int) (bool, error) { @@ -100,6 +104,10 @@ type Config struct { // Peerings defines the list of pairwise peerings that should be established // between clusters. Peerings []*Peering + + // NetworkAreas defines the list of pairwise NetworkArea that should be established + // between clusters. + NetworkAreas []*NetworkArea } func (c *Config) Cluster(name string) *Cluster { @@ -293,6 +301,10 @@ type Cluster struct { // Segments is a map of network segment name and the ports Segments map[string]int + + // DisableGossipEncryption disables gossip encryption on the cluster + // Default is false to enable gossip encryption + DisableGossipEncryption bool `json:",omitempty"` } func (c *Cluster) inheritFromExisting(existing *Cluster) { @@ -1055,6 +1067,13 @@ type Peering struct { Accepting PeerCluster } +// NetworkArea - a pair of clusters that are peered together +// through network area. PeerCluster type is reused here. +type NetworkArea struct { + Primary PeerCluster + Secondary PeerCluster +} + type PeerCluster struct { Name string Partition string