mirror of https://github.com/status-im/consul.git
chore: apply enterprise changes that were missed to some testing files (#19504)
This should align between CE ef35525
and ENT 7f95226dbe40151c8f17dd4464784b60cf358dc1 in:
- testing/integration/consul-container
- test-integ
- testing/deployer
This commit is contained in:
parent
fd128f4947
commit
65592d91a8
|
@ -232,8 +232,6 @@ func (s *ac1BasicSuite) test(t *testing.T, ct *commonTopo) {
|
||||||
// probably not worth the speed boost
|
// probably not worth the speed boost
|
||||||
ct.Assert.HealthyWithPeer(t, dc.Name, svcServerHTTP.ID, LocalPeerName(peer, "default"))
|
ct.Assert.HealthyWithPeer(t, dc.Name, svcServerHTTP.ID, LocalPeerName(peer, "default"))
|
||||||
ct.Assert.HealthyWithPeer(t, dc.Name, svcServerTCP.ID, LocalPeerName(peer, "default"))
|
ct.Assert.HealthyWithPeer(t, dc.Name, svcServerTCP.ID, LocalPeerName(peer, "default"))
|
||||||
ct.Assert.UpstreamEndpointHealthy(t, svcClientTCP, ac.upstreamTCP)
|
|
||||||
ct.Assert.UpstreamEndpointHealthy(t, svcClientTCP, ac.upstreamHTTP)
|
|
||||||
|
|
||||||
tcs := []struct {
|
tcs := []struct {
|
||||||
acSub int
|
acSub int
|
||||||
|
|
|
@ -172,7 +172,6 @@ func (s *ac3SvcDefaultsSuite) test(t *testing.T, ct *commonTopo) {
|
||||||
// these could be done parallel with each other, but complexity
|
// these could be done parallel with each other, but complexity
|
||||||
// probably not worth the speed boost
|
// probably not worth the speed boost
|
||||||
ct.Assert.HealthyWithPeer(t, dc.Name, svcServer.ID, LocalPeerName(peer, "default"))
|
ct.Assert.HealthyWithPeer(t, dc.Name, svcServer.ID, LocalPeerName(peer, "default"))
|
||||||
ct.Assert.UpstreamEndpointHealthy(t, svcClient, s.upstream)
|
|
||||||
// TODO: we need to let the upstream start serving properly before we do this. if it
|
// TODO: we need to let the upstream start serving properly before we do this. if it
|
||||||
// isn't ready and returns a 5xx (which it will do if it's not up yet!), it will stick
|
// isn't ready and returns a 5xx (which it will do if it's not up yet!), it will stick
|
||||||
// in a down state for PassiveHealthCheck.Interval
|
// in a down state for PassiveHealthCheck.Interval
|
||||||
|
|
|
@ -159,7 +159,6 @@ func (s *ac4ProxyDefaultsSuite) test(t *testing.T, ct *commonTopo) {
|
||||||
|
|
||||||
// preconditions check
|
// preconditions check
|
||||||
ct.Assert.HealthyWithPeer(t, dc.Name, serverSVC.ID, LocalPeerName(peer, "default"))
|
ct.Assert.HealthyWithPeer(t, dc.Name, serverSVC.ID, LocalPeerName(peer, "default"))
|
||||||
ct.Assert.UpstreamEndpointHealthy(t, clientSVC, s.upstream)
|
|
||||||
ct.Assert.FortioFetch2HeaderEcho(t, clientSVC, s.upstream)
|
ct.Assert.FortioFetch2HeaderEcho(t, clientSVC, s.upstream)
|
||||||
|
|
||||||
t.Run("Validate services exist in catalog", func(t *testing.T) {
|
t.Run("Validate services exist in catalog", func(t *testing.T) {
|
||||||
|
|
|
@ -30,11 +30,15 @@ type ac5_2PQFailoverSuite struct {
|
||||||
serverSID topology.ServiceID
|
serverSID topology.ServiceID
|
||||||
nodeServer topology.NodeID
|
nodeServer topology.NodeID
|
||||||
}
|
}
|
||||||
|
type nodeKey struct {
|
||||||
|
dc string
|
||||||
|
partition string
|
||||||
|
}
|
||||||
|
|
||||||
var ac5_2Context = make(map[nodeKey]ac5_2PQFailoverSuite)
|
var ac5_2Context = make(map[nodeKey]ac5_2PQFailoverSuite)
|
||||||
|
|
||||||
func TestAC5PreparedQueryFailover(t *testing.T) {
|
func TestAC5PreparedQueryFailover(t *testing.T) {
|
||||||
ct := NewCommonTopo(t)
|
ct := newCommonTopo(t, "dc2", true, true)
|
||||||
s := &ac5_2PQFailoverSuite{}
|
s := &ac5_2PQFailoverSuite{}
|
||||||
s.setup(t, ct)
|
s.setup(t, ct)
|
||||||
ct.Launch(t)
|
ct.Launch(t)
|
||||||
|
|
|
@ -14,419 +14,453 @@ import (
|
||||||
"github.com/hashicorp/consul/test/integration/consul-container/libs/utils"
|
"github.com/hashicorp/consul/test/integration/consul-container/libs/utils"
|
||||||
)
|
)
|
||||||
|
|
||||||
// note: unlike other *Suite structs that are per-peering direction,
|
|
||||||
// this one is special and does all directions itself, because the
|
|
||||||
// setup is not exactly symmetrical
|
|
||||||
type ac6FailoversSuite struct {
|
type ac6FailoversSuite struct {
|
||||||
ac6 map[nodeKey]ac6FailoversContext
|
// inputs
|
||||||
}
|
// with all false, this gives us a scenario with:
|
||||||
type ac6FailoversContext struct {
|
// - a "near" server in the accepter cluster (DC1), partitition default, namespace default
|
||||||
clientSID topology.ServiceID
|
// - a "far" server in the dialer cluster (DC2), partition default, namespace default
|
||||||
serverSID topology.ServiceID
|
// - a client in the accepter cluster (DC1), partition default, namespace default, with:
|
||||||
|
// - upstream near server (DC1)
|
||||||
|
// - failover to far server (DC2)
|
||||||
|
//
|
||||||
|
// TODO: technically if NearInDial && !FarInAcc (i.e., near == far), then we're not doing peering at all,
|
||||||
|
// and could do this test in a single DC
|
||||||
|
|
||||||
|
// when true, put the client (and its default upstream server) in the dialer peer; otherwise, put client in accepter
|
||||||
|
NearInDial bool
|
||||||
|
// when true, put the client (and its default upstream server) in the nondefault partition/namespace; otherwise in the default
|
||||||
|
NearInPartAlt bool
|
||||||
|
NearInNSAlt bool
|
||||||
|
// when true, put far server to the accepter peer; otherwise the dialer
|
||||||
|
FarInAcc bool
|
||||||
|
// when true, put far server to nondefault partition/namespace (ENT-only); otherwise, failover to default
|
||||||
|
FarInPartAlt bool
|
||||||
|
FarInNSAlt bool
|
||||||
|
|
||||||
|
// launch outputs, for querying during test
|
||||||
|
clientSID topology.ServiceID
|
||||||
|
// near = same DC as client; far = other DC
|
||||||
|
nearServerSID topology.ServiceID
|
||||||
// used to remove the node and trigger failover
|
// used to remove the node and trigger failover
|
||||||
serverNode topology.NodeID
|
nearServerNode topology.NodeID
|
||||||
}
|
farServerSID topology.ServiceID
|
||||||
type nodeKey struct {
|
farServerNode topology.NodeID
|
||||||
dc string
|
|
||||||
partition string
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Note: this test cannot share topo
|
// Note: this test cannot share topo
|
||||||
func TestAC6Failovers(t *testing.T) {
|
func TestAC6Failovers(t *testing.T) {
|
||||||
ct := NewCommonTopo(t)
|
// bit banging to get all permutations of all params
|
||||||
s := &ac6FailoversSuite{}
|
const nParams = 3
|
||||||
s.setup(t, ct)
|
// i.e 2**nParams
|
||||||
ct.Launch(t)
|
const n = int(1) << nParams
|
||||||
s.test(t, ct)
|
for i := 0; i < n; i++ {
|
||||||
}
|
s := ac6FailoversSuite{
|
||||||
|
// xth bit == 1
|
||||||
func (s *ac6FailoversSuite) setup(t *testing.T, ct *commonTopo) {
|
NearInDial: (i>>0)&1 == 1,
|
||||||
// TODO: update setups to loop through a cluster's partitions+namespaces internally
|
NearInPartAlt: (i>>1)&1 == 1,
|
||||||
s.setupAC6Failovers(ct, ct.DC1, ct.DC2)
|
FarInPartAlt: (i>>2)&1 == 1,
|
||||||
s.setupAC6Failovers(ct, ct.DC2, ct.DC1)
|
|
||||||
s.setupAC6FailoversDC3(ct, ct.DC3, ct.DC1, ct.DC2)
|
|
||||||
}
|
|
||||||
|
|
||||||
// dc1 is peered with dc2 and dc3.
|
|
||||||
// dc1 has an ac6-client in "default" and "part1" partitions (only default in CE).
|
|
||||||
// ac6-client has a single upstream ac6-failover-svc in its respective partition^.
|
|
||||||
//
|
|
||||||
// ac6-failover-svc has the following failovers:
|
|
||||||
// - peer-dc2-default
|
|
||||||
// - peer-dc2-part1 (not in CE)
|
|
||||||
// - peer-dc3-default
|
|
||||||
//
|
|
||||||
// This setup is mirrored from dc2->dc1 as well
|
|
||||||
// (both dcs have dc3 as the last failover target)
|
|
||||||
//
|
|
||||||
// ^NOTE: There are no cross-partition upstreams because MeshGatewayMode = local
|
|
||||||
// and failover information gets stripped out by the mesh gateways so we
|
|
||||||
// can't test failovers.
|
|
||||||
func (s *ac6FailoversSuite) setupAC6Failovers(ct *commonTopo, clu, peerClu *topology.Cluster) {
|
|
||||||
for _, part := range clu.Partitions {
|
|
||||||
partition := part.Name
|
|
||||||
|
|
||||||
// There is a peering per partition in the peered cluster
|
|
||||||
var peers []string
|
|
||||||
for _, peerPart := range peerClu.Partitions {
|
|
||||||
peers = append(peers, LocalPeerName(peerClu, peerPart.Name))
|
|
||||||
}
|
}
|
||||||
|
// ensure the servers are always in separate DCs
|
||||||
// Make an HTTP server with various failover targets
|
s.FarInAcc = s.NearInDial
|
||||||
serverSID := topology.ServiceID{
|
t.Run(fmt.Sprintf("%02d_%s", i, s.testName()), func(t *testing.T) {
|
||||||
Name: "ac6-failover-svc",
|
t.Parallel()
|
||||||
Partition: partition,
|
ct := NewCommonTopo(t)
|
||||||
}
|
s.setup(t, ct)
|
||||||
server := NewFortioServiceWithDefaults(
|
ct.Launch(t)
|
||||||
clu.Datacenter,
|
s.test(t, ct)
|
||||||
serverSID,
|
|
||||||
nil,
|
|
||||||
)
|
|
||||||
// Export to all known peers
|
|
||||||
ct.ExportService(clu, partition,
|
|
||||||
api.ExportedService{
|
|
||||||
Name: server.ID.Name,
|
|
||||||
Consumers: func() []api.ServiceConsumer {
|
|
||||||
var consumers []api.ServiceConsumer
|
|
||||||
for _, peer := range peers {
|
|
||||||
consumers = append(consumers, api.ServiceConsumer{
|
|
||||||
Peer: peer,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return consumers
|
|
||||||
}(),
|
|
||||||
},
|
|
||||||
)
|
|
||||||
serverNode := ct.AddServiceNode(clu, serviceExt{Service: server})
|
|
||||||
|
|
||||||
clu.InitialConfigEntries = append(clu.InitialConfigEntries,
|
|
||||||
&api.ServiceConfigEntry{
|
|
||||||
Kind: api.ServiceDefaults,
|
|
||||||
Name: server.ID.Name,
|
|
||||||
Partition: ConfigEntryPartition(partition),
|
|
||||||
Protocol: "http",
|
|
||||||
},
|
|
||||||
&api.ServiceResolverConfigEntry{
|
|
||||||
Kind: api.ServiceResolver,
|
|
||||||
Name: server.ID.Name,
|
|
||||||
Partition: ConfigEntryPartition(partition),
|
|
||||||
Failover: map[string]api.ServiceResolverFailover{
|
|
||||||
"*": {
|
|
||||||
Targets: func() []api.ServiceResolverFailoverTarget {
|
|
||||||
// Make a failover target for every partition in the peer cluster
|
|
||||||
var targets []api.ServiceResolverFailoverTarget
|
|
||||||
for _, peer := range peers {
|
|
||||||
targets = append(targets, api.ServiceResolverFailoverTarget{
|
|
||||||
Peer: peer,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
// Just hard code default partition for dc3, since the exhaustive
|
|
||||||
// testing will be done against dc2.
|
|
||||||
targets = append(targets, api.ServiceResolverFailoverTarget{
|
|
||||||
Peer: "peer-dc3-default",
|
|
||||||
})
|
|
||||||
return targets
|
|
||||||
}(),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
// Make client which will dial server
|
|
||||||
clientSID := topology.ServiceID{
|
|
||||||
Name: "ac6-client",
|
|
||||||
Partition: partition,
|
|
||||||
}
|
|
||||||
client := NewFortioServiceWithDefaults(
|
|
||||||
clu.Datacenter,
|
|
||||||
clientSID,
|
|
||||||
func(s *topology.Service) {
|
|
||||||
// Upstream per partition
|
|
||||||
s.Upstreams = []*topology.Upstream{
|
|
||||||
{
|
|
||||||
ID: topology.ServiceID{
|
|
||||||
Name: server.ID.Name,
|
|
||||||
Partition: part.Name,
|
|
||||||
},
|
|
||||||
LocalPort: 5000,
|
|
||||||
// exposed so we can hit it directly
|
|
||||||
// TODO: we shouldn't do this; it's not realistic
|
|
||||||
LocalAddress: "0.0.0.0",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
},
|
|
||||||
)
|
|
||||||
ct.ExportService(clu, partition,
|
|
||||||
api.ExportedService{
|
|
||||||
Name: client.ID.Name,
|
|
||||||
Consumers: func() []api.ServiceConsumer {
|
|
||||||
var consumers []api.ServiceConsumer
|
|
||||||
// Export to each peer
|
|
||||||
for _, peer := range peers {
|
|
||||||
consumers = append(consumers, api.ServiceConsumer{
|
|
||||||
Peer: peer,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return consumers
|
|
||||||
}(),
|
|
||||||
},
|
|
||||||
)
|
|
||||||
ct.AddServiceNode(clu, serviceExt{Service: client})
|
|
||||||
|
|
||||||
clu.InitialConfigEntries = append(clu.InitialConfigEntries,
|
|
||||||
&api.ServiceConfigEntry{
|
|
||||||
Kind: api.ServiceDefaults,
|
|
||||||
Name: client.ID.Name,
|
|
||||||
Partition: ConfigEntryPartition(partition),
|
|
||||||
Protocol: "http",
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
// Add intention allowing local and peered clients to call server
|
|
||||||
clu.InitialConfigEntries = append(clu.InitialConfigEntries,
|
|
||||||
&api.ServiceIntentionsConfigEntry{
|
|
||||||
Kind: api.ServiceIntentions,
|
|
||||||
Name: server.ID.Name,
|
|
||||||
Partition: ConfigEntryPartition(partition),
|
|
||||||
// SourceIntention for local client and peered clients
|
|
||||||
Sources: func() []*api.SourceIntention {
|
|
||||||
ixns := []*api.SourceIntention{
|
|
||||||
{
|
|
||||||
Name: client.ID.Name,
|
|
||||||
Partition: ConfigEntryPartition(part.Name),
|
|
||||||
Action: api.IntentionActionAllow,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
for _, peer := range peers {
|
|
||||||
ixns = append(ixns, &api.SourceIntention{
|
|
||||||
Name: client.ID.Name,
|
|
||||||
Peer: peer,
|
|
||||||
Action: api.IntentionActionAllow,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return ixns
|
|
||||||
}(),
|
|
||||||
},
|
|
||||||
)
|
|
||||||
if s.ac6 == nil {
|
|
||||||
s.ac6 = map[nodeKey]ac6FailoversContext{}
|
|
||||||
}
|
|
||||||
s.ac6[nodeKey{clu.Datacenter, partition}] = struct {
|
|
||||||
clientSID topology.ServiceID
|
|
||||||
serverSID topology.ServiceID
|
|
||||||
serverNode topology.NodeID
|
|
||||||
}{
|
|
||||||
clientSID: clientSID,
|
|
||||||
serverSID: serverSID,
|
|
||||||
serverNode: serverNode.ID(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *ac6FailoversSuite) setupAC6FailoversDC3(ct *commonTopo, clu, peer1, peer2 *topology.Cluster) {
|
|
||||||
var peers []string
|
|
||||||
for _, part := range peer1.Partitions {
|
|
||||||
peers = append(peers, LocalPeerName(peer1, part.Name))
|
|
||||||
}
|
|
||||||
for _, part := range peer2.Partitions {
|
|
||||||
peers = append(peers, LocalPeerName(peer2, part.Name))
|
|
||||||
}
|
|
||||||
|
|
||||||
partition := "default"
|
|
||||||
|
|
||||||
// Make an HTTP server
|
|
||||||
server := NewFortioServiceWithDefaults(
|
|
||||||
clu.Datacenter,
|
|
||||||
topology.ServiceID{
|
|
||||||
Name: "ac6-failover-svc",
|
|
||||||
Partition: partition,
|
|
||||||
},
|
|
||||||
nil,
|
|
||||||
)
|
|
||||||
|
|
||||||
ct.AddServiceNode(clu, serviceExt{
|
|
||||||
Service: server,
|
|
||||||
Config: &api.ServiceConfigEntry{
|
|
||||||
Kind: api.ServiceDefaults,
|
|
||||||
Name: server.ID.Name,
|
|
||||||
Partition: ConfigEntryPartition(partition),
|
|
||||||
Protocol: "http",
|
|
||||||
},
|
|
||||||
Intentions: &api.ServiceIntentionsConfigEntry{
|
|
||||||
Kind: api.ServiceIntentions,
|
|
||||||
Name: server.ID.Name,
|
|
||||||
Partition: ConfigEntryPartition(partition),
|
|
||||||
Sources: func() []*api.SourceIntention {
|
|
||||||
var ixns []*api.SourceIntention
|
|
||||||
for _, peer := range peers {
|
|
||||||
ixns = append(ixns, &api.SourceIntention{
|
|
||||||
Name: "ac6-client",
|
|
||||||
Peer: peer,
|
|
||||||
Action: api.IntentionActionAllow,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return ixns
|
|
||||||
}(),
|
|
||||||
},
|
|
||||||
Exports: func() []api.ServiceConsumer {
|
|
||||||
var consumers []api.ServiceConsumer
|
|
||||||
for _, peer := range peers {
|
|
||||||
consumers = append(consumers, api.ServiceConsumer{
|
|
||||||
Peer: peer,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return consumers
|
|
||||||
}(),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *ac6FailoversSuite) test(t *testing.T, ct *commonTopo) {
|
|
||||||
dc1 := ct.Sprawl.Topology().Clusters["dc1"]
|
|
||||||
dc2 := ct.Sprawl.Topology().Clusters["dc2"]
|
|
||||||
|
|
||||||
type testcase struct {
|
|
||||||
name string
|
|
||||||
cluster *topology.Cluster
|
|
||||||
peer *topology.Cluster
|
|
||||||
partition string
|
|
||||||
}
|
|
||||||
tcs := []testcase{
|
|
||||||
{
|
|
||||||
name: "dc1 default partition failovers",
|
|
||||||
cluster: dc1,
|
|
||||||
peer: dc2, // dc3 is hardcoded
|
|
||||||
partition: "default",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "dc1 part1 partition failovers",
|
|
||||||
cluster: dc1,
|
|
||||||
peer: dc2, // dc3 is hardcoded
|
|
||||||
partition: "part1",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "dc2 default partition failovers",
|
|
||||||
cluster: dc2,
|
|
||||||
peer: dc1, // dc3 is hardcoded
|
|
||||||
partition: "default",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "dc2 part1 partition failovers",
|
|
||||||
cluster: dc2,
|
|
||||||
peer: dc1, // dc3 is hardcoded
|
|
||||||
partition: "part1",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
for _, tc := range tcs {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
// NOTE: *not parallel* because we mutate resources that are shared
|
|
||||||
// between test cases (disable/enable nodes)
|
|
||||||
if !utils.IsEnterprise() && tc.partition != "default" {
|
|
||||||
t.Skip("skipping enterprise test")
|
|
||||||
}
|
|
||||||
partition := tc.partition
|
|
||||||
clu := tc.cluster
|
|
||||||
peerClu := tc.peer
|
|
||||||
|
|
||||||
svcs := clu.ServicesByID(s.ac6[nodeKey{clu.Datacenter, partition}].clientSID)
|
|
||||||
require.Len(t, svcs, 1, "expected exactly one client in datacenter")
|
|
||||||
|
|
||||||
serverSID := s.ac6[nodeKey{clu.Datacenter, partition}].serverSID
|
|
||||||
serverSID.Normalize()
|
|
||||||
|
|
||||||
client := svcs[0]
|
|
||||||
require.Len(t, client.Upstreams, 1, "expected one upstream for client")
|
|
||||||
|
|
||||||
u := client.Upstreams[0]
|
|
||||||
ct.Assert.CatalogServiceExists(t, clu.Name, u.ID.Name, utils.CompatQueryOpts(&api.QueryOptions{
|
|
||||||
Partition: u.ID.Partition,
|
|
||||||
}))
|
|
||||||
|
|
||||||
t.Cleanup(func() {
|
|
||||||
cfg := ct.Sprawl.Config()
|
|
||||||
for _, part := range clu.Partitions {
|
|
||||||
EnableNode(t, cfg, clu.Name, s.ac6[nodeKey{clu.Datacenter, part.Name}].serverNode)
|
|
||||||
}
|
|
||||||
for _, part := range peerClu.Partitions {
|
|
||||||
EnableNode(t, cfg, peerClu.Name, s.ac6[nodeKey{peerClu.Datacenter, part.Name}].serverNode)
|
|
||||||
}
|
|
||||||
require.NoError(t, ct.Sprawl.Relaunch(cfg))
|
|
||||||
})
|
|
||||||
|
|
||||||
fmt.Println("### preconditions")
|
|
||||||
// TODO: deduce this number, instead of hard-coding
|
|
||||||
nFailoverTargets := 4
|
|
||||||
// in CE, we don't have failover targets for non-default partitions
|
|
||||||
if !utils.IsEnterprise() {
|
|
||||||
nFailoverTargets = 3
|
|
||||||
}
|
|
||||||
for i := 0; i < nFailoverTargets; i++ {
|
|
||||||
ct.Assert.UpstreamEndpointStatus(t, client, fmt.Sprintf("failover-target~%d~%s", i, clusterPrefix(u, clu.Datacenter)), "HEALTHY", 1)
|
|
||||||
}
|
|
||||||
|
|
||||||
ct.Assert.FortioFetch2FortioName(t, client, u, clu.Name, serverSID)
|
|
||||||
|
|
||||||
if t.Failed() {
|
|
||||||
t.Fatalf("failed preconditions")
|
|
||||||
}
|
|
||||||
|
|
||||||
fmt.Println("### Failover to peer target")
|
|
||||||
cfg := ct.Sprawl.Config()
|
|
||||||
DisableNode(t, cfg, clu.Name, s.ac6[nodeKey{clu.Datacenter, partition}].serverNode)
|
|
||||||
require.NoError(t, ct.Sprawl.Relaunch(cfg))
|
|
||||||
// Clusters for imported services rely on outlier detection for
|
|
||||||
// failovers, NOT eds_health_status. This means that killing the
|
|
||||||
// node above does not actually make the envoy cluster UNHEALTHY
|
|
||||||
// so we do not assert for it.
|
|
||||||
expectUID := topology.ServiceID{
|
|
||||||
Name: u.ID.Name,
|
|
||||||
Partition: "default",
|
|
||||||
}
|
|
||||||
expectUID.Normalize()
|
|
||||||
ct.Assert.FortioFetch2FortioName(t, client, u, peerClu.Name, expectUID)
|
|
||||||
|
|
||||||
if utils.IsEnterprise() {
|
|
||||||
fmt.Println("### Failover to peer target in non-default partition")
|
|
||||||
cfg = ct.Sprawl.Config()
|
|
||||||
DisableNode(t, cfg, clu.Name, s.ac6[nodeKey{clu.Datacenter, partition}].serverNode)
|
|
||||||
DisableNode(t, cfg, peerClu.Name, s.ac6[nodeKey{peerClu.Datacenter, "default"}].serverNode)
|
|
||||||
require.NoError(t, ct.Sprawl.Relaunch(cfg))
|
|
||||||
// Retry until outlier_detection deems the cluster
|
|
||||||
// unhealthy and fails over to peer part1.
|
|
||||||
expectUID = topology.ServiceID{
|
|
||||||
Name: u.ID.Name,
|
|
||||||
Partition: "part1",
|
|
||||||
}
|
|
||||||
expectUID.Normalize()
|
|
||||||
ct.Assert.FortioFetch2FortioName(t, client, u, peerClu.Name, expectUID)
|
|
||||||
}
|
|
||||||
|
|
||||||
fmt.Println("### Failover to dc3 peer target")
|
|
||||||
cfg = ct.Sprawl.Config()
|
|
||||||
DisableNode(t, cfg, clu.Name, s.ac6[nodeKey{clu.Datacenter, partition}].serverNode)
|
|
||||||
// Disable all partitions for peer
|
|
||||||
for _, part := range peerClu.Partitions {
|
|
||||||
DisableNode(t, cfg, peerClu.Name, s.ac6[nodeKey{peerClu.Datacenter, part.Name}].serverNode)
|
|
||||||
}
|
|
||||||
require.NoError(t, ct.Sprawl.Relaunch(cfg))
|
|
||||||
// This will retry until outlier_detection deems the cluster
|
|
||||||
// unhealthy and fails over to dc3.
|
|
||||||
expectUID = topology.ServiceID{
|
|
||||||
Name: u.ID.Name,
|
|
||||||
Partition: "default",
|
|
||||||
}
|
|
||||||
expectUID.Normalize()
|
|
||||||
ct.Assert.FortioFetch2FortioName(t, client, u, "dc3", expectUID)
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func clusterPrefix(u *topology.Upstream, dc string) string {
|
func TestNET5029Failovers(t *testing.T) {
|
||||||
u.ID.Normalize()
|
// TODO: *.{a,b} are not actually peering tests, and should technically be moved elsewhere
|
||||||
switch u.ID.Partition {
|
suites := map[string]ac6FailoversSuite{
|
||||||
case "default":
|
"1.a": {
|
||||||
return fmt.Sprintf("%s.%s.%s.internal", u.ID.Name, u.ID.Namespace, dc)
|
FarInAcc: true,
|
||||||
default:
|
FarInPartAlt: true,
|
||||||
return fmt.Sprintf("%s.%s.%s.%s.internal-v1", u.ID.Name, u.ID.Namespace, u.ID.Partition, dc)
|
},
|
||||||
|
"1.b": {
|
||||||
|
FarInAcc: true,
|
||||||
|
FarInNSAlt: true,
|
||||||
|
},
|
||||||
|
"1.c": {
|
||||||
|
FarInNSAlt: true,
|
||||||
|
},
|
||||||
|
"1.d": {
|
||||||
|
FarInPartAlt: true,
|
||||||
|
},
|
||||||
|
"2.a": {
|
||||||
|
FarInAcc: true,
|
||||||
|
NearInPartAlt: true,
|
||||||
|
},
|
||||||
|
"2.b": {
|
||||||
|
FarInAcc: true,
|
||||||
|
NearInNSAlt: true,
|
||||||
|
},
|
||||||
|
"2.c": {
|
||||||
|
NearInDial: true,
|
||||||
|
NearInNSAlt: true,
|
||||||
|
FarInAcc: true,
|
||||||
|
},
|
||||||
|
"2.d": {
|
||||||
|
NearInDial: true,
|
||||||
|
NearInPartAlt: true,
|
||||||
|
FarInAcc: true,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for name, s := range suites {
|
||||||
|
s := s
|
||||||
|
t.Run(fmt.Sprintf("%s_%s", name, s.testName()), func(t *testing.T) {
|
||||||
|
if name == "1.b" {
|
||||||
|
t.Skip("TODO: fails with 503/504")
|
||||||
|
}
|
||||||
|
t.Parallel()
|
||||||
|
ct := NewCommonTopo(t)
|
||||||
|
s.setup(t, ct)
|
||||||
|
ct.Launch(t)
|
||||||
|
s.test(t, ct)
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestAC6Failovers_AllPermutations(t *testing.T) {
|
||||||
|
//
|
||||||
|
t.Skip("Too many permutations")
|
||||||
|
// bit banging to get all permutations of all params
|
||||||
|
const nParams = 6
|
||||||
|
// i.e 2**nParams
|
||||||
|
const n = int(1) << nParams
|
||||||
|
for i := 0; i < n; i++ {
|
||||||
|
s := ac6FailoversSuite{
|
||||||
|
// xth bit == 1
|
||||||
|
NearInDial: (i>>0)&1 == 1,
|
||||||
|
FarInAcc: (i>>1)&1 == 1,
|
||||||
|
NearInPartAlt: (i>>2)&1 == 1,
|
||||||
|
FarInPartAlt: (i>>3)&1 == 1,
|
||||||
|
NearInNSAlt: (i>>4)&1 == 1,
|
||||||
|
FarInNSAlt: (i>>5)&1 == 1,
|
||||||
|
}
|
||||||
|
t.Run(fmt.Sprintf("%02d_%s", i, s.testName()), func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
ct := NewCommonTopo(t)
|
||||||
|
s.setup(t, ct)
|
||||||
|
ct.Launch(t)
|
||||||
|
s.test(t, ct)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *ac6FailoversSuite) testName() (ret string) {
|
||||||
|
switch s.NearInDial {
|
||||||
|
case true:
|
||||||
|
ret += "dial"
|
||||||
|
default:
|
||||||
|
ret += "acc"
|
||||||
|
}
|
||||||
|
ret += "."
|
||||||
|
switch s.NearInPartAlt {
|
||||||
|
case true:
|
||||||
|
ret += "alt"
|
||||||
|
default:
|
||||||
|
ret += "default"
|
||||||
|
}
|
||||||
|
ret += "."
|
||||||
|
switch s.NearInNSAlt {
|
||||||
|
case true:
|
||||||
|
ret += "alt"
|
||||||
|
default:
|
||||||
|
ret += "default"
|
||||||
|
}
|
||||||
|
|
||||||
|
ret += "->"
|
||||||
|
|
||||||
|
switch s.FarInAcc {
|
||||||
|
case true:
|
||||||
|
ret += "acc"
|
||||||
|
default:
|
||||||
|
ret += "dial"
|
||||||
|
}
|
||||||
|
ret += "."
|
||||||
|
switch s.FarInPartAlt {
|
||||||
|
case true:
|
||||||
|
ret += "alt"
|
||||||
|
default:
|
||||||
|
ret += "default"
|
||||||
|
}
|
||||||
|
ret += "."
|
||||||
|
switch s.FarInNSAlt {
|
||||||
|
case true:
|
||||||
|
ret += "alt"
|
||||||
|
default:
|
||||||
|
ret += "default"
|
||||||
|
}
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *ac6FailoversSuite) setup(t *testing.T, ct *commonTopo) {
|
||||||
|
if !utils.IsEnterprise() && (s.NearInPartAlt || s.FarInPartAlt) {
|
||||||
|
t.Skip("ENT required for nondefault partitions")
|
||||||
|
}
|
||||||
|
|
||||||
|
nearClu := ct.DC1
|
||||||
|
farClu := ct.DC2
|
||||||
|
if s.NearInDial {
|
||||||
|
nearClu = ct.DC2
|
||||||
|
}
|
||||||
|
if s.FarInAcc {
|
||||||
|
farClu = ct.DC1
|
||||||
|
}
|
||||||
|
|
||||||
|
// - server in clientPartition/DC (main target)
|
||||||
|
nearServerSID := topology.ServiceID{
|
||||||
|
Name: "ac6-server",
|
||||||
|
Partition: ConfigEntryPartition("default"),
|
||||||
|
Namespace: "default",
|
||||||
|
}
|
||||||
|
if s.NearInPartAlt {
|
||||||
|
nearServerSID.Partition = "part1"
|
||||||
|
}
|
||||||
|
if s.NearInNSAlt {
|
||||||
|
nearServerSID.Namespace = "ns1"
|
||||||
|
}
|
||||||
|
nearServer := NewFortioServiceWithDefaults(
|
||||||
|
nearClu.Datacenter,
|
||||||
|
nearServerSID,
|
||||||
|
nil,
|
||||||
|
)
|
||||||
|
nearServerNode := ct.AddServiceNode(nearClu, serviceExt{Service: nearServer})
|
||||||
|
|
||||||
|
nearClu.InitialConfigEntries = append(nearClu.InitialConfigEntries,
|
||||||
|
&api.ServiceConfigEntry{
|
||||||
|
Kind: api.ServiceDefaults,
|
||||||
|
Name: nearServerSID.Name,
|
||||||
|
Partition: ConfigEntryPartition(nearServerSID.Partition),
|
||||||
|
Namespace: nearServerSID.Namespace,
|
||||||
|
Protocol: "http",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
// - server in otherPartition/otherDC
|
||||||
|
farServerSID := topology.ServiceID{
|
||||||
|
Name: nearServerSID.Name,
|
||||||
|
Partition: "default",
|
||||||
|
Namespace: "default",
|
||||||
|
}
|
||||||
|
if s.FarInPartAlt {
|
||||||
|
farServerSID.Partition = "part1"
|
||||||
|
}
|
||||||
|
if s.FarInNSAlt {
|
||||||
|
farServerSID.Namespace = "ns1"
|
||||||
|
}
|
||||||
|
farServer := NewFortioServiceWithDefaults(
|
||||||
|
farClu.Datacenter,
|
||||||
|
farServerSID,
|
||||||
|
nil,
|
||||||
|
)
|
||||||
|
farServerNode := ct.AddServiceNode(farClu, serviceExt{Service: farServer})
|
||||||
|
if nearClu != farClu {
|
||||||
|
ct.ExportService(farClu, farServerSID.Partition,
|
||||||
|
api.ExportedService{
|
||||||
|
Name: farServerSID.Name,
|
||||||
|
Namespace: farServerSID.Namespace,
|
||||||
|
Consumers: []api.ServiceConsumer{
|
||||||
|
{
|
||||||
|
Peer: LocalPeerName(nearClu, nearServerSID.Partition),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
} else if nearClu == farClu && farServerSID.Partition != nearServerSID.Partition {
|
||||||
|
ct.ExportService(farClu, farServerSID.Partition,
|
||||||
|
api.ExportedService{
|
||||||
|
Name: farServerSID.Name,
|
||||||
|
Namespace: farServerSID.Namespace,
|
||||||
|
Consumers: []api.ServiceConsumer{
|
||||||
|
{
|
||||||
|
// this must not be "", or else it is basically ignored altogether
|
||||||
|
// TODO: bug? if this whole struct is empty, that should be an error
|
||||||
|
Partition: topology.PartitionOrDefault(nearServerSID.Partition),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
var targets []api.ServiceResolverFailoverTarget
|
||||||
|
if nearClu != farClu {
|
||||||
|
targets = []api.ServiceResolverFailoverTarget{
|
||||||
|
{
|
||||||
|
Service: farServerSID.Name,
|
||||||
|
Peer: LocalPeerName(farClu, farServerSID.Partition),
|
||||||
|
Namespace: farServerSID.Namespace,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
part := ConfigEntryPartition(farServerSID.Partition)
|
||||||
|
// weird exception here where target partition set to "" means "inherit from parent"
|
||||||
|
// TODO: bug? docs say "" -> default:
|
||||||
|
// https://developer.hashicorp.com/consul/docs/connect/config-entries/service-resolver#failover-targets-partition
|
||||||
|
if farServerSID.Partition == "default" && nearServerSID.Partition != "default" {
|
||||||
|
part = "default"
|
||||||
|
}
|
||||||
|
targets = []api.ServiceResolverFailoverTarget{
|
||||||
|
{
|
||||||
|
Service: farServerSID.Name,
|
||||||
|
Partition: part,
|
||||||
|
Namespace: farServerSID.Namespace,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
nearClu.InitialConfigEntries = append(nearClu.InitialConfigEntries,
|
||||||
|
&api.ServiceConfigEntry{
|
||||||
|
Kind: api.ServiceDefaults,
|
||||||
|
Name: farServerSID.Name,
|
||||||
|
Partition: ConfigEntryPartition(farServerSID.Partition),
|
||||||
|
Namespace: farServerSID.Namespace,
|
||||||
|
Protocol: "http",
|
||||||
|
},
|
||||||
|
&api.ServiceResolverConfigEntry{
|
||||||
|
Kind: api.ServiceResolver,
|
||||||
|
Name: nearServerSID.Name,
|
||||||
|
Partition: ConfigEntryPartition(nearServerSID.Partition),
|
||||||
|
Namespace: nearServerSID.Namespace,
|
||||||
|
Failover: map[string]api.ServiceResolverFailover{
|
||||||
|
"*": {
|
||||||
|
Targets: targets,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
clientSID := topology.ServiceID{
|
||||||
|
Name: "ac6-client",
|
||||||
|
Partition: nearServerSID.Partition,
|
||||||
|
Namespace: nearServerSID.Namespace,
|
||||||
|
}
|
||||||
|
client := NewFortioServiceWithDefaults(
|
||||||
|
nearClu.Datacenter,
|
||||||
|
clientSID,
|
||||||
|
func(s *topology.Service) {
|
||||||
|
// Upstream per partition
|
||||||
|
s.Upstreams = []*topology.Upstream{
|
||||||
|
{
|
||||||
|
ID: topology.ServiceID{
|
||||||
|
Name: nearServerSID.Name,
|
||||||
|
Partition: nearServerSID.Partition,
|
||||||
|
Namespace: nearServerSID.Namespace,
|
||||||
|
},
|
||||||
|
LocalPort: 5000,
|
||||||
|
// exposed so we can hit it directly
|
||||||
|
// TODO: we shouldn't do this; it's not realistic
|
||||||
|
LocalAddress: "0.0.0.0",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
},
|
||||||
|
)
|
||||||
|
ct.AddServiceNode(nearClu, serviceExt{Service: client})
|
||||||
|
nearClu.InitialConfigEntries = append(nearClu.InitialConfigEntries,
|
||||||
|
&api.ServiceConfigEntry{
|
||||||
|
Kind: api.ServiceDefaults,
|
||||||
|
Name: clientSID.Name,
|
||||||
|
Partition: ConfigEntryPartition(clientSID.Partition),
|
||||||
|
Namespace: clientSID.Namespace,
|
||||||
|
Protocol: "http",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
// intentions
|
||||||
|
nearClu.InitialConfigEntries = append(nearClu.InitialConfigEntries,
|
||||||
|
&api.ServiceIntentionsConfigEntry{
|
||||||
|
Kind: api.ServiceIntentions,
|
||||||
|
Name: nearServerSID.Name,
|
||||||
|
Partition: ConfigEntryPartition(nearServerSID.Partition),
|
||||||
|
Namespace: nearServerSID.Namespace,
|
||||||
|
Sources: []*api.SourceIntention{{
|
||||||
|
Name: clientSID.Name,
|
||||||
|
Namespace: clientSID.Namespace,
|
||||||
|
// in this field, "" -> destination partition, so no ConfigEntryPartition :eyeroll:
|
||||||
|
// https://developer.hashicorp.com/consul/docs/connect/config-entries/service-intentions#sources-partition
|
||||||
|
Partition: topology.PartitionOrDefault(clientSID.Partition),
|
||||||
|
Action: api.IntentionActionAllow,
|
||||||
|
}},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
farSource := api.SourceIntention{
|
||||||
|
Name: clientSID.Name,
|
||||||
|
Namespace: clientSID.Namespace,
|
||||||
|
Peer: LocalPeerName(nearClu, clientSID.Partition),
|
||||||
|
Action: api.IntentionActionAllow,
|
||||||
|
}
|
||||||
|
if nearClu == farClu {
|
||||||
|
farSource.Peer = ""
|
||||||
|
// in this field, "" -> destination partition, so no ConfigEntryPartition :eyeroll:
|
||||||
|
// https://developer.hashicorp.com/consul/docs/connect/config-entries/service-intentions#sources-partition
|
||||||
|
farSource.Partition = topology.PartitionOrDefault(clientSID.Partition)
|
||||||
|
}
|
||||||
|
farClu.InitialConfigEntries = append(farClu.InitialConfigEntries,
|
||||||
|
&api.ServiceIntentionsConfigEntry{
|
||||||
|
Kind: api.ServiceIntentions,
|
||||||
|
Name: farServerSID.Name,
|
||||||
|
Partition: ConfigEntryPartition(farServerSID.Partition),
|
||||||
|
Namespace: farServerSID.Namespace,
|
||||||
|
Sources: []*api.SourceIntention{&farSource},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
s.clientSID = clientSID
|
||||||
|
s.nearServerSID = nearServerSID
|
||||||
|
s.farServerSID = farServerSID
|
||||||
|
s.nearServerNode = nearServerNode.ID()
|
||||||
|
s.farServerNode = farServerNode.ID()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *ac6FailoversSuite) test(t *testing.T, ct *commonTopo) {
|
||||||
|
// NOTE: *not parallel* because we mutate resources that are shared
|
||||||
|
// between test cases (disable/enable nodes)
|
||||||
|
|
||||||
|
nearClu := ct.Sprawl.Topology().Clusters["dc1"]
|
||||||
|
farClu := ct.Sprawl.Topology().Clusters["dc2"]
|
||||||
|
if s.NearInDial {
|
||||||
|
nearClu = ct.Sprawl.Topology().Clusters["dc2"]
|
||||||
|
}
|
||||||
|
if s.FarInAcc {
|
||||||
|
farClu = ct.Sprawl.Topology().Clusters["dc1"]
|
||||||
|
}
|
||||||
|
|
||||||
|
svcs := nearClu.ServicesByID(s.clientSID)
|
||||||
|
require.Len(t, svcs, 1, "expected exactly one client in datacenter")
|
||||||
|
|
||||||
|
client := svcs[0]
|
||||||
|
require.Len(t, client.Upstreams, 1, "expected one upstream for client")
|
||||||
|
upstream := client.Upstreams[0]
|
||||||
|
|
||||||
|
fmt.Println("### preconditions")
|
||||||
|
|
||||||
|
// this is the server in the same DC and partitions as client
|
||||||
|
serverSID := s.nearServerSID
|
||||||
|
serverSID.Normalize()
|
||||||
|
ct.Assert.FortioFetch2FortioName(t, client, upstream, nearClu.Name, serverSID)
|
||||||
|
|
||||||
|
ct.Assert.CatalogServiceExists(t, nearClu.Name, upstream.ID.Name, utils.CompatQueryOpts(&api.QueryOptions{
|
||||||
|
Partition: upstream.ID.Partition,
|
||||||
|
Namespace: upstream.ID.Namespace,
|
||||||
|
}))
|
||||||
|
|
||||||
|
if t.Failed() {
|
||||||
|
t.Fatal("failed preconditions")
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Println("### failover")
|
||||||
|
|
||||||
|
cfg := ct.Sprawl.Config()
|
||||||
|
DisableNode(t, cfg, nearClu.Name, s.nearServerNode)
|
||||||
|
require.NoError(t, ct.Sprawl.RelaunchWithPhase(cfg, "failover"))
|
||||||
|
// Clusters for imported services rely on outlier detection for
|
||||||
|
// failovers, NOT eds_health_status. This means that killing the
|
||||||
|
// node above does not actually make the envoy cluster UNHEALTHY
|
||||||
|
// so we do not assert for it.
|
||||||
|
expectSID := s.farServerSID
|
||||||
|
expectSID.Normalize()
|
||||||
|
ct.Assert.FortioFetch2FortioName(t, client, upstream, farClu.Name, expectSID)
|
||||||
|
}
|
||||||
|
|
|
@ -11,8 +11,6 @@ import (
|
||||||
"github.com/hashicorp/consul/api"
|
"github.com/hashicorp/consul/api"
|
||||||
"github.com/hashicorp/consul/testing/deployer/topology"
|
"github.com/hashicorp/consul/testing/deployer/topology"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
|
|
||||||
"github.com/hashicorp/consul/test-integ/topoutil"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// TestRotateGW ensures that peered services continue to be able to talk to their
|
// TestRotateGW ensures that peered services continue to be able to talk to their
|
||||||
|
@ -143,10 +141,10 @@ func (s *suiteRotateGW) setup(t *testing.T, ct *commonTopo) {
|
||||||
// add a second mesh gateway "new"
|
// add a second mesh gateway "new"
|
||||||
s.newMGWNodeName = fmt.Sprintf("new-%s-default-mgw", clu.Name)
|
s.newMGWNodeName = fmt.Sprintf("new-%s-default-mgw", clu.Name)
|
||||||
nodeKind := topology.NodeKindClient
|
nodeKind := topology.NodeKindClient
|
||||||
if clu.Datacenter == agentlessDC {
|
if clu.Datacenter == ct.agentlessDC {
|
||||||
nodeKind = topology.NodeKindDataplane
|
nodeKind = topology.NodeKindDataplane
|
||||||
}
|
}
|
||||||
clu.Nodes = append(clu.Nodes, topoutil.NewTopologyMeshGatewaySet(
|
_, mgwNodes := newTopologyMeshGatewaySet(
|
||||||
nodeKind,
|
nodeKind,
|
||||||
"default",
|
"default",
|
||||||
s.newMGWNodeName,
|
s.newMGWNodeName,
|
||||||
|
@ -155,7 +153,8 @@ func (s *suiteRotateGW) setup(t *testing.T, ct *commonTopo) {
|
||||||
func(i int, node *topology.Node) {
|
func(i int, node *topology.Node) {
|
||||||
node.Disabled = true
|
node.Disabled = true
|
||||||
},
|
},
|
||||||
)...)
|
)
|
||||||
|
clu.Nodes = append(clu.Nodes, mgwNodes...)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *suiteRotateGW) test(t *testing.T, ct *commonTopo) {
|
func (s *suiteRotateGW) test(t *testing.T, ct *commonTopo) {
|
||||||
|
|
|
@ -8,12 +8,13 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/hashicorp/consul/test/integration/consul-container/libs/utils"
|
|
||||||
"github.com/hashicorp/consul/testing/deployer/topology"
|
|
||||||
"github.com/mitchellh/copystructure"
|
"github.com/mitchellh/copystructure"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
|
|
||||||
|
"github.com/hashicorp/consul/test/integration/consul-container/libs/utils"
|
||||||
|
"github.com/hashicorp/consul/testing/deployer/topology"
|
||||||
|
|
||||||
"github.com/hashicorp/consul/api"
|
"github.com/hashicorp/consul/api"
|
||||||
"github.com/hashicorp/consul/sdk/testutil/retry"
|
"github.com/hashicorp/consul/sdk/testutil/retry"
|
||||||
)
|
)
|
||||||
|
@ -165,7 +166,7 @@ func (s *ac7_2RotateLeaderSuite) test(t *testing.T, ct *commonTopo) {
|
||||||
found := 0
|
found := 0
|
||||||
foundI := 0
|
foundI := 0
|
||||||
for i, svc := range ceAsES.Services {
|
for i, svc := range ceAsES.Services {
|
||||||
if svc.Name == s.sidServer.Name && svc.Namespace == utils.DefaultToEmpty(s.sidServer.Namespace) {
|
if svc.Name == s.sidServer.Name && utils.DefaultToEmpty(svc.Namespace) == utils.DefaultToEmpty(s.sidServer.Namespace) {
|
||||||
found += 1
|
found += 1
|
||||||
foundI = i
|
foundI = i
|
||||||
}
|
}
|
||||||
|
@ -176,7 +177,7 @@ func (s *ac7_2RotateLeaderSuite) test(t *testing.T, ct *commonTopo) {
|
||||||
_, _, err = clPeer.ConfigEntries().Set(ceAsES, nil)
|
_, _, err = clPeer.ConfigEntries().Set(ceAsES, nil)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
t.Cleanup(func() {
|
t.Cleanup(func() {
|
||||||
//restore for next pairing
|
// restore for next pairing
|
||||||
_, _, err = clPeer.ConfigEntries().Set(origCE.(*api.ExportedServicesConfigEntry), nil)
|
_, _, err = clPeer.ConfigEntries().Set(origCE.(*api.ExportedServicesConfigEntry), nil)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
})
|
})
|
||||||
|
|
|
@ -40,6 +40,7 @@ type commonTopo struct {
|
||||||
// shortcuts to corresponding entry in Cfg
|
// shortcuts to corresponding entry in Cfg
|
||||||
DC1 *topology.Cluster
|
DC1 *topology.Cluster
|
||||||
DC2 *topology.Cluster
|
DC2 *topology.Cluster
|
||||||
|
// nil if includeDC3 is false
|
||||||
DC3 *topology.Cluster
|
DC3 *topology.Cluster
|
||||||
|
|
||||||
// set after Launch. Should be considered read-only
|
// set after Launch. Should be considered read-only
|
||||||
|
@ -48,66 +49,101 @@ type commonTopo struct {
|
||||||
|
|
||||||
// track per-DC services to prevent duplicates
|
// track per-DC services to prevent duplicates
|
||||||
services map[string]map[topology.ServiceID]struct{}
|
services map[string]map[topology.ServiceID]struct{}
|
||||||
|
|
||||||
|
// if zero, no DCs are agentless
|
||||||
|
agentlessDC string
|
||||||
|
|
||||||
|
// if true, create DC3 and associated links (currently only used by ac5.2)
|
||||||
|
includeDC3 bool
|
||||||
|
|
||||||
|
peerThroughMGW bool
|
||||||
}
|
}
|
||||||
|
|
||||||
const agentlessDC = "dc2"
|
func NewCommonTopoWithoutAgentless(t *testing.T) *commonTopo {
|
||||||
|
t.Helper()
|
||||||
|
return newCommonTopo(t, "", false, true)
|
||||||
|
}
|
||||||
|
|
||||||
func NewCommonTopo(t *testing.T) *commonTopo {
|
func NewCommonTopo(t *testing.T) *commonTopo {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
return newCommonTopo(t, "dc2", false, true)
|
||||||
|
}
|
||||||
|
|
||||||
ct := commonTopo{}
|
func newCommonTopo(t *testing.T, agentlessDC string, includeDC3 bool, peerThroughMGW bool) *commonTopo {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
ct := commonTopo{
|
||||||
|
agentlessDC: agentlessDC,
|
||||||
|
includeDC3: includeDC3,
|
||||||
|
peerThroughMGW: peerThroughMGW,
|
||||||
|
}
|
||||||
|
|
||||||
const nServers = 3
|
const nServers = 3
|
||||||
|
|
||||||
// Make 3-server clusters in dc1 and dc2
|
// Make 3-server clusters in dc1 and dc2
|
||||||
// For simplicity, the Name and Datacenter of the clusters are the same.
|
// For simplicity, the Name and Datacenter of the clusters are the same.
|
||||||
// dc1 and dc2 should be symmetric.
|
// dc1 and dc2 should be symmetric.
|
||||||
dc1 := clusterWithJustServers("dc1", nServers)
|
dc1 := ct.clusterWithJustServers("dc1", nServers)
|
||||||
ct.DC1 = dc1
|
ct.DC1 = dc1
|
||||||
dc2 := clusterWithJustServers("dc2", nServers)
|
dc2 := ct.clusterWithJustServers("dc2", nServers)
|
||||||
ct.DC2 = dc2
|
ct.DC2 = dc2
|
||||||
// dc3 is a failover cluster for both dc1 and dc2
|
clusters := []*topology.Cluster{dc1, dc2}
|
||||||
dc3 := clusterWithJustServers("dc3", 1)
|
|
||||||
// dc3 is only used for certain failover scenarios and does not need tenancies
|
var dc3 *topology.Cluster
|
||||||
dc3.Partitions = []*topology.Partition{{Name: "default"}}
|
|
||||||
ct.DC3 = dc3
|
if ct.includeDC3 {
|
||||||
|
// dc3 is a failover cluster for both dc1 and dc2
|
||||||
|
dc3 = ct.clusterWithJustServers("dc3", 1)
|
||||||
|
// dc3 is only used for certain failover scenarios and does not need tenancies
|
||||||
|
dc3.Partitions = []*topology.Partition{{Name: "default"}}
|
||||||
|
ct.DC3 = dc3
|
||||||
|
// dc3 is only used for certain failover scenarios and does not need tenancies
|
||||||
|
dc3.Partitions = []*topology.Partition{{Name: "default"}}
|
||||||
|
|
||||||
|
clusters = append(clusters, dc3)
|
||||||
|
}
|
||||||
|
|
||||||
injectTenancies(dc1)
|
injectTenancies(dc1)
|
||||||
injectTenancies(dc2)
|
injectTenancies(dc2)
|
||||||
// dc3 is only used for certain failover scenarios and does not need tenancies
|
// dc3 doesn't get tenancies
|
||||||
dc3.Partitions = []*topology.Partition{{Name: "default"}}
|
|
||||||
|
|
||||||
ct.services = map[string]map[topology.ServiceID]struct{}{}
|
ct.services = map[string]map[topology.ServiceID]struct{}{}
|
||||||
for _, dc := range []*topology.Cluster{dc1, dc2, dc3} {
|
for _, dc := range clusters {
|
||||||
ct.services[dc.Datacenter] = map[topology.ServiceID]struct{}{}
|
ct.services[dc.Datacenter] = map[topology.ServiceID]struct{}{}
|
||||||
}
|
}
|
||||||
|
|
||||||
peerings := addPeerings(dc1, dc2)
|
peerings := addPeerings(dc1, dc2)
|
||||||
peerings = append(peerings, addPeerings(dc1, dc3)...)
|
if ct.includeDC3 {
|
||||||
peerings = append(peerings, addPeerings(dc2, dc3)...)
|
peerings = append(peerings, addPeerings(dc1, dc3)...)
|
||||||
|
peerings = append(peerings, addPeerings(dc2, dc3)...)
|
||||||
|
}
|
||||||
|
|
||||||
addMeshGateways(dc1)
|
ct.addMeshGateways(dc1)
|
||||||
addMeshGateways(dc2)
|
ct.addMeshGateways(dc2)
|
||||||
addMeshGateways(dc3)
|
if ct.includeDC3 {
|
||||||
|
ct.addMeshGateways(dc3)
|
||||||
|
}
|
||||||
|
|
||||||
setupGlobals(dc1)
|
ct.setupGlobals(dc1)
|
||||||
setupGlobals(dc2)
|
ct.setupGlobals(dc2)
|
||||||
setupGlobals(dc3)
|
if ct.includeDC3 {
|
||||||
|
ct.setupGlobals(dc3)
|
||||||
|
}
|
||||||
|
|
||||||
|
networks := []*topology.Network{
|
||||||
|
{Name: "wan", Type: "wan"},
|
||||||
|
{Name: dc1.Datacenter}, // "dc1" LAN
|
||||||
|
{Name: dc2.Datacenter}, // "dc2" LAN
|
||||||
|
}
|
||||||
|
if ct.includeDC3 {
|
||||||
|
networks = append(networks, &topology.Network{Name: dc3.Datacenter})
|
||||||
|
}
|
||||||
|
|
||||||
// Build final configuration
|
// Build final configuration
|
||||||
ct.Cfg = &topology.Config{
|
ct.Cfg = &topology.Config{
|
||||||
Images: utils.TargetImages(),
|
Images: utils.TargetImages(),
|
||||||
Networks: []*topology.Network{
|
Networks: networks,
|
||||||
{Name: dc1.Datacenter}, // "dc1" LAN
|
Clusters: clusters,
|
||||||
{Name: dc2.Datacenter}, // "dc2" LAN
|
|
||||||
{Name: dc3.Datacenter}, // "dc3" LAN
|
|
||||||
{Name: "wan", Type: "wan"},
|
|
||||||
},
|
|
||||||
Clusters: []*topology.Cluster{
|
|
||||||
dc1,
|
|
||||||
dc2,
|
|
||||||
dc3,
|
|
||||||
},
|
|
||||||
Peerings: peerings,
|
Peerings: peerings,
|
||||||
}
|
}
|
||||||
return &ct
|
return &ct
|
||||||
|
@ -142,10 +178,12 @@ func (ct *commonTopo) postLaunchChecks(t *testing.T) {
|
||||||
for _, e := range clu.InitialConfigEntries {
|
for _, e := range clu.InitialConfigEntries {
|
||||||
if e.GetKind() == api.ExportedServices {
|
if e.GetKind() == api.ExportedServices {
|
||||||
asExport := e.(*api.ExportedServicesConfigEntry)
|
asExport := e.(*api.ExportedServicesConfigEntry)
|
||||||
// do we care about the partition?
|
|
||||||
for _, svc := range asExport.Services {
|
for _, svc := range asExport.Services {
|
||||||
for _, con := range svc.Consumers {
|
for _, con := range svc.Consumers {
|
||||||
// do we care about con.Partition?
|
// if Peer is unset, this is an export to another partition in the same DC, so we don't need to check it
|
||||||
|
if con.Peer == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
// TODO: surely there is code to normalize this
|
// TODO: surely there is code to normalize this
|
||||||
partition := asExport.Partition
|
partition := asExport.Partition
|
||||||
if partition == "" {
|
if partition == "" {
|
||||||
|
@ -183,6 +221,9 @@ func (ct *commonTopo) postLaunchChecks(t *testing.T) {
|
||||||
// PeerName is how you'd address a remote dc+partition locally
|
// PeerName is how you'd address a remote dc+partition locally
|
||||||
// as your peer name.
|
// as your peer name.
|
||||||
func LocalPeerName(clu *topology.Cluster, partition string) string {
|
func LocalPeerName(clu *topology.Cluster, partition string) string {
|
||||||
|
if partition == "" {
|
||||||
|
partition = "default"
|
||||||
|
}
|
||||||
return fmt.Sprintf("peer-%s-%s", clu.Datacenter, partition)
|
return fmt.Sprintf("peer-%s-%s", clu.Datacenter, partition)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -227,7 +268,7 @@ func (ct *commonTopo) AddServiceNode(clu *topology.Cluster, svc serviceExt) *top
|
||||||
nodeKind := topology.NodeKindClient
|
nodeKind := topology.NodeKindClient
|
||||||
// TODO: bug in deployer somewhere; it should guard against a KindDataplane node with
|
// TODO: bug in deployer somewhere; it should guard against a KindDataplane node with
|
||||||
// DisableServiceMesh services on it; dataplane is only for service-mesh
|
// DisableServiceMesh services on it; dataplane is only for service-mesh
|
||||||
if !svc.DisableServiceMesh && clu.Datacenter == agentlessDC {
|
if !svc.DisableServiceMesh && clu.Datacenter == ct.agentlessDC {
|
||||||
nodeKind = topology.NodeKindDataplane
|
nodeKind = topology.NodeKindDataplane
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -306,13 +347,9 @@ func (ct *commonTopo) ClusterByDatacenter(t *testing.T, name string) *topology.C
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Since CE config entries do not contain the partition field,
|
// Deprecated: topoutil.ConfigEntryPartition
|
||||||
// this func converts default partition to empty string.
|
|
||||||
func ConfigEntryPartition(p string) string {
|
func ConfigEntryPartition(p string) string {
|
||||||
if p == "default" {
|
return topoutil.ConfigEntryPartition(p)
|
||||||
return "" // make this CE friendly
|
|
||||||
}
|
|
||||||
return p
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// DisableNode is a no-op if the node is already disabled.
|
// DisableNode is a no-op if the node is already disabled.
|
||||||
|
@ -335,7 +372,7 @@ func EnableNode(t *testing.T, cfg *topology.Config, clusterName string, nid topo
|
||||||
return cfg
|
return cfg
|
||||||
}
|
}
|
||||||
|
|
||||||
func setupGlobals(clu *topology.Cluster) {
|
func (ct *commonTopo) setupGlobals(clu *topology.Cluster) {
|
||||||
for _, part := range clu.Partitions {
|
for _, part := range clu.Partitions {
|
||||||
clu.InitialConfigEntries = append(clu.InitialConfigEntries,
|
clu.InitialConfigEntries = append(clu.InitialConfigEntries,
|
||||||
&api.ProxyConfigEntry{
|
&api.ProxyConfigEntry{
|
||||||
|
@ -349,35 +386,63 @@ func setupGlobals(clu *topology.Cluster) {
|
||||||
Mode: api.MeshGatewayModeLocal,
|
Mode: api.MeshGatewayModeLocal,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
&api.MeshConfigEntry{
|
|
||||||
Peering: &api.PeeringMeshConfig{
|
|
||||||
PeerThroughMeshGateways: true,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
)
|
)
|
||||||
|
if ct.peerThroughMGW {
|
||||||
|
clu.InitialConfigEntries = append(clu.InitialConfigEntries,
|
||||||
|
&api.MeshConfigEntry{
|
||||||
|
Peering: &api.PeeringMeshConfig{
|
||||||
|
PeerThroughMeshGateways: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// addMeshGateways adds a mesh gateway for every partition in the cluster.
|
// addMeshGateways adds a mesh gateway for every partition in the cluster.
|
||||||
// Assumes that the LAN network name is equal to datacenter name.
|
// Assumes that the LAN network name is equal to datacenter name.
|
||||||
func addMeshGateways(c *topology.Cluster) {
|
func (ct *commonTopo) addMeshGateways(c *topology.Cluster) {
|
||||||
nodeKind := topology.NodeKindClient
|
nodeKind := topology.NodeKindClient
|
||||||
if c.Datacenter == agentlessDC {
|
if c.Datacenter == ct.agentlessDC {
|
||||||
nodeKind = topology.NodeKindDataplane
|
nodeKind = topology.NodeKindDataplane
|
||||||
}
|
}
|
||||||
for _, p := range c.Partitions {
|
for _, p := range c.Partitions {
|
||||||
c.Nodes = topology.MergeSlices(c.Nodes, topoutil.NewTopologyMeshGatewaySet(
|
sid, nodes := newTopologyMeshGatewaySet(
|
||||||
nodeKind,
|
nodeKind,
|
||||||
p.Name,
|
p.Name,
|
||||||
fmt.Sprintf("%s-%s-mgw", c.Name, p.Name),
|
fmt.Sprintf("%s-%s-mgw", c.Name, p.Name),
|
||||||
1,
|
1,
|
||||||
[]string{c.Datacenter, "wan"},
|
[]string{c.Datacenter, "wan"},
|
||||||
nil,
|
nil,
|
||||||
))
|
)
|
||||||
|
c.Nodes = topology.MergeSlices(c.Nodes, nodes)
|
||||||
|
// for services exported in the same cluster between partitions, we need
|
||||||
|
// to export the mesh gateway (but not for peering)
|
||||||
|
// https://github.com/hashicorp/consul/pull/19052
|
||||||
|
consumers := []api.ServiceConsumer{}
|
||||||
|
for _, cp := range c.Partitions {
|
||||||
|
if cp.Name == p.Name {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
consumers = append(consumers, api.ServiceConsumer{
|
||||||
|
Partition: cp.Name,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if len(consumers) > 0 {
|
||||||
|
ct.ExportService(c, p.Name, api.ExportedService{
|
||||||
|
Name: sid.Name,
|
||||||
|
Namespace: sid.Namespace,
|
||||||
|
Consumers: consumers,
|
||||||
|
})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func clusterWithJustServers(name string, numServers int) *topology.Cluster {
|
func (ct *commonTopo) clusterWithJustServers(name string, numServers int) *topology.Cluster {
|
||||||
|
nets := []string{name}
|
||||||
|
if !ct.peerThroughMGW {
|
||||||
|
nets = append(nets, "wan")
|
||||||
|
}
|
||||||
return &topology.Cluster{
|
return &topology.Cluster{
|
||||||
Enterprise: utils.IsEnterprise(),
|
Enterprise: utils.IsEnterprise(),
|
||||||
Name: name,
|
Name: name,
|
||||||
|
@ -385,7 +450,7 @@ func clusterWithJustServers(name string, numServers int) *topology.Cluster {
|
||||||
Nodes: topoutil.NewTopologyServerSet(
|
Nodes: topoutil.NewTopologyServerSet(
|
||||||
name+"-server",
|
name+"-server",
|
||||||
numServers,
|
numServers,
|
||||||
[]string{name},
|
nets,
|
||||||
nil,
|
nil,
|
||||||
),
|
),
|
||||||
}
|
}
|
||||||
|
@ -446,3 +511,16 @@ func NewFortioServiceWithDefaults(
|
||||||
) *topology.Service {
|
) *topology.Service {
|
||||||
return topoutil.NewFortioServiceWithDefaults(cluster, sid, topology.NodeVersionV1, mut)
|
return topoutil.NewFortioServiceWithDefaults(cluster, sid, topology.NodeVersionV1, mut)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func newTopologyMeshGatewaySet(
|
||||||
|
nodeKind topology.NodeKind,
|
||||||
|
partition string,
|
||||||
|
namePrefix string,
|
||||||
|
num int,
|
||||||
|
networks []string,
|
||||||
|
mutateFn func(i int, node *topology.Node),
|
||||||
|
) (topology.ServiceID, []*topology.Node) {
|
||||||
|
nodes := topoutil.NewTopologyMeshGatewaySet(nodeKind, partition, namePrefix, num, networks, mutateFn)
|
||||||
|
sid := nodes[0].Services[0].ID
|
||||||
|
return sid, nodes
|
||||||
|
}
|
||||||
|
|
|
@ -198,24 +198,6 @@ func (a *Asserter) HealthyWithPeer(t *testing.T, cluster string, sid topology.Se
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func (a *Asserter) UpstreamEndpointHealthy(t *testing.T, svc *topology.Service, upstream *topology.Upstream) {
|
|
||||||
t.Helper()
|
|
||||||
node := svc.Node
|
|
||||||
ip := node.LocalAddress()
|
|
||||||
port := svc.EnvoyAdminPort
|
|
||||||
addr := fmt.Sprintf("%s:%d", ip, port)
|
|
||||||
|
|
||||||
client := a.mustGetHTTPClient(t, node.Cluster)
|
|
||||||
libassert.AssertUpstreamEndpointStatusWithClient(t,
|
|
||||||
client,
|
|
||||||
addr,
|
|
||||||
// TODO: what is default? namespace? partition?
|
|
||||||
fmt.Sprintf("%s.default.%s.external", upstream.ID.Name, upstream.Peer),
|
|
||||||
"HEALTHY",
|
|
||||||
1,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
type testingT interface {
|
type testingT interface {
|
||||||
require.TestingT
|
require.TestingT
|
||||||
Helper()
|
Helper()
|
||||||
|
|
|
@ -96,6 +96,10 @@ func NewTopologyMeshGatewaySet(
|
||||||
mutateFn func(i int, node *topology.Node),
|
mutateFn func(i int, node *topology.Node),
|
||||||
) []*topology.Node {
|
) []*topology.Node {
|
||||||
var out []*topology.Node
|
var out []*topology.Node
|
||||||
|
sid := topology.ServiceID{
|
||||||
|
Name: "mesh-gateway",
|
||||||
|
Partition: ConfigEntryPartition(partition),
|
||||||
|
}
|
||||||
for i := 1; i <= num; i++ {
|
for i := 1; i <= num; i++ {
|
||||||
name := namePrefix + strconv.Itoa(i)
|
name := namePrefix + strconv.Itoa(i)
|
||||||
|
|
||||||
|
@ -104,7 +108,7 @@ func NewTopologyMeshGatewaySet(
|
||||||
Partition: partition,
|
Partition: partition,
|
||||||
Name: name,
|
Name: name,
|
||||||
Services: []*topology.Service{{
|
Services: []*topology.Service{{
|
||||||
ID: topology.ServiceID{Name: "mesh-gateway"},
|
ID: sid,
|
||||||
Port: 8443,
|
Port: 8443,
|
||||||
EnvoyAdminPort: 19000,
|
EnvoyAdminPort: 19000,
|
||||||
IsMeshGateway: true,
|
IsMeshGateway: true,
|
||||||
|
@ -122,3 +126,12 @@ func NewTopologyMeshGatewaySet(
|
||||||
}
|
}
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Since CE config entries do not contain the partition field,
|
||||||
|
// this func converts default partition to empty string.
|
||||||
|
func ConfigEntryPartition(p string) string {
|
||||||
|
if p == "default" {
|
||||||
|
return "" // make this CE friendly
|
||||||
|
}
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
|
|
@ -16,13 +16,13 @@ import (
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
goretry "github.com/avast/retry-go"
|
goretry "github.com/avast/retry-go"
|
||||||
"github.com/hashicorp/consul/api"
|
|
||||||
"github.com/hashicorp/consul/sdk/testutil/retry"
|
|
||||||
"github.com/hashicorp/serf/serf"
|
"github.com/hashicorp/serf/serf"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"github.com/teris-io/shortid"
|
"github.com/teris-io/shortid"
|
||||||
"github.com/testcontainers/testcontainers-go"
|
"github.com/testcontainers/testcontainers-go"
|
||||||
|
|
||||||
|
"github.com/hashicorp/consul/api"
|
||||||
|
"github.com/hashicorp/consul/sdk/testutil/retry"
|
||||||
"github.com/hashicorp/consul/test/integration/consul-container/libs/utils"
|
"github.com/hashicorp/consul/test/integration/consul-container/libs/utils"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
@ -4,17 +4,19 @@
|
||||||
package ratelimit
|
package ratelimit
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
|
|
||||||
"github.com/hashicorp/consul/api"
|
"github.com/hashicorp/consul/api"
|
||||||
"github.com/hashicorp/consul/sdk/testutil/retry"
|
"github.com/hashicorp/consul/sdk/testutil/retry"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
|
||||||
libcluster "github.com/hashicorp/consul/test/integration/consul-container/libs/cluster"
|
libcluster "github.com/hashicorp/consul/test/integration/consul-container/libs/cluster"
|
||||||
libtopology "github.com/hashicorp/consul/test/integration/consul-container/libs/topology"
|
libtopology "github.com/hashicorp/consul/test/integration/consul-container/libs/topology"
|
||||||
|
"github.com/hashicorp/consul/test/integration/consul-container/libs/utils"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
|
@ -45,10 +47,11 @@ func TestServerRequestRateLimit(t *testing.T) {
|
||||||
expectMetric bool
|
expectMetric bool
|
||||||
}
|
}
|
||||||
type testCase struct {
|
type testCase struct {
|
||||||
description string
|
description string
|
||||||
cmd string
|
cmd string
|
||||||
operations []operation
|
operations []operation
|
||||||
mode string
|
mode string
|
||||||
|
enterpriseOnly bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// getKV and putKV are net/RPC calls
|
// getKV and putKV are net/RPC calls
|
||||||
|
@ -69,6 +72,30 @@ func TestServerRequestRateLimit(t *testing.T) {
|
||||||
rateLimitType: "global/write",
|
rateLimitType: "global/write",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// listPartition and putPartition are gRPC calls
|
||||||
|
listPartition := action{
|
||||||
|
function: func(client *api.Client) error {
|
||||||
|
ctx := context.Background()
|
||||||
|
_, _, err := client.Partitions().List(ctx, nil)
|
||||||
|
return err
|
||||||
|
},
|
||||||
|
rateLimitOperation: "/partition.PartitionService/List",
|
||||||
|
rateLimitType: "global/read",
|
||||||
|
}
|
||||||
|
|
||||||
|
putPartition := action{
|
||||||
|
function: func(client *api.Client) error {
|
||||||
|
ctx := context.Background()
|
||||||
|
p := api.Partition{
|
||||||
|
Name: "ptest",
|
||||||
|
}
|
||||||
|
_, _, err := client.Partitions().Create(ctx, &p, nil)
|
||||||
|
return err
|
||||||
|
},
|
||||||
|
rateLimitOperation: "/partition.PartitionService/Write",
|
||||||
|
rateLimitType: "global/write",
|
||||||
|
}
|
||||||
|
|
||||||
testCases := []testCase{
|
testCases := []testCase{
|
||||||
// HTTP & net/RPC
|
// HTTP & net/RPC
|
||||||
{
|
{
|
||||||
|
@ -128,9 +155,73 @@ func TestServerRequestRateLimit(t *testing.T) {
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
// gRPC
|
||||||
|
{
|
||||||
|
description: "GRPC / Mode: disabled - errors: no / exceeded logs: no / metrics: no",
|
||||||
|
cmd: `-hcl=limits { request_limits { mode = "disabled" read_rate = 0 write_rate = 0 }}`,
|
||||||
|
mode: "disabled",
|
||||||
|
operations: []operation{
|
||||||
|
{
|
||||||
|
action: putPartition,
|
||||||
|
expectedErrorMsg: "",
|
||||||
|
expectExceededLog: false,
|
||||||
|
expectMetric: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
action: listPartition,
|
||||||
|
expectedErrorMsg: "",
|
||||||
|
expectExceededLog: false,
|
||||||
|
expectMetric: false,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
enterpriseOnly: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
description: "GRPC / Mode: permissive - errors: no / exceeded logs: yes / metrics: no",
|
||||||
|
cmd: `-hcl=limits { request_limits { mode = "permissive" read_rate = 0 write_rate = 0 }}`,
|
||||||
|
mode: "permissive",
|
||||||
|
operations: []operation{
|
||||||
|
{
|
||||||
|
action: putPartition,
|
||||||
|
expectedErrorMsg: "",
|
||||||
|
expectExceededLog: true,
|
||||||
|
expectMetric: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
action: listPartition,
|
||||||
|
expectedErrorMsg: "",
|
||||||
|
expectExceededLog: true,
|
||||||
|
expectMetric: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
enterpriseOnly: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
description: "GRPC / Mode: enforcing - errors: yes / exceeded logs: yes / metrics: yes",
|
||||||
|
cmd: `-hcl=limits { request_limits { mode = "enforcing" read_rate = 0 write_rate = 0 }}`,
|
||||||
|
mode: "enforcing",
|
||||||
|
operations: []operation{
|
||||||
|
{
|
||||||
|
action: putPartition,
|
||||||
|
expectedErrorMsg: nonRetryableErrorMsg,
|
||||||
|
expectExceededLog: true,
|
||||||
|
expectMetric: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
action: listPartition,
|
||||||
|
expectedErrorMsg: retryableErrorMsg,
|
||||||
|
expectExceededLog: true,
|
||||||
|
expectMetric: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
enterpriseOnly: true,
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tc := range testCases {
|
for _, tc := range testCases {
|
||||||
|
if tc.enterpriseOnly && !utils.IsEnterprise() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
tc := tc
|
tc := tc
|
||||||
t.Run(tc.description, func(t *testing.T) {
|
t.Run(tc.description, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
|
@ -98,23 +98,26 @@ func DockerImages(
|
||||||
run *runner.Runner,
|
run *runner.Runner,
|
||||||
t *topology.Topology,
|
t *topology.Topology,
|
||||||
) error {
|
) error {
|
||||||
logw := logger.Named("docker").StandardWriter(&hclog.StandardLoggerOptions{ForceLevel: hclog.Debug})
|
|
||||||
|
|
||||||
built := make(map[string]struct{})
|
built := make(map[string]struct{})
|
||||||
for _, c := range t.Clusters {
|
for _, c := range t.Clusters {
|
||||||
for _, n := range c.Nodes {
|
for _, n := range c.Nodes {
|
||||||
joint := n.Images.EnvoyConsulImage()
|
joint := n.Images.EnvoyConsulImage()
|
||||||
if _, ok := built[joint]; joint != "" && !ok {
|
if _, ok := built[joint]; joint != "" && !ok {
|
||||||
logger.Info("building image", "image", joint)
|
logger.Info("building envoy+consul image", "image", joint)
|
||||||
err := run.DockerExec(context.TODO(), []string{
|
logw := logger.Named("docker_envoy_consul").StandardWriter(&hclog.StandardLoggerOptions{ForceLevel: hclog.Debug})
|
||||||
|
|
||||||
|
err := run.DockerExecWithStderr(context.TODO(), []string{
|
||||||
"build",
|
"build",
|
||||||
|
// provenance causes non-idempotent builds, which leads to spurious terraform replacements
|
||||||
|
"--provenance=false",
|
||||||
"--build-arg",
|
"--build-arg",
|
||||||
"CONSUL_IMAGE=" + n.Images.Consul,
|
"CONSUL_IMAGE=" + n.Images.Consul,
|
||||||
"--build-arg",
|
"--build-arg",
|
||||||
"ENVOY_IMAGE=" + n.Images.Envoy,
|
"ENVOY_IMAGE=" + n.Images.Envoy,
|
||||||
"-t", joint,
|
"-t", joint,
|
||||||
"-",
|
"-",
|
||||||
}, logw, strings.NewReader(dockerfileEnvoy))
|
}, logw, logw, strings.NewReader(dockerfileEnvoy))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
@ -124,14 +127,16 @@ func DockerImages(
|
||||||
|
|
||||||
cdp := n.Images.LocalDataplaneImage()
|
cdp := n.Images.LocalDataplaneImage()
|
||||||
if _, ok := built[cdp]; cdp != "" && !ok {
|
if _, ok := built[cdp]; cdp != "" && !ok {
|
||||||
logger.Info("building image", "image", cdp)
|
logger.Info("building dataplane image", "image", cdp)
|
||||||
err := run.DockerExec(context.TODO(), []string{
|
logw := logger.Named("docker_dataplane").StandardWriter(&hclog.StandardLoggerOptions{ForceLevel: hclog.Debug})
|
||||||
|
err := run.DockerExecWithStderr(context.TODO(), []string{
|
||||||
"build",
|
"build",
|
||||||
|
"--provenance=false",
|
||||||
"--build-arg",
|
"--build-arg",
|
||||||
"DATAPLANE_IMAGE=" + n.Images.Dataplane,
|
"DATAPLANE_IMAGE=" + n.Images.Dataplane,
|
||||||
"-t", cdp,
|
"-t", cdp,
|
||||||
"-",
|
"-",
|
||||||
}, logw, strings.NewReader(dockerfileDataplane))
|
}, logw, logw, strings.NewReader(dockerfileDataplane))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
@ -142,7 +147,8 @@ func DockerImages(
|
||||||
cdpTproxy := n.Images.LocalDataplaneTProxyImage()
|
cdpTproxy := n.Images.LocalDataplaneTProxyImage()
|
||||||
if _, ok := built[cdpTproxy]; cdpTproxy != "" && !ok {
|
if _, ok := built[cdpTproxy]; cdpTproxy != "" && !ok {
|
||||||
logger.Info("building image", "image", cdpTproxy)
|
logger.Info("building image", "image", cdpTproxy)
|
||||||
err := run.DockerExec(context.TODO(), []string{
|
logw := logger.Named("docker_dataplane_tproxy").StandardWriter(&hclog.StandardLoggerOptions{ForceLevel: hclog.Debug})
|
||||||
|
err := run.DockerExecWithStderr(context.TODO(), []string{
|
||||||
"build",
|
"build",
|
||||||
"--build-arg",
|
"--build-arg",
|
||||||
"DATAPLANE_IMAGE=" + n.Images.Dataplane,
|
"DATAPLANE_IMAGE=" + n.Images.Dataplane,
|
||||||
|
@ -150,7 +156,7 @@ func DockerImages(
|
||||||
"CONSUL_IMAGE=" + n.Images.Consul,
|
"CONSUL_IMAGE=" + n.Images.Consul,
|
||||||
"-t", cdpTproxy,
|
"-t", cdpTproxy,
|
||||||
"-",
|
"-",
|
||||||
}, logw, strings.NewReader(dockerfileDataplaneForTProxy))
|
}, logw, logw, strings.NewReader(dockerfileDataplaneForTProxy))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
|
@ -7,6 +7,7 @@ import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"log"
|
||||||
"net/http"
|
"net/http"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
@ -62,6 +63,8 @@ func (s *Sprawl) initPeerings() error {
|
||||||
req1.Partition = peering.Accepting.Partition
|
req1.Partition = peering.Accepting.Partition
|
||||||
}
|
}
|
||||||
|
|
||||||
|
s.awaitMeshGateways()
|
||||||
|
|
||||||
GENTOKEN:
|
GENTOKEN:
|
||||||
resp, _, err := acceptingClient.Peerings().GenerateToken(context.Background(), req1, nil)
|
resp, _, err := acceptingClient.Peerings().GenerateToken(context.Background(), req1, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -108,6 +111,7 @@ func (s *Sprawl) initPeerings() error {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Sprawl) waitForPeeringEstablishment() error {
|
func (s *Sprawl) waitForPeeringEstablishment() error {
|
||||||
|
s.awaitMeshGateways()
|
||||||
var (
|
var (
|
||||||
logger = s.logger.Named("peering")
|
logger = s.logger.Named("peering")
|
||||||
)
|
)
|
||||||
|
@ -181,3 +185,64 @@ func (s *Sprawl) checkPeeringDirection(logger hclog.Logger, client *api.Client,
|
||||||
}
|
}
|
||||||
logger.Debug("peering is active", "dur", time.Since(startTime).Round(time.Second))
|
logger.Debug("peering is active", "dur", time.Since(startTime).Round(time.Second))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *Sprawl) awaitMeshGateways() {
|
||||||
|
startTime := time.Now()
|
||||||
|
s.logger.Info("awaiting mesh gateways")
|
||||||
|
// TODO: maybe a better way to do this
|
||||||
|
mgws := []*topology.Service{}
|
||||||
|
for _, clu := range s.topology.Clusters {
|
||||||
|
for _, node := range clu.Nodes {
|
||||||
|
for _, svc := range node.Services {
|
||||||
|
if svc.IsMeshGateway {
|
||||||
|
mgws = append(mgws, svc)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TODO: parallel
|
||||||
|
for _, mgw := range mgws {
|
||||||
|
cl := s.clients[mgw.Node.Cluster]
|
||||||
|
logger := s.logger.With("cluster", mgw.Node.Cluster, "sid", mgw.ID, "nid", mgw.Node.ID())
|
||||||
|
logger.Info("awaiting MGW readiness")
|
||||||
|
RETRY:
|
||||||
|
// TODO: not sure if there's a better way to check if the MGW is ready
|
||||||
|
svcs, _, err := cl.Catalog().Service(mgw.ID.Name, "", &api.QueryOptions{
|
||||||
|
Namespace: mgw.ID.Namespace,
|
||||||
|
Partition: mgw.ID.Partition,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
logger.Debug("fetching MGW service", "err", err)
|
||||||
|
time.Sleep(time.Second)
|
||||||
|
goto RETRY
|
||||||
|
}
|
||||||
|
if len(svcs) < 1 {
|
||||||
|
logger.Debug("no MGW service in catalog yet")
|
||||||
|
time.Sleep(time.Second)
|
||||||
|
goto RETRY
|
||||||
|
}
|
||||||
|
if len(svcs) > 1 {
|
||||||
|
// not sure when this would happen
|
||||||
|
log.Fatalf("expected 1 MGW service, actually: %#v", svcs)
|
||||||
|
}
|
||||||
|
|
||||||
|
entries, _, err := cl.Health().Service(mgw.ID.Name, "", true, &api.QueryOptions{
|
||||||
|
Namespace: mgw.ID.Namespace,
|
||||||
|
Partition: mgw.ID.Partition,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
logger.Debug("fetching MGW checks", "err", err)
|
||||||
|
time.Sleep(time.Second)
|
||||||
|
goto RETRY
|
||||||
|
}
|
||||||
|
if len(entries) != 1 {
|
||||||
|
logger.Debug("expected 1 MGW entry", "entries", entries)
|
||||||
|
time.Sleep(time.Second)
|
||||||
|
goto RETRY
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.Debug("MGW ready", "entry", *(entries[0]), "dur", time.Since(startTime).Round(time.Second))
|
||||||
|
}
|
||||||
|
s.logger.Info("mesh gateways ready", "dur", time.Since(startTime).Round(time.Second))
|
||||||
|
}
|
||||||
|
|
|
@ -33,10 +33,12 @@ import (
|
||||||
|
|
||||||
// Sprawl is the definition of a complete running Consul deployment topology.
|
// Sprawl is the definition of a complete running Consul deployment topology.
|
||||||
type Sprawl struct {
|
type Sprawl struct {
|
||||||
logger hclog.Logger
|
logger hclog.Logger
|
||||||
runner *runner.Runner
|
// set after initial Launch is complete
|
||||||
license string
|
launchLogger hclog.Logger
|
||||||
secrets secrets.Store
|
runner *runner.Runner
|
||||||
|
license string
|
||||||
|
secrets secrets.Store
|
||||||
|
|
||||||
workdir string
|
workdir string
|
||||||
|
|
||||||
|
@ -212,11 +214,20 @@ func Launch(
|
||||||
return nil, fmt.Errorf("error gathering diagnostic details: %w", err)
|
return nil, fmt.Errorf("error gathering diagnostic details: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
s.launchLogger = s.logger
|
||||||
|
|
||||||
return s, nil
|
return s, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Sprawl) Relaunch(
|
func (s *Sprawl) Relaunch(
|
||||||
cfg *topology.Config,
|
cfg *topology.Config,
|
||||||
|
) error {
|
||||||
|
return s.RelaunchWithPhase(cfg, "")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Sprawl) RelaunchWithPhase(
|
||||||
|
cfg *topology.Config,
|
||||||
|
phase string,
|
||||||
) error {
|
) error {
|
||||||
// Copy this BEFORE compiling so we capture the original definition, without denorms.
|
// Copy this BEFORE compiling so we capture the original definition, without denorms.
|
||||||
var err error
|
var err error
|
||||||
|
@ -225,6 +236,10 @@ func (s *Sprawl) Relaunch(
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if phase != "" {
|
||||||
|
s.logger = s.launchLogger.Named(phase)
|
||||||
|
}
|
||||||
|
|
||||||
newTopology, err := topology.Recompile(s.logger.Named("recompile"), cfg, s.topology)
|
newTopology, err := topology.Recompile(s.logger.Named("recompile"), cfg, s.topology)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("topology.Compile: %w", err)
|
return fmt.Errorf("topology.Compile: %w", err)
|
||||||
|
|
Loading…
Reference in New Issue