2023-03-28 18:39:22 +00:00
// Copyright (c) HashiCorp, Inc.
2023-08-11 13:12:13 +00:00
// SPDX-License-Identifier: BUSL-1.1
2023-03-28 18:39:22 +00:00
2021-04-29 18:54:05 +00:00
package xds
import (
2021-06-15 20:21:07 +00:00
"errors"
2023-07-31 19:24:33 +00:00
"fmt"
2023-01-31 20:50:30 +00:00
"strconv"
2021-09-21 14:58:56 +00:00
"strings"
2023-02-24 22:00:31 +00:00
"sync"
2021-04-29 18:54:05 +00:00
"sync/atomic"
"testing"
"time"
2023-01-31 20:50:30 +00:00
"github.com/armon/go-metrics"
2023-07-31 19:24:33 +00:00
envoy_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
envoy_listener_v3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
2021-04-29 18:54:05 +00:00
envoy_discovery_v3 "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
2023-06-08 14:26:11 +00:00
"github.com/hashicorp/go-hclog"
goversion "github.com/hashicorp/go-version"
2021-04-29 18:54:05 +00:00
"github.com/stretchr/testify/require"
2021-06-15 20:21:07 +00:00
rpcstatus "google.golang.org/genproto/googleapis/rpc/status"
2021-04-29 18:54:05 +00:00
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
2023-01-11 14:39:10 +00:00
"google.golang.org/protobuf/proto"
2021-04-29 18:54:05 +00:00
"github.com/hashicorp/consul/acl"
2022-09-09 14:02:01 +00:00
"github.com/hashicorp/consul/agent/grpc-external/limiter"
2021-04-29 18:54:05 +00:00
"github.com/hashicorp/consul/agent/proxycfg"
"github.com/hashicorp/consul/agent/structs"
2023-02-23 19:52:18 +00:00
"github.com/hashicorp/consul/api"
2023-06-08 14:26:11 +00:00
"github.com/hashicorp/consul/envoyextensions/extensioncommon"
2023-02-06 17:14:35 +00:00
"github.com/hashicorp/consul/envoyextensions/xdscommon"
2022-05-10 20:25:51 +00:00
"github.com/hashicorp/consul/sdk/testutil"
2023-02-23 19:52:18 +00:00
"github.com/hashicorp/consul/sdk/testutil/retry"
2023-01-31 20:50:30 +00:00
"github.com/hashicorp/consul/version"
2021-04-29 18:54:05 +00:00
)
// NOTE: For these tests, prefer not using xDS protobuf "factory" methods if
// possible to avoid using them to test themselves.
//
// Stick to very straightforward stuff in xds_protocol_helpers_test.go.
func TestServer_DeltaAggregatedResources_v3_BasicProtocol_TCP ( t * testing . T ) {
2022-12-08 19:46:42 +00:00
aclResolve := func ( id string ) ( acl . Authorizer , error ) {
// Allow all
return acl . RootAuthorizer ( "manage" ) , nil
}
2023-01-18 18:33:21 +00:00
scenario := newTestServerDeltaScenario ( t , aclResolve , "web-sidecar-proxy" , "" , 0 )
2022-12-08 19:46:42 +00:00
mgr , errCh , envoy := scenario . mgr , scenario . errCh , scenario . envoy
2021-04-29 18:54:05 +00:00
2022-12-08 19:46:42 +00:00
sid := structs . NewServiceID ( "web-sidecar-proxy" , nil )
2021-04-29 18:54:05 +00:00
2022-12-08 19:46:42 +00:00
// Register the proxy to create state needed to Watch() on
mgr . RegisterProxy ( t , sid )
2021-04-29 18:54:05 +00:00
2022-12-08 19:46:42 +00:00
var snap * proxycfg . ConfigSnapshot
2021-04-29 18:54:05 +00:00
2022-12-08 19:46:42 +00:00
testutil . RunStep ( t , "initial setup" , func ( t * testing . T ) {
2023-05-23 11:55:06 +00:00
snap = newTestSnapshot ( t , nil , "" ,
func ( ns * structs . NodeService ) {
// Add extension for local proxy.
ns . Proxy . EnvoyExtensions = [ ] structs . EnvoyExtension {
{
Name : api . BuiltinLuaExtension ,
Arguments : map [ string ] interface { } {
"ProxyType" : "connect-proxy" ,
"Listener" : "inbound" ,
"Script" : "x = 0" ,
} ,
2023-01-31 20:50:30 +00:00
} ,
2023-05-23 11:55:06 +00:00
}
} )
2021-04-29 18:54:05 +00:00
2022-12-08 19:46:42 +00:00
// Send initial cluster discover. We'll assume we are testing a partial
// reconnect and include some initial resource versions that will be
// cleaned up.
envoy . SendDeltaReq ( t , xdscommon . ClusterType , & envoy_discovery_v3 . DeltaDiscoveryRequest {
InitialResourceVersions : mustMakeVersionMap ( t ,
makeTestCluster ( t , snap , "tcp:geo-cache" ) ,
) ,
} )
2021-05-14 18:59:13 +00:00
2022-12-08 19:46:42 +00:00
// Check no response sent yet
assertDeltaChanBlocked ( t , envoy . deltaStream . sendCh )
2021-04-29 18:54:05 +00:00
2022-12-08 19:46:42 +00:00
requireProtocolVersionGauge ( t , scenario , "v3" , 1 )
2021-04-29 18:54:05 +00:00
2022-12-08 19:46:42 +00:00
// Deliver a new snapshot (tcp with one tcp upstream)
mgr . DeliverConfig ( t , sid , snap )
} )
2021-04-29 18:54:05 +00:00
2022-12-08 19:46:42 +00:00
testutil . RunStep ( t , "first sync" , func ( t * testing . T ) {
assertDeltaResponseSent ( t , envoy . deltaStream . sendCh , & envoy_discovery_v3 . DeltaDiscoveryResponse {
TypeUrl : xdscommon . ClusterType ,
Nonce : hexString ( 1 ) ,
Resources : makeTestResources ( t ,
makeTestCluster ( t , snap , "tcp:local_app" ) ,
makeTestCluster ( t , snap , "tcp:db" ) ,
// SAME_AS_INITIAL_VERSION: makeTestCluster(t, snap, "tcp:geo-cache"),
) ,
} )
2021-04-29 18:54:05 +00:00
2022-12-08 19:46:42 +00:00
// Envoy then tries to discover endpoints for those clusters.
envoy . SendDeltaReq ( t , xdscommon . EndpointType , & envoy_discovery_v3 . DeltaDiscoveryRequest {
// We'll assume we are testing a partial "reconnect"
InitialResourceVersions : mustMakeVersionMap ( t ,
makeTestEndpoints ( t , snap , "tcp:geo-cache" ) ,
) ,
ResourceNamesSubscribe : [ ] string {
"db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" ,
// "geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul",
//
// Include "fake-endpoints" here to test subscribing to an unknown
// thing and have consul tell us there's no data for it.
"fake-endpoints" ,
} ,
} )
2021-04-29 18:54:05 +00:00
2022-12-08 19:46:42 +00:00
// We should get a response immediately since the config is already present in
// the server for endpoints. Note that this should not be racy if the server
// is behaving well since the Cluster send above should be blocked until we
// deliver a new config version.
assertDeltaResponseSent ( t , envoy . deltaStream . sendCh , & envoy_discovery_v3 . DeltaDiscoveryResponse {
TypeUrl : xdscommon . EndpointType ,
Nonce : hexString ( 2 ) ,
Resources : makeTestResources ( t ,
makeTestEndpoints ( t , snap , "tcp:db" ) ,
// SAME_AS_INITIAL_VERSION: makeTestEndpoints(t, snap, "tcp:geo-cache"),
// SAME_AS_INITIAL_VERSION: "fake-endpoints",
) ,
} )
2021-04-29 18:54:05 +00:00
2022-12-08 19:46:42 +00:00
// After receiving the endpoints Envoy sends an ACK for the cluster
envoy . SendDeltaReqACK ( t , xdscommon . ClusterType , 1 )
2022-11-08 01:10:42 +00:00
2022-12-08 19:46:42 +00:00
// We are caught up, so there should be nothing queued to send.
assertDeltaChanBlocked ( t , envoy . deltaStream . sendCh )
2021-04-29 18:54:05 +00:00
2022-12-08 19:46:42 +00:00
// Envoy now sends listener request
envoy . SendDeltaReq ( t , xdscommon . ListenerType , nil )
2021-04-29 18:54:05 +00:00
2022-12-08 19:46:42 +00:00
// It also (in parallel) issues the endpoint ACK
envoy . SendDeltaReqACK ( t , xdscommon . EndpointType , 2 )
2021-06-15 20:21:07 +00:00
2022-12-08 19:46:42 +00:00
// And should get a response immediately.
assertDeltaResponseSent ( t , envoy . deltaStream . sendCh , & envoy_discovery_v3 . DeltaDiscoveryResponse {
TypeUrl : xdscommon . ListenerType ,
Nonce : hexString ( 3 ) ,
Resources : makeTestResources ( t ,
makeTestListener ( t , snap , "tcp:public_listener" ) ,
makeTestListener ( t , snap , "tcp:db" ) ,
makeTestListener ( t , snap , "tcp:geo-cache" ) ,
) ,
} )
2021-06-15 20:21:07 +00:00
2022-12-08 19:46:42 +00:00
// We are caught up, so there should be nothing queued to send.
assertDeltaChanBlocked ( t , envoy . deltaStream . sendCh )
2021-06-15 20:21:07 +00:00
2022-12-08 19:46:42 +00:00
// ACKs the listener
envoy . SendDeltaReqACK ( t , xdscommon . ListenerType , 3 )
2021-06-15 20:21:07 +00:00
2022-12-08 19:46:42 +00:00
// If Envoy re-subscribes to something even if there are no changes we send a
// fresh copy.
envoy . SendDeltaReq ( t , xdscommon . EndpointType , & envoy_discovery_v3 . DeltaDiscoveryRequest {
ResourceNamesSubscribe : [ ] string {
"geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul" ,
} ,
} )
2021-06-15 20:21:07 +00:00
2022-12-08 19:46:42 +00:00
assertDeltaResponseSent ( t , envoy . deltaStream . sendCh , & envoy_discovery_v3 . DeltaDiscoveryResponse {
TypeUrl : xdscommon . EndpointType ,
Nonce : hexString ( 4 ) ,
Resources : makeTestResources ( t ,
makeTestEndpoints ( t , snap , "tcp:geo-cache" ) ,
) ,
} )
2021-06-15 20:21:07 +00:00
2022-12-08 19:46:42 +00:00
envoy . SendDeltaReqACK ( t , xdscommon . EndpointType , 4 )
2021-04-29 18:54:05 +00:00
2022-12-08 19:46:42 +00:00
// We are caught up, so there should be nothing queued to send.
assertDeltaChanBlocked ( t , envoy . deltaStream . sendCh )
2023-01-31 20:50:30 +00:00
requireExtensionMetrics ( t , scenario , api . BuiltinLuaExtension , sid , nil )
2022-12-08 19:46:42 +00:00
} )
2021-04-29 18:54:05 +00:00
2022-12-08 19:46:42 +00:00
deleteAllButOneEndpoint := func ( snap * proxycfg . ConfigSnapshot , uid proxycfg . UpstreamID , targetID string ) {
snap . ConnectProxy . ConfigSnapshotUpstreams . WatchedUpstreamEndpoints [ uid ] [ targetID ] =
snap . ConnectProxy . ConfigSnapshotUpstreams . WatchedUpstreamEndpoints [ uid ] [ targetID ] [ 0 : 1 ]
}
2021-06-15 20:21:07 +00:00
2022-12-08 19:46:42 +00:00
testutil . RunStep ( t , "avoid sending config for unsubscribed resource" , func ( t * testing . T ) {
envoy . SendDeltaReq ( t , xdscommon . EndpointType , & envoy_discovery_v3 . DeltaDiscoveryRequest {
ResourceNamesUnsubscribe : [ ] string {
"db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" ,
} ,
} )
2021-04-29 18:54:05 +00:00
2022-12-08 19:46:42 +00:00
assertDeltaChanBlocked ( t , envoy . deltaStream . sendCh )
2021-04-29 18:54:05 +00:00
2022-12-08 19:46:42 +00:00
// now reconfigure the snapshot and JUST edit the endpoints to strike one of the two current endpoints for db.
2023-05-23 11:55:06 +00:00
snap = newTestSnapshot ( t , snap , "" , nil )
2022-12-08 19:46:42 +00:00
deleteAllButOneEndpoint ( snap , UID ( "db" ) , "db.default.default.dc1" )
mgr . DeliverConfig ( t , sid , snap )
2021-04-29 18:54:05 +00:00
2022-12-08 19:46:42 +00:00
// We never send an EDS reply about this change because Envoy is not subscribed to db anymore.
assertDeltaChanBlocked ( t , envoy . deltaStream . sendCh )
} )
2021-06-15 20:21:07 +00:00
2022-12-08 19:46:42 +00:00
testutil . RunStep ( t , "restore endpoint subscription" , func ( t * testing . T ) {
// Restore db's deleted endpoints by generating a new snapshot.
2023-05-23 11:55:06 +00:00
snap = newTestSnapshot ( t , snap , "" , nil )
2022-12-08 19:46:42 +00:00
mgr . DeliverConfig ( t , sid , snap )
2021-06-16 16:57:43 +00:00
2022-12-08 19:46:42 +00:00
// We never send an EDS reply about this change because Envoy is still not subscribed to db.
assertDeltaChanBlocked ( t , envoy . deltaStream . sendCh )
2021-06-15 20:21:07 +00:00
2022-12-08 19:46:42 +00:00
// When Envoy re-subscribes to db we send the endpoints for it.
envoy . SendDeltaReq ( t , xdscommon . EndpointType , & envoy_discovery_v3 . DeltaDiscoveryRequest {
ResourceNamesSubscribe : [ ] string {
"db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" ,
} ,
} )
assertDeltaResponseSent ( t , envoy . deltaStream . sendCh , & envoy_discovery_v3 . DeltaDiscoveryResponse {
TypeUrl : xdscommon . EndpointType ,
Nonce : hexString ( 5 ) ,
Resources : makeTestResources ( t ,
makeTestEndpoints ( t , snap , "tcp:db" ) ,
) ,
} )
2021-06-15 20:21:07 +00:00
2022-12-08 19:46:42 +00:00
envoy . SendDeltaReqACK ( t , xdscommon . EndpointType , 5 )
2021-04-29 18:54:05 +00:00
2022-12-08 19:46:42 +00:00
assertDeltaChanBlocked ( t , envoy . deltaStream . sendCh )
} )
2022-02-10 22:37:36 +00:00
2022-12-08 19:46:42 +00:00
// NOTE: this has to be the last subtest since it kills the stream
testutil . RunStep ( t , "simulate an envoy error sending an update to envoy" , func ( t * testing . T ) {
// Force sends to fail
envoy . SetSendErr ( errors . New ( "test error" ) )
2021-04-29 18:54:05 +00:00
2022-12-08 19:46:42 +00:00
// Trigger only an EDS update by deleting endpoints again.
deleteAllButOneEndpoint ( snap , UID ( "db" ) , "db.default.default.dc1" )
2022-02-10 22:37:36 +00:00
2022-12-08 19:46:42 +00:00
// We never send any replies about this change because we died.
assertDeltaChanBlocked ( t , envoy . deltaStream . sendCh )
} )
2022-03-15 14:07:40 +00:00
2022-12-08 19:46:42 +00:00
envoy . Close ( )
select {
case err := <- errCh :
require . NoError ( t , err )
case <- time . After ( 50 * time . Millisecond ) :
t . Fatalf ( "timed out waiting for handler to finish" )
2022-02-10 22:37:36 +00:00
}
}
func TestServer_DeltaAggregatedResources_v3_NackLoop ( t * testing . T ) {
aclResolve := func ( id string ) ( acl . Authorizer , error ) {
// Allow all
return acl . RootAuthorizer ( "manage" ) , nil
}
2023-01-18 18:33:21 +00:00
scenario := newTestServerDeltaScenario ( t , aclResolve , "web-sidecar-proxy" , "" , 0 )
2022-02-10 22:37:36 +00:00
mgr , errCh , envoy := scenario . mgr , scenario . errCh , scenario . envoy
sid := structs . NewServiceID ( "web-sidecar-proxy" , nil )
// Register the proxy to create state needed to Watch() on
mgr . RegisterProxy ( t , sid )
var snap * proxycfg . ConfigSnapshot
2022-05-10 20:25:51 +00:00
testutil . RunStep ( t , "initial setup" , func ( t * testing . T ) {
2023-05-23 11:55:06 +00:00
snap = newTestSnapshot ( t , nil , "" , nil )
2022-02-10 22:37:36 +00:00
// Plug in a bad port for the public listener
snap . Port = 1
// Send initial cluster discover.
2022-03-08 19:37:24 +00:00
envoy . SendDeltaReq ( t , xdscommon . ClusterType , & envoy_discovery_v3 . DeltaDiscoveryRequest { } )
2022-02-10 22:37:36 +00:00
// Check no response sent yet
assertDeltaChanBlocked ( t , envoy . deltaStream . sendCh )
requireProtocolVersionGauge ( t , scenario , "v3" , 1 )
// Deliver a new snapshot (tcp with one tcp upstream)
mgr . DeliverConfig ( t , sid , snap )
} )
2022-11-08 01:10:42 +00:00
testutil . RunStep ( t , "simulate Envoy NACKing initial listener" , func ( t * testing . T ) {
2021-06-15 20:21:07 +00:00
assertDeltaResponseSent ( t , envoy . deltaStream . sendCh , & envoy_discovery_v3 . DeltaDiscoveryResponse {
2022-03-08 19:37:24 +00:00
TypeUrl : xdscommon . ClusterType ,
2022-02-10 22:37:36 +00:00
Nonce : hexString ( 1 ) ,
2021-06-15 20:21:07 +00:00
Resources : makeTestResources ( t ,
2022-02-10 22:37:36 +00:00
makeTestCluster ( t , snap , "tcp:local_app" ) ,
makeTestCluster ( t , snap , "tcp:db" ) ,
makeTestCluster ( t , snap , "tcp:geo-cache" ) ,
2021-06-15 20:21:07 +00:00
) ,
} )
2021-04-29 18:54:05 +00:00
2022-02-10 22:37:36 +00:00
// Envoy then tries to discover endpoints for those clusters.
2022-03-08 19:37:24 +00:00
envoy . SendDeltaReq ( t , xdscommon . EndpointType , & envoy_discovery_v3 . DeltaDiscoveryRequest {
2022-02-10 22:37:36 +00:00
ResourceNamesSubscribe : [ ] string {
"db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" ,
"geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul" ,
} ,
} )
// We should get a response immediately since the config is already present in
// the server for endpoints. Note that this should not be racy if the server
// is behaving well since the Cluster send above should be blocked until we
// deliver a new config version.
2021-06-15 20:21:07 +00:00
assertDeltaResponseSent ( t , envoy . deltaStream . sendCh , & envoy_discovery_v3 . DeltaDiscoveryResponse {
2022-03-08 19:37:24 +00:00
TypeUrl : xdscommon . EndpointType ,
2022-02-10 22:37:36 +00:00
Nonce : hexString ( 2 ) ,
Resources : makeTestResources ( t ,
makeTestEndpoints ( t , snap , "tcp:db" ) ,
makeTestEndpoints ( t , snap , "tcp:geo-cache" ) ,
) ,
} )
2022-11-08 01:10:42 +00:00
// After receiving the endpoints Envoy sends an ACK for the clusters
envoy . SendDeltaReqACK ( t , xdscommon . ClusterType , 1 )
// We are caught up, so there should be nothing queued to send.
2022-02-10 22:37:36 +00:00
assertDeltaChanBlocked ( t , envoy . deltaStream . sendCh )
// Envoy now sends listener request
2022-03-08 19:37:24 +00:00
envoy . SendDeltaReq ( t , xdscommon . ListenerType , nil )
2022-02-10 22:37:36 +00:00
// It also (in parallel) issues the endpoint ACK
2022-03-08 19:37:24 +00:00
envoy . SendDeltaReqACK ( t , xdscommon . EndpointType , 2 )
2022-02-10 22:37:36 +00:00
// And should get a response immediately.
assertDeltaResponseSent ( t , envoy . deltaStream . sendCh , & envoy_discovery_v3 . DeltaDiscoveryResponse {
2022-03-08 19:37:24 +00:00
TypeUrl : xdscommon . ListenerType ,
2022-02-10 22:37:36 +00:00
Nonce : hexString ( 3 ) ,
2021-06-15 20:21:07 +00:00
Resources : makeTestResources ( t ,
2022-02-10 22:37:36 +00:00
// Response contains public_listener with port that Envoy can't bind to
makeTestListener ( t , snap , "tcp:bad_public_listener" ) ,
makeTestListener ( t , snap , "tcp:db" ) ,
makeTestListener ( t , snap , "tcp:geo-cache" ) ,
2021-06-15 20:21:07 +00:00
) ,
} )
2022-11-08 01:10:42 +00:00
// We are caught up, so there should be nothing queued to send.
2022-02-10 22:37:36 +00:00
assertDeltaChanBlocked ( t , envoy . deltaStream . sendCh )
2022-11-08 01:10:42 +00:00
// Envoy NACKs the listener update due to the bad public listener
2022-03-08 19:37:24 +00:00
envoy . SendDeltaReqNACK ( t , xdscommon . ListenerType , 3 , & rpcstatus . Status { } )
2021-06-15 20:21:07 +00:00
2022-02-10 22:37:36 +00:00
// Consul should not respond until a new snapshot is delivered
2022-11-08 01:10:42 +00:00
// because the current snapshot is known to be bad.
2021-06-15 20:21:07 +00:00
assertDeltaChanBlocked ( t , envoy . deltaStream . sendCh )
} )
2022-05-10 20:25:51 +00:00
testutil . RunStep ( t , "simulate envoy NACKing a listener update" , func ( t * testing . T ) {
2022-02-10 22:37:36 +00:00
// Correct the port and deliver a new snapshot
snap . Port = 9999
2021-06-15 20:21:07 +00:00
mgr . DeliverConfig ( t , sid , snap )
2021-06-16 16:57:43 +00:00
2022-02-10 22:37:36 +00:00
// And should send a response immediately.
assertDeltaResponseSent ( t , envoy . deltaStream . sendCh , & envoy_discovery_v3 . DeltaDiscoveryResponse {
2022-03-08 19:37:24 +00:00
TypeUrl : xdscommon . ListenerType ,
2022-02-10 22:37:36 +00:00
Nonce : hexString ( 4 ) ,
Resources : makeTestResources ( t ,
// Send a public listener that Envoy will accept
makeTestListener ( t , snap , "tcp:public_listener" ) ,
makeTestListener ( t , snap , "tcp:db" ) ,
makeTestListener ( t , snap , "tcp:geo-cache" ) ,
) ,
} )
// New listener is acked now
2022-03-08 19:37:24 +00:00
envoy . SendDeltaReqACK ( t , xdscommon . EndpointType , 4 )
2022-02-10 22:37:36 +00:00
2021-06-16 16:57:43 +00:00
assertDeltaChanBlocked ( t , envoy . deltaStream . sendCh )
2021-06-15 20:21:07 +00:00
} )
2021-04-29 18:54:05 +00:00
envoy . Close ( )
select {
case err := <- errCh :
2022-02-10 22:37:36 +00:00
require . NoError ( t , err )
2021-04-29 18:54:05 +00:00
case <- time . After ( 50 * time . Millisecond ) :
t . Fatalf ( "timed out waiting for handler to finish" )
}
}
func TestServer_DeltaAggregatedResources_v3_BasicProtocol_HTTP2 ( t * testing . T ) {
aclResolve := func ( id string ) ( acl . Authorizer , error ) {
// Allow all
return acl . RootAuthorizer ( "manage" ) , nil
}
2023-01-18 18:33:21 +00:00
scenario := newTestServerDeltaScenario ( t , aclResolve , "web-sidecar-proxy" , "" , 0 )
2021-04-29 18:54:05 +00:00
mgr , errCh , envoy := scenario . mgr , scenario . errCh , scenario . envoy
sid := structs . NewServiceID ( "web-sidecar-proxy" , nil )
// Register the proxy to create state needed to Watch() on
mgr . RegisterProxy ( t , sid )
// Send initial cluster discover (empty payload)
2022-03-08 19:37:24 +00:00
envoy . SendDeltaReq ( t , xdscommon . ClusterType , nil )
2021-04-29 18:54:05 +00:00
// Check no response sent yet
assertDeltaChanBlocked ( t , envoy . deltaStream . sendCh )
// Deliver a new snapshot (tcp with one http upstream)
2023-05-23 11:55:06 +00:00
snap := newTestSnapshot ( t , nil , "http2" , nil , & structs . ServiceConfigEntry {
2021-04-29 18:54:05 +00:00
Kind : structs . ServiceDefaults ,
Name : "db" ,
Protocol : "http2" ,
} )
mgr . DeliverConfig ( t , sid , snap )
2022-05-10 20:25:51 +00:00
testutil . RunStep ( t , "no-rds" , func ( t * testing . T ) {
2021-04-29 18:54:05 +00:00
assertDeltaResponseSent ( t , envoy . deltaStream . sendCh , & envoy_discovery_v3 . DeltaDiscoveryResponse {
2022-03-08 19:37:24 +00:00
TypeUrl : xdscommon . ClusterType ,
2021-04-29 18:54:05 +00:00
Nonce : hexString ( 1 ) ,
Resources : makeTestResources ( t ,
makeTestCluster ( t , snap , "tcp:local_app" ) ,
makeTestCluster ( t , snap , "http2:db" ) ,
makeTestCluster ( t , snap , "tcp:geo-cache" ) ,
) ,
} )
// Envoy then tries to discover endpoints for those clusters.
2022-03-08 19:37:24 +00:00
envoy . SendDeltaReq ( t , xdscommon . EndpointType , & envoy_discovery_v3 . DeltaDiscoveryRequest {
2021-04-29 18:54:05 +00:00
ResourceNamesSubscribe : [ ] string {
"db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" ,
"geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul" ,
} ,
} )
// We should get a response immediately since the config is already present in
// the server for endpoints. Note that this should not be racy if the server
// is behaving well since the Cluster send above should be blocked until we
// deliver a new config version.
assertDeltaResponseSent ( t , envoy . deltaStream . sendCh , & envoy_discovery_v3 . DeltaDiscoveryResponse {
2022-03-08 19:37:24 +00:00
TypeUrl : xdscommon . EndpointType ,
2021-04-29 18:54:05 +00:00
Nonce : hexString ( 2 ) ,
Resources : makeTestResources ( t ,
makeTestEndpoints ( t , snap , "http2:db" ) ,
makeTestEndpoints ( t , snap , "tcp:geo-cache" ) ,
) ,
} )
2022-11-08 01:10:42 +00:00
// After receiving the endpoints Envoy sends an ACK for the clusters
envoy . SendDeltaReqACK ( t , xdscommon . ClusterType , 1 )
// We are caught up, so there should be nothing queued to send.
2021-04-29 18:54:05 +00:00
assertDeltaChanBlocked ( t , envoy . deltaStream . sendCh )
// Envoy now sends listener request
2022-03-08 19:37:24 +00:00
envoy . SendDeltaReq ( t , xdscommon . ListenerType , nil )
2021-04-29 18:54:05 +00:00
// It also (in parallel) issues the endpoint ACK
2022-03-08 19:37:24 +00:00
envoy . SendDeltaReqACK ( t , xdscommon . EndpointType , 2 )
2021-04-29 18:54:05 +00:00
// And should get a response immediately.
assertDeltaResponseSent ( t , envoy . deltaStream . sendCh , & envoy_discovery_v3 . DeltaDiscoveryResponse {
2022-03-08 19:37:24 +00:00
TypeUrl : xdscommon . ListenerType ,
2021-04-29 18:54:05 +00:00
Nonce : hexString ( 3 ) ,
Resources : makeTestResources ( t ,
makeTestListener ( t , snap , "tcp:public_listener" ) ,
makeTestListener ( t , snap , "http2:db" ) ,
makeTestListener ( t , snap , "tcp:geo-cache" ) ,
) ,
} )
2022-11-08 01:10:42 +00:00
// We are caught up, so there should be nothing queued to send.
2021-04-29 18:54:05 +00:00
assertDeltaChanBlocked ( t , envoy . deltaStream . sendCh )
// ACKs the listener
2022-03-08 19:37:24 +00:00
envoy . SendDeltaReqACK ( t , xdscommon . ListenerType , 3 )
2021-04-29 18:54:05 +00:00
2022-11-08 01:10:42 +00:00
// We are caught up, so there should be nothing queued to send.
2021-04-29 18:54:05 +00:00
assertDeltaChanBlocked ( t , envoy . deltaStream . sendCh )
} )
// -- reconfigure with a no-op discovery chain
2023-05-23 11:55:06 +00:00
snap = newTestSnapshot ( t , snap , "http2" , nil , & structs . ServiceConfigEntry {
2021-04-29 18:54:05 +00:00
Kind : structs . ServiceDefaults ,
Name : "db" ,
Protocol : "http2" ,
} , & structs . ServiceRouterConfigEntry {
Kind : structs . ServiceRouter ,
Name : "db" ,
Routes : nil ,
} )
mgr . DeliverConfig ( t , sid , snap )
2022-05-10 20:25:51 +00:00
testutil . RunStep ( t , "with-rds" , func ( t * testing . T ) {
2021-04-29 18:54:05 +00:00
// Just the "db" listener sees a change
assertDeltaResponseSent ( t , envoy . deltaStream . sendCh , & envoy_discovery_v3 . DeltaDiscoveryResponse {
2022-03-08 19:37:24 +00:00
TypeUrl : xdscommon . ListenerType ,
2021-04-29 18:54:05 +00:00
Nonce : hexString ( 4 ) ,
Resources : makeTestResources ( t ,
makeTestListener ( t , snap , "http2:db:rds" ) ,
) ,
} )
2022-11-08 01:10:42 +00:00
// We are caught up, so there should be nothing queued to send.
2021-04-29 18:54:05 +00:00
assertDeltaChanBlocked ( t , envoy . deltaStream . sendCh )
// Envoy now sends routes request
2022-03-08 19:37:24 +00:00
envoy . SendDeltaReq ( t , xdscommon . RouteType , & envoy_discovery_v3 . DeltaDiscoveryRequest {
2021-04-29 18:54:05 +00:00
ResourceNamesSubscribe : [ ] string {
"db" ,
} ,
} )
// And should get a response immediately.
assertDeltaResponseSent ( t , envoy . deltaStream . sendCh , & envoy_discovery_v3 . DeltaDiscoveryResponse {
2022-03-08 19:37:24 +00:00
TypeUrl : xdscommon . RouteType ,
2021-04-29 18:54:05 +00:00
Nonce : hexString ( 5 ) ,
Resources : makeTestResources ( t ,
makeTestRoute ( t , "http2:db" ) ,
) ,
} )
2022-11-08 01:10:42 +00:00
// After receiving the routes, Envoy sends acks back for the listener and routes.
envoy . SendDeltaReqACK ( t , xdscommon . ListenerType , 4 )
2022-03-08 19:37:24 +00:00
envoy . SendDeltaReqACK ( t , xdscommon . RouteType , 5 )
2021-04-29 18:54:05 +00:00
assertDeltaChanBlocked ( t , envoy . deltaStream . sendCh )
} )
envoy . Close ( )
select {
case err := <- errCh :
require . NoError ( t , err )
case <- time . After ( 50 * time . Millisecond ) :
t . Fatalf ( "timed out waiting for handler to finish" )
}
}
2021-09-21 14:58:56 +00:00
func TestServer_DeltaAggregatedResources_v3_SlowEndpointPopulation ( t * testing . T ) {
// This illustrates a scenario related to https://github.com/hashicorp/consul/issues/10563
aclResolve := func ( id string ) ( acl . Authorizer , error ) {
// Allow all
return acl . RootAuthorizer ( "manage" ) , nil
}
2023-01-18 18:33:21 +00:00
scenario := newTestServerDeltaScenario ( t , aclResolve , "web-sidecar-proxy" , "" , 0 )
2021-09-21 14:58:56 +00:00
server , mgr , errCh , envoy := scenario . server , scenario . mgr , scenario . errCh , scenario . envoy
// This mutateFn causes any endpoint with a name containing "geo-cache" to be
// omitted from the response while the hack is active.
var slowHackDisabled uint32
2022-03-08 19:37:24 +00:00
server . ResourceMapMutateFn = func ( resourceMap * xdscommon . IndexedResources ) {
2021-09-21 14:58:56 +00:00
if atomic . LoadUint32 ( & slowHackDisabled ) == 1 {
return
}
2022-03-08 19:37:24 +00:00
if em , ok := resourceMap . Index [ xdscommon . EndpointType ] ; ok {
2021-09-21 14:58:56 +00:00
for k := range em {
if strings . Contains ( k , "geo-cache" ) {
delete ( em , k )
}
}
}
}
sid := structs . NewServiceID ( "web-sidecar-proxy" , nil )
// Register the proxy to create state needed to Watch() on
mgr . RegisterProxy ( t , sid )
var snap * proxycfg . ConfigSnapshot
2022-05-10 20:25:51 +00:00
testutil . RunStep ( t , "get into initial state" , func ( t * testing . T ) {
2023-05-23 11:55:06 +00:00
snap = newTestSnapshot ( t , nil , "" , nil )
2021-09-21 14:58:56 +00:00
// Send initial cluster discover.
2022-03-08 19:37:24 +00:00
envoy . SendDeltaReq ( t , xdscommon . ClusterType , & envoy_discovery_v3 . DeltaDiscoveryRequest { } )
2021-09-21 14:58:56 +00:00
// Check no response sent yet
assertDeltaChanBlocked ( t , envoy . deltaStream . sendCh )
requireProtocolVersionGauge ( t , scenario , "v3" , 1 )
// Deliver a new snapshot (tcp with one tcp upstream)
mgr . DeliverConfig ( t , sid , snap )
assertDeltaResponseSent ( t , envoy . deltaStream . sendCh , & envoy_discovery_v3 . DeltaDiscoveryResponse {
2022-03-08 19:37:24 +00:00
TypeUrl : xdscommon . ClusterType ,
2021-09-21 14:58:56 +00:00
Nonce : hexString ( 1 ) ,
Resources : makeTestResources ( t ,
makeTestCluster ( t , snap , "tcp:local_app" ) ,
makeTestCluster ( t , snap , "tcp:db" ) ,
makeTestCluster ( t , snap , "tcp:geo-cache" ) ,
) ,
} )
// Envoy then tries to discover endpoints for those clusters.
2022-03-08 19:37:24 +00:00
envoy . SendDeltaReq ( t , xdscommon . EndpointType , & envoy_discovery_v3 . DeltaDiscoveryRequest {
2021-09-21 14:58:56 +00:00
ResourceNamesSubscribe : [ ] string {
"db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" ,
"geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul" ,
} ,
} )
// We should get a response immediately since the config is already present in
// the server for endpoints. Note that this should not be racy if the server
// is behaving well since the Cluster send above should be blocked until we
// deliver a new config version.
//
// NOTE: we do NOT return back geo-cache yet
assertDeltaResponseSent ( t , envoy . deltaStream . sendCh , & envoy_discovery_v3 . DeltaDiscoveryResponse {
2022-03-08 19:37:24 +00:00
TypeUrl : xdscommon . EndpointType ,
2021-09-21 14:58:56 +00:00
Nonce : hexString ( 2 ) ,
Resources : makeTestResources ( t ,
makeTestEndpoints ( t , snap , "tcp:db" ) ,
// makeTestEndpoints(t, snap, "tcp:geo-cache"),
) ,
} )
2022-11-08 01:10:42 +00:00
// After receiving the endpoints Envoy sends an ACK for the clusters.
// Envoy aims to wait to receive endpoints before ACKing clusters,
// but because it received an update for at least one of the clusters it cares about
// then it will ACK despite not having received an update for all clusters.
// This behavior was observed against Envoy v1.21 and v1.23.
envoy . SendDeltaReqACK ( t , xdscommon . ClusterType , 1 )
// We are caught up, so there should be nothing queued to send.
2021-09-21 14:58:56 +00:00
assertDeltaChanBlocked ( t , envoy . deltaStream . sendCh )
// Envoy now sends listener request
2022-03-08 19:37:24 +00:00
envoy . SendDeltaReq ( t , xdscommon . ListenerType , nil )
2021-09-21 14:58:56 +00:00
// It also (in parallel) issues the endpoint ACK
2022-03-08 19:37:24 +00:00
envoy . SendDeltaReqACK ( t , xdscommon . EndpointType , 2 )
2021-09-21 14:58:56 +00:00
// And should get a response immediately.
assertDeltaResponseSent ( t , envoy . deltaStream . sendCh , & envoy_discovery_v3 . DeltaDiscoveryResponse {
2022-03-08 19:37:24 +00:00
TypeUrl : xdscommon . ListenerType ,
2021-09-21 14:58:56 +00:00
Nonce : hexString ( 3 ) ,
Resources : makeTestResources ( t ,
makeTestListener ( t , snap , "tcp:public_listener" ) ,
makeTestListener ( t , snap , "tcp:db" ) ,
makeTestListener ( t , snap , "tcp:geo-cache" ) ,
) ,
} )
2022-11-08 01:10:42 +00:00
// We are caught up, so there should be nothing queued to send.
2021-09-21 14:58:56 +00:00
assertDeltaChanBlocked ( t , envoy . deltaStream . sendCh )
// ACKs the listener
2022-03-08 19:37:24 +00:00
envoy . SendDeltaReqACK ( t , xdscommon . ListenerType , 3 )
2021-09-21 14:58:56 +00:00
} )
// Disable hack. Need to wait for one more event to wake up the loop.
atomic . StoreUint32 ( & slowHackDisabled , 1 )
2022-05-10 20:25:51 +00:00
testutil . RunStep ( t , "delayed endpoint update finally comes in" , func ( t * testing . T ) {
2021-09-21 14:58:56 +00:00
// Trigger the xds.Server select{} to wake up and notice our hack is disabled.
// The actual contents of this change are irrelevant.
2023-05-23 11:55:06 +00:00
snap = newTestSnapshot ( t , snap , "" , nil )
2021-09-21 14:58:56 +00:00
mgr . DeliverConfig ( t , sid , snap )
assertDeltaResponseSent ( t , envoy . deltaStream . sendCh , & envoy_discovery_v3 . DeltaDiscoveryResponse {
2022-03-08 19:37:24 +00:00
TypeUrl : xdscommon . EndpointType ,
2021-09-21 14:58:56 +00:00
Nonce : hexString ( 4 ) ,
Resources : makeTestResources ( t ,
makeTestEndpoints ( t , snap , "tcp:geo-cache" ) ,
) ,
} )
2022-11-08 01:10:42 +00:00
// We are caught up, so there should be nothing queued to send.
2021-09-21 14:58:56 +00:00
assertDeltaChanBlocked ( t , envoy . deltaStream . sendCh )
// It also (in parallel) issues the endpoint ACK
2022-03-08 19:37:24 +00:00
envoy . SendDeltaReqACK ( t , xdscommon . EndpointType , 4 )
2021-09-21 14:58:56 +00:00
} )
envoy . Close ( )
select {
case err := <- errCh :
require . NoError ( t , err )
case <- time . After ( 50 * time . Millisecond ) :
t . Fatalf ( "timed out waiting for handler to finish" )
}
}
2021-06-14 22:20:27 +00:00
func TestServer_DeltaAggregatedResources_v3_BasicProtocol_TCP_clusterChangesImpactEndpoints ( t * testing . T ) {
aclResolve := func ( id string ) ( acl . Authorizer , error ) {
// Allow all
return acl . RootAuthorizer ( "manage" ) , nil
}
2023-01-18 18:33:21 +00:00
scenario := newTestServerDeltaScenario ( t , aclResolve , "web-sidecar-proxy" , "" , 0 )
2021-06-14 22:20:27 +00:00
mgr , errCh , envoy := scenario . mgr , scenario . errCh , scenario . envoy
sid := structs . NewServiceID ( "web-sidecar-proxy" , nil )
// Register the proxy to create state needed to Watch() on
mgr . RegisterProxy ( t , sid )
var snap * proxycfg . ConfigSnapshot
2022-05-10 20:25:51 +00:00
testutil . RunStep ( t , "get into initial state" , func ( t * testing . T ) {
2023-05-23 11:55:06 +00:00
snap = newTestSnapshot ( t , nil , "" , nil )
2021-06-14 22:20:27 +00:00
// Send initial cluster discover.
2022-03-08 19:37:24 +00:00
envoy . SendDeltaReq ( t , xdscommon . ClusterType , & envoy_discovery_v3 . DeltaDiscoveryRequest { } )
2021-06-14 22:20:27 +00:00
// Check no response sent yet
assertDeltaChanBlocked ( t , envoy . deltaStream . sendCh )
requireProtocolVersionGauge ( t , scenario , "v3" , 1 )
// Deliver a new snapshot (tcp with one tcp upstream)
mgr . DeliverConfig ( t , sid , snap )
assertDeltaResponseSent ( t , envoy . deltaStream . sendCh , & envoy_discovery_v3 . DeltaDiscoveryResponse {
2022-03-08 19:37:24 +00:00
TypeUrl : xdscommon . ClusterType ,
2021-06-14 22:20:27 +00:00
Nonce : hexString ( 1 ) ,
Resources : makeTestResources ( t ,
makeTestCluster ( t , snap , "tcp:local_app" ) ,
makeTestCluster ( t , snap , "tcp:db" ) ,
makeTestCluster ( t , snap , "tcp:geo-cache" ) ,
) ,
} )
// Envoy then tries to discover endpoints for those clusters.
2022-03-08 19:37:24 +00:00
envoy . SendDeltaReq ( t , xdscommon . EndpointType , & envoy_discovery_v3 . DeltaDiscoveryRequest {
2021-06-14 22:20:27 +00:00
ResourceNamesSubscribe : [ ] string {
"db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" ,
"geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul" ,
} ,
} )
// We should get a response immediately since the config is already present in
// the server for endpoints. Note that this should not be racy if the server
// is behaving well since the Cluster send above should be blocked until we
// deliver a new config version.
assertDeltaResponseSent ( t , envoy . deltaStream . sendCh , & envoy_discovery_v3 . DeltaDiscoveryResponse {
2022-03-08 19:37:24 +00:00
TypeUrl : xdscommon . EndpointType ,
2021-06-14 22:20:27 +00:00
Nonce : hexString ( 2 ) ,
Resources : makeTestResources ( t ,
makeTestEndpoints ( t , snap , "tcp:db" ) ,
makeTestEndpoints ( t , snap , "tcp:geo-cache" ) ,
) ,
} )
2022-11-08 01:10:42 +00:00
// After receiving the endpoints Envoy sends an ACK for the clusters
envoy . SendDeltaReqACK ( t , xdscommon . ClusterType , 1 )
// We are caught up, so there should be nothing queued to send.
2021-06-14 22:20:27 +00:00
assertDeltaChanBlocked ( t , envoy . deltaStream . sendCh )
// Envoy now sends listener request
2022-03-08 19:37:24 +00:00
envoy . SendDeltaReq ( t , xdscommon . ListenerType , nil )
2021-06-14 22:20:27 +00:00
// It also (in parallel) issues the endpoint ACK
2022-03-08 19:37:24 +00:00
envoy . SendDeltaReqACK ( t , xdscommon . EndpointType , 2 )
2021-06-14 22:20:27 +00:00
// And should get a response immediately.
assertDeltaResponseSent ( t , envoy . deltaStream . sendCh , & envoy_discovery_v3 . DeltaDiscoveryResponse {
2022-03-08 19:37:24 +00:00
TypeUrl : xdscommon . ListenerType ,
2021-06-14 22:20:27 +00:00
Nonce : hexString ( 3 ) ,
Resources : makeTestResources ( t ,
makeTestListener ( t , snap , "tcp:public_listener" ) ,
makeTestListener ( t , snap , "tcp:db" ) ,
makeTestListener ( t , snap , "tcp:geo-cache" ) ,
) ,
} )
2022-11-08 01:10:42 +00:00
// We are caught up, so there should be nothing queued to send.
2021-06-14 22:20:27 +00:00
assertDeltaChanBlocked ( t , envoy . deltaStream . sendCh )
// ACKs the listener
2022-03-08 19:37:24 +00:00
envoy . SendDeltaReqACK ( t , xdscommon . ListenerType , 3 )
2021-06-14 22:20:27 +00:00
} )
2022-05-10 20:25:51 +00:00
testutil . RunStep ( t , "trigger cluster update needing implicit endpoint replacements" , func ( t * testing . T ) {
2021-06-14 22:20:27 +00:00
// Update the snapshot in a way that causes a single cluster update.
2023-05-23 11:55:06 +00:00
snap = newTestSnapshot ( t , snap , "" , nil , & structs . ServiceResolverConfigEntry {
2021-06-14 22:20:27 +00:00
Kind : structs . ServiceResolver ,
Name : "db" ,
ConnectTimeout : 1337 * time . Second ,
} )
mgr . DeliverConfig ( t , sid , snap )
// The cluster is updated
assertDeltaResponseSent ( t , envoy . deltaStream . sendCh , & envoy_discovery_v3 . DeltaDiscoveryResponse {
2022-03-08 19:37:24 +00:00
TypeUrl : xdscommon . ClusterType ,
2021-06-14 22:20:27 +00:00
Nonce : hexString ( 4 ) ,
Resources : makeTestResources ( t ,
// SAME makeTestCluster(t, snap, "tcp:local_app"),
makeTestCluster ( t , snap , "tcp:db:timeout" ) ,
// SAME makeTestCluster(t, snap, "tcp:geo-cache"),
) ,
} )
// And we re-send the endpoints for the updated cluster after getting the
// ACK for the cluster.
assertDeltaResponseSent ( t , envoy . deltaStream . sendCh , & envoy_discovery_v3 . DeltaDiscoveryResponse {
2022-03-08 19:37:24 +00:00
TypeUrl : xdscommon . EndpointType ,
2021-06-14 22:20:27 +00:00
Nonce : hexString ( 5 ) ,
Resources : makeTestResources ( t ,
makeTestEndpoints ( t , snap , "tcp:db" ) ,
// SAME makeTestEndpoints(t, snap, "tcp:geo-cache"),
) ,
} )
2022-11-08 01:10:42 +00:00
// Envoy then ACK's the clusters and the endpoints.
envoy . SendDeltaReqACK ( t , xdscommon . ClusterType , 4 )
2022-03-08 19:37:24 +00:00
envoy . SendDeltaReqACK ( t , xdscommon . EndpointType , 5 )
2021-06-14 22:20:27 +00:00
2022-11-08 01:10:42 +00:00
// We are caught up, so there should be nothing queued to send.
2021-06-14 22:20:27 +00:00
assertDeltaChanBlocked ( t , envoy . deltaStream . sendCh )
} )
envoy . Close ( )
select {
case err := <- errCh :
require . NoError ( t , err )
case <- time . After ( 50 * time . Millisecond ) :
t . Fatalf ( "timed out waiting for handler to finish" )
}
}
func TestServer_DeltaAggregatedResources_v3_BasicProtocol_HTTP2_RDS_listenerChangesImpactRoutes ( t * testing . T ) {
aclResolve := func ( id string ) ( acl . Authorizer , error ) {
// Allow all
return acl . RootAuthorizer ( "manage" ) , nil
}
2023-01-18 18:33:21 +00:00
scenario := newTestServerDeltaScenario ( t , aclResolve , "web-sidecar-proxy" , "" , 0 )
2021-06-14 22:20:27 +00:00
mgr , errCh , envoy := scenario . mgr , scenario . errCh , scenario . envoy
sid := structs . NewServiceID ( "web-sidecar-proxy" , nil )
// Register the proxy to create state needed to Watch() on
mgr . RegisterProxy ( t , sid )
var snap * proxycfg . ConfigSnapshot
2022-05-10 20:25:51 +00:00
testutil . RunStep ( t , "get into initial state" , func ( t * testing . T ) {
2021-06-14 22:20:27 +00:00
// Send initial cluster discover (empty payload)
2022-03-08 19:37:24 +00:00
envoy . SendDeltaReq ( t , xdscommon . ClusterType , nil )
2021-06-14 22:20:27 +00:00
// Check no response sent yet
assertDeltaChanBlocked ( t , envoy . deltaStream . sendCh )
// Deliver a new snapshot (tcp with one http upstream with no-op disco chain)
2023-05-23 11:55:06 +00:00
snap = newTestSnapshot ( t , nil , "http2" , nil , & structs . ServiceConfigEntry {
2021-06-14 22:20:27 +00:00
Kind : structs . ServiceDefaults ,
Name : "db" ,
Protocol : "http2" ,
} , & structs . ServiceRouterConfigEntry {
Kind : structs . ServiceRouter ,
Name : "db" ,
Routes : nil ,
} )
mgr . DeliverConfig ( t , sid , snap )
assertDeltaResponseSent ( t , envoy . deltaStream . sendCh , & envoy_discovery_v3 . DeltaDiscoveryResponse {
2022-03-08 19:37:24 +00:00
TypeUrl : xdscommon . ClusterType ,
2021-06-14 22:20:27 +00:00
Nonce : hexString ( 1 ) ,
Resources : makeTestResources ( t ,
makeTestCluster ( t , snap , "tcp:local_app" ) ,
makeTestCluster ( t , snap , "http2:db" ) ,
makeTestCluster ( t , snap , "tcp:geo-cache" ) ,
) ,
} )
// Envoy then tries to discover endpoints for those clusters.
2022-03-08 19:37:24 +00:00
envoy . SendDeltaReq ( t , xdscommon . EndpointType , & envoy_discovery_v3 . DeltaDiscoveryRequest {
2021-06-14 22:20:27 +00:00
ResourceNamesSubscribe : [ ] string {
"db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" ,
"geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul" ,
} ,
} )
// We should get a response immediately since the config is already present in
// the server for endpoints. Note that this should not be racy if the server
// is behaving well since the Cluster send above should be blocked until we
// deliver a new config version.
assertDeltaResponseSent ( t , envoy . deltaStream . sendCh , & envoy_discovery_v3 . DeltaDiscoveryResponse {
2022-03-08 19:37:24 +00:00
TypeUrl : xdscommon . EndpointType ,
2021-06-14 22:20:27 +00:00
Nonce : hexString ( 2 ) ,
Resources : makeTestResources ( t ,
makeTestEndpoints ( t , snap , "http2:db" ) ,
makeTestEndpoints ( t , snap , "tcp:geo-cache" ) ,
) ,
} )
2022-11-08 01:10:42 +00:00
// After receiving the endpoints Envoy sends an ACK for the clusters
envoy . SendDeltaReqACK ( t , xdscommon . ClusterType , 1 )
// We are caught up, so there should be nothing queued to send.
2021-06-14 22:20:27 +00:00
assertDeltaChanBlocked ( t , envoy . deltaStream . sendCh )
// Envoy now sends listener request
2022-03-08 19:37:24 +00:00
envoy . SendDeltaReq ( t , xdscommon . ListenerType , nil )
2021-06-14 22:20:27 +00:00
// It also (in parallel) issues the endpoint ACK
2022-03-08 19:37:24 +00:00
envoy . SendDeltaReqACK ( t , xdscommon . EndpointType , 2 )
2021-06-14 22:20:27 +00:00
// And should get a response immediately.
assertDeltaResponseSent ( t , envoy . deltaStream . sendCh , & envoy_discovery_v3 . DeltaDiscoveryResponse {
2022-03-08 19:37:24 +00:00
TypeUrl : xdscommon . ListenerType ,
2021-06-14 22:20:27 +00:00
Nonce : hexString ( 3 ) ,
Resources : makeTestResources ( t ,
makeTestListener ( t , snap , "tcp:public_listener" ) ,
makeTestListener ( t , snap , "http2:db:rds" ) ,
makeTestListener ( t , snap , "tcp:geo-cache" ) ,
) ,
} )
2022-11-08 01:10:42 +00:00
// We are caught up, so there should be nothing queued to send.
2021-06-14 22:20:27 +00:00
assertDeltaChanBlocked ( t , envoy . deltaStream . sendCh )
// Envoy now sends routes request
2022-03-08 19:37:24 +00:00
envoy . SendDeltaReq ( t , xdscommon . RouteType , & envoy_discovery_v3 . DeltaDiscoveryRequest {
2021-06-14 22:20:27 +00:00
ResourceNamesSubscribe : [ ] string {
"db" ,
} ,
} )
// And should get a response immediately.
assertDeltaResponseSent ( t , envoy . deltaStream . sendCh , & envoy_discovery_v3 . DeltaDiscoveryResponse {
2022-03-08 19:37:24 +00:00
TypeUrl : xdscommon . RouteType ,
2021-06-14 22:20:27 +00:00
Nonce : hexString ( 4 ) ,
Resources : makeTestResources ( t ,
makeTestRoute ( t , "http2:db" ) ,
) ,
} )
2022-11-08 01:10:42 +00:00
// After receiving the routes, Envoy sends acks back for the listener and routes.
envoy . SendDeltaReqACK ( t , xdscommon . ListenerType , 3 )
2022-03-08 19:37:24 +00:00
envoy . SendDeltaReqACK ( t , xdscommon . RouteType , 4 )
2021-06-14 22:20:27 +00:00
assertDeltaChanBlocked ( t , envoy . deltaStream . sendCh )
} )
2022-05-10 20:25:51 +00:00
testutil . RunStep ( t , "trigger listener update needing implicit route replacements" , func ( t * testing . T ) {
2021-06-14 22:20:27 +00:00
// Update the snapshot in a way that causes a single listener update.
//
// Downgrade from http2 to http
2023-05-23 11:55:06 +00:00
snap = newTestSnapshot ( t , snap , "http" , nil , & structs . ServiceConfigEntry {
2021-06-14 22:20:27 +00:00
Kind : structs . ServiceDefaults ,
Name : "db" ,
Protocol : "http" ,
} , & structs . ServiceRouterConfigEntry {
Kind : structs . ServiceRouter ,
Name : "db" ,
Routes : nil ,
} )
mgr . DeliverConfig ( t , sid , snap )
// db cluster is refreshed (unrelated to the test scenario other than it's required)
assertDeltaResponseSent ( t , envoy . deltaStream . sendCh , & envoy_discovery_v3 . DeltaDiscoveryResponse {
2022-03-08 19:37:24 +00:00
TypeUrl : xdscommon . ClusterType ,
2021-06-14 22:20:27 +00:00
Nonce : hexString ( 5 ) ,
Resources : makeTestResources ( t ,
makeTestCluster ( t , snap , "http:db" ) ,
) ,
} )
2022-03-08 19:37:24 +00:00
envoy . SendDeltaReqACK ( t , xdscommon . ClusterType , 5 )
2021-06-14 22:20:27 +00:00
// The behaviors of Cluster updates triggering re-sends of Endpoint updates
// tested in TestServer_DeltaAggregatedResources_v3_BasicProtocol_TCP_clusterChangesImpactEndpoints
// triggers here. It is not explicitly under test, but we have to get past
// this exchange to get to the part we care about.
assertDeltaResponseSent ( t , envoy . deltaStream . sendCh , & envoy_discovery_v3 . DeltaDiscoveryResponse {
2022-03-08 19:37:24 +00:00
TypeUrl : xdscommon . EndpointType ,
2022-02-08 16:36:48 +00:00
Nonce : hexString ( 6 ) ,
2021-06-14 22:20:27 +00:00
Resources : makeTestResources ( t ,
makeTestEndpoints ( t , snap , "http:db" ) ,
) ,
} )
2022-03-08 19:37:24 +00:00
envoy . SendDeltaReqACK ( t , xdscommon . EndpointType , 6 )
2022-02-08 16:36:48 +00:00
// the listener is updated
assertDeltaResponseSent ( t , envoy . deltaStream . sendCh , & envoy_discovery_v3 . DeltaDiscoveryResponse {
2022-03-08 19:37:24 +00:00
TypeUrl : xdscommon . ListenerType ,
2022-02-08 16:36:48 +00:00
Nonce : hexString ( 7 ) ,
Resources : makeTestResources ( t ,
makeTestListener ( t , snap , "http:db:rds" ) ,
) ,
} )
2021-06-14 22:20:27 +00:00
// THE ACTUAL THING WE CARE ABOUT: replaced route config
assertDeltaResponseSent ( t , envoy . deltaStream . sendCh , & envoy_discovery_v3 . DeltaDiscoveryResponse {
2022-03-08 19:37:24 +00:00
TypeUrl : xdscommon . RouteType ,
2021-06-14 22:20:27 +00:00
Nonce : hexString ( 8 ) ,
Resources : makeTestResources ( t ,
makeTestRoute ( t , "http2:db" ) ,
) ,
} )
2022-11-08 01:10:42 +00:00
// After receiving the routes, Envoy sends acks back for the listener and routes.
envoy . SendDeltaReqACK ( t , xdscommon . ListenerType , 7 )
2022-03-08 19:37:24 +00:00
envoy . SendDeltaReqACK ( t , xdscommon . RouteType , 8 )
2021-06-14 22:20:27 +00:00
assertDeltaChanBlocked ( t , envoy . deltaStream . sendCh )
} )
envoy . Close ( )
select {
case err := <- errCh :
require . NoError ( t , err )
case <- time . After ( 50 * time . Millisecond ) :
t . Fatalf ( "timed out waiting for handler to finish" )
}
}
2021-04-29 18:54:05 +00:00
func TestServer_DeltaAggregatedResources_v3_ACLEnforcement ( t * testing . T ) {
tests := [ ] struct {
name string
defaultDeny bool
acl string
token string
wantDenied bool
cfgSnap * proxycfg . ConfigSnapshot
} {
// Note that although we've stubbed actual ACL checks in the testManager
// ConnectAuthorize mock, by asserting against specific reason strings here
// even in the happy case which can't match the default one returned by the
// mock we are implicitly validating that the implementation used the
// correct token from the context.
{
name : "no ACLs configured" ,
defaultDeny : false ,
wantDenied : false ,
} ,
{
name : "default deny, no token" ,
defaultDeny : true ,
wantDenied : true ,
} ,
{
name : "default deny, write token" ,
defaultDeny : true ,
acl : ` service "web" { policy = "write" } ` ,
token : "service-write-on-web" ,
wantDenied : false ,
} ,
{
name : "default deny, read token" ,
defaultDeny : true ,
acl : ` service "web" { policy = "read" } ` ,
token : "service-write-on-web" ,
wantDenied : true ,
} ,
{
name : "default deny, write token on different service" ,
defaultDeny : true ,
acl : ` service "not-web" { policy = "write" } ` ,
token : "service-write-on-not-web" ,
wantDenied : true ,
} ,
{
name : "ingress default deny, write token on different service" ,
defaultDeny : true ,
acl : ` service "not-ingress" { policy = "write" } ` ,
token : "service-write-on-not-ingress" ,
wantDenied : true ,
2022-03-07 17:47:14 +00:00
cfgSnap : proxycfg . TestConfigSnapshotIngressGateway ( t , true , "tcp" , "default" , nil , nil , nil ) ,
2021-04-29 18:54:05 +00:00
} ,
}
for _ , tt := range tests {
t . Run ( tt . name , func ( t * testing . T ) {
2023-02-24 22:00:31 +00:00
var stopped bool
lock := & sync . RWMutex { }
defer func ( ) {
lock . Lock ( )
stopped = true
lock . Unlock ( )
} ( )
2023-02-23 19:52:18 +00:00
// aclResolve may be called in a goroutine even after a
// testcase tt returns. Capture the variable as tc so the
// values don't swap in the next iteration.
tc := tt
2021-04-29 18:54:05 +00:00
aclResolve := func ( id string ) ( acl . Authorizer , error ) {
2023-02-23 19:52:18 +00:00
if ! tc . defaultDeny {
2021-04-29 18:54:05 +00:00
// Allow all
return acl . RootAuthorizer ( "allow" ) , nil
}
2023-02-23 19:52:18 +00:00
if tc . acl == "" {
2021-04-29 18:54:05 +00:00
// No token and defaultDeny is denied
return acl . RootAuthorizer ( "deny" ) , nil
}
2023-02-24 22:00:31 +00:00
lock . RLock ( )
defer lock . RUnlock ( )
if stopped {
return acl . DenyAll ( ) . ToAllowAuthorizer ( ) , nil
}
2021-04-29 18:54:05 +00:00
// Ensure the correct token was passed
2023-02-23 19:52:18 +00:00
require . Equal ( t , tc . token , id )
2021-04-29 18:54:05 +00:00
// Parse the ACL and enforce it
2023-02-23 19:52:18 +00:00
policy , err := acl . NewPolicyFromSource ( tc . acl , nil , nil )
2021-04-29 18:54:05 +00:00
require . NoError ( t , err )
return acl . NewPolicyAuthorizerWithDefaults ( acl . RootAuthorizer ( "deny" ) , [ ] * acl . Policy { policy } , nil )
}
2023-01-18 18:33:21 +00:00
scenario := newTestServerDeltaScenario ( t , aclResolve , "web-sidecar-proxy" , tt . token , 0 )
2021-04-29 18:54:05 +00:00
mgr , errCh , envoy := scenario . mgr , scenario . errCh , scenario . envoy
sid := structs . NewServiceID ( "web-sidecar-proxy" , nil )
// Register the proxy to create state needed to Watch() on
mgr . RegisterProxy ( t , sid )
// Deliver a new snapshot
snap := tt . cfgSnap
if snap == nil {
2023-05-23 11:55:06 +00:00
snap = newTestSnapshot ( t , nil , "" , nil )
2021-04-29 18:54:05 +00:00
}
mgr . DeliverConfig ( t , sid , snap )
// Send initial listener discover, in real life Envoy always sends cluster
// first but it doesn't really matter and listener has a response that
// includes the token in the ext rbac filter so lets us test more stuff.
2022-03-08 19:37:24 +00:00
envoy . SendDeltaReq ( t , xdscommon . ListenerType , nil )
2021-04-29 18:54:05 +00:00
2023-01-16 17:31:56 +00:00
// If there is no token, check that we increment the gauge
if tt . token == "" {
2023-02-23 19:52:18 +00:00
retry . Run ( t , func ( r * retry . R ) {
data := scenario . sink . Data ( )
require . Len ( r , data , 1 )
item := data [ 0 ]
val , ok := item . Gauges [ "consul.xds.test.xds.server.streamsUnauthenticated" ]
require . True ( r , ok )
require . Equal ( r , float32 ( 1 ) , val . Value )
} )
2023-01-16 17:31:56 +00:00
}
2021-04-29 18:54:05 +00:00
if ! tt . wantDenied {
assertDeltaResponseSent ( t , envoy . deltaStream . sendCh , & envoy_discovery_v3 . DeltaDiscoveryResponse {
2022-03-08 19:37:24 +00:00
TypeUrl : xdscommon . ListenerType ,
2021-04-29 18:54:05 +00:00
Nonce : hexString ( 1 ) ,
Resources : makeTestResources ( t ,
makeTestListener ( t , snap , "tcp:public_listener" ) ,
makeTestListener ( t , snap , "tcp:db" ) ,
makeTestListener ( t , snap , "tcp:geo-cache" ) ,
) ,
} )
// Close the client stream since all is well. We _don't_ do this in the
// expected error case because we want to verify the error closes the
// stream from server side.
envoy . Close ( )
}
select {
case err := <- errCh :
if tt . wantDenied {
require . Error ( t , err )
2022-03-11 02:48:27 +00:00
status , ok := status . FromError ( err )
require . True ( t , ok )
require . Equal ( t , codes . PermissionDenied , status . Code ( ) )
require . Contains ( t , err . Error ( ) , "Permission denied" )
2021-04-29 18:54:05 +00:00
mgr . AssertWatchCancelled ( t , sid )
} else {
require . NoError ( t , err )
}
case <- time . After ( 50 * time . Millisecond ) :
t . Fatalf ( "timed out waiting for handler to finish" )
}
2023-01-16 17:31:56 +00:00
// If there is no token, check that we decrement the gauge
if tt . token == "" {
2023-02-23 19:52:18 +00:00
retry . Run ( t , func ( r * retry . R ) {
data := scenario . sink . Data ( )
require . Len ( r , data , 1 )
item := data [ 0 ]
val , ok := item . Gauges [ "consul.xds.test.xds.server.streamsUnauthenticated" ]
require . True ( r , ok )
require . Equal ( r , float32 ( 0 ) , val . Value )
} )
2023-01-16 17:31:56 +00:00
}
2021-04-29 18:54:05 +00:00
} )
}
}
func TestServer_DeltaAggregatedResources_v3_ACLTokenDeleted_StreamTerminatedDuringDiscoveryRequest ( t * testing . T ) {
if testing . Short ( ) {
t . Skip ( "too slow for testing.Short" )
}
aclRules := ` service "web" { policy = "write" } `
token := "service-write-on-web"
2023-02-06 15:35:52 +00:00
policy , err := acl . NewPolicyFromSource ( aclRules , nil , nil )
2021-04-29 18:54:05 +00:00
require . NoError ( t , err )
var validToken atomic . Value
validToken . Store ( token )
aclResolve := func ( id string ) ( acl . Authorizer , error ) {
if token := validToken . Load ( ) ; token == nil || id != token . ( string ) {
return nil , acl . ErrNotFound
}
return acl . NewPolicyAuthorizerWithDefaults ( acl . RootAuthorizer ( "deny" ) , [ ] * acl . Policy { policy } , nil )
}
scenario := newTestServerDeltaScenario ( t , aclResolve , "web-sidecar-proxy" , token ,
100 * time . Millisecond , // Make this short.
)
mgr , errCh , envoy := scenario . mgr , scenario . errCh , scenario . envoy
getError := func ( ) ( gotErr error , ok bool ) {
select {
case err := <- errCh :
return err , true
default :
return nil , false
}
}
sid := structs . NewServiceID ( "web-sidecar-proxy" , nil )
// Register the proxy to create state needed to Watch() on
mgr . RegisterProxy ( t , sid )
// Send initial cluster discover (OK)
2022-03-08 19:37:24 +00:00
envoy . SendDeltaReq ( t , xdscommon . ClusterType , nil )
2021-04-29 18:54:05 +00:00
{
err , ok := getError ( )
require . NoError ( t , err )
require . False ( t , ok )
}
// Check no response sent yet
assertDeltaChanBlocked ( t , envoy . deltaStream . sendCh )
{
err , ok := getError ( )
require . NoError ( t , err )
require . False ( t , ok )
}
// Deliver a new snapshot
2023-05-23 11:55:06 +00:00
snap := newTestSnapshot ( t , nil , "" , nil )
2021-04-29 18:54:05 +00:00
mgr . DeliverConfig ( t , sid , snap )
assertDeltaResponseSent ( t , envoy . deltaStream . sendCh , & envoy_discovery_v3 . DeltaDiscoveryResponse {
2022-03-08 19:37:24 +00:00
TypeUrl : xdscommon . ClusterType ,
2021-04-29 18:54:05 +00:00
Nonce : hexString ( 1 ) ,
Resources : makeTestResources ( t ,
makeTestCluster ( t , snap , "tcp:local_app" ) ,
makeTestCluster ( t , snap , "tcp:db" ) ,
makeTestCluster ( t , snap , "tcp:geo-cache" ) ,
) ,
} )
// It also (in parallel) issues the next cluster request (which acts as an ACK
// of the version we sent)
2022-03-08 19:37:24 +00:00
envoy . SendDeltaReq ( t , xdscommon . ClusterType , nil )
2021-04-29 18:54:05 +00:00
// Check no response sent yet
assertDeltaChanBlocked ( t , envoy . deltaStream . sendCh )
{
err , ok := getError ( )
require . NoError ( t , err )
require . False ( t , ok )
}
// Now nuke the ACL token while there's no activity.
validToken . Store ( "" )
select {
case err := <- errCh :
require . Error ( t , err )
gerr , ok := status . FromError ( err )
require . Truef ( t , ok , "not a grpc status error: type='%T' value=%v" , err , err )
require . Equal ( t , codes . Unauthenticated , gerr . Code ( ) )
require . Equal ( t , "unauthenticated: ACL not found" , gerr . Message ( ) )
mgr . AssertWatchCancelled ( t , sid )
case <- time . After ( 200 * time . Millisecond ) :
t . Fatalf ( "timed out waiting for handler to finish" )
}
}
func TestServer_DeltaAggregatedResources_v3_ACLTokenDeleted_StreamTerminatedInBackground ( t * testing . T ) {
if testing . Short ( ) {
t . Skip ( "too slow for testing.Short" )
}
aclRules := ` service "web" { policy = "write" } `
token := "service-write-on-web"
2023-02-06 15:35:52 +00:00
policy , err := acl . NewPolicyFromSource ( aclRules , nil , nil )
2021-04-29 18:54:05 +00:00
require . NoError ( t , err )
var validToken atomic . Value
validToken . Store ( token )
aclResolve := func ( id string ) ( acl . Authorizer , error ) {
if token := validToken . Load ( ) ; token == nil || id != token . ( string ) {
return nil , acl . ErrNotFound
}
return acl . NewPolicyAuthorizerWithDefaults ( acl . RootAuthorizer ( "deny" ) , [ ] * acl . Policy { policy } , nil )
}
scenario := newTestServerDeltaScenario ( t , aclResolve , "web-sidecar-proxy" , token ,
100 * time . Millisecond , // Make this short.
)
mgr , errCh , envoy := scenario . mgr , scenario . errCh , scenario . envoy
getError := func ( ) ( gotErr error , ok bool ) {
select {
case err := <- errCh :
return err , true
default :
return nil , false
}
}
sid := structs . NewServiceID ( "web-sidecar-proxy" , nil )
// Register the proxy to create state needed to Watch() on
mgr . RegisterProxy ( t , sid )
// Send initial cluster discover (OK)
2022-03-08 19:37:24 +00:00
envoy . SendDeltaReq ( t , xdscommon . ClusterType , nil )
2021-04-29 18:54:05 +00:00
{
err , ok := getError ( )
require . NoError ( t , err )
require . False ( t , ok )
}
// Check no response sent yet
assertDeltaChanBlocked ( t , envoy . deltaStream . sendCh )
{
err , ok := getError ( )
require . NoError ( t , err )
require . False ( t , ok )
}
// Deliver a new snapshot
2023-05-23 11:55:06 +00:00
snap := newTestSnapshot ( t , nil , "" , nil )
2021-04-29 18:54:05 +00:00
mgr . DeliverConfig ( t , sid , snap )
assertDeltaResponseSent ( t , envoy . deltaStream . sendCh , & envoy_discovery_v3 . DeltaDiscoveryResponse {
2022-03-08 19:37:24 +00:00
TypeUrl : xdscommon . ClusterType ,
2021-04-29 18:54:05 +00:00
Nonce : hexString ( 1 ) ,
Resources : makeTestResources ( t ,
makeTestCluster ( t , snap , "tcp:local_app" ) ,
makeTestCluster ( t , snap , "tcp:db" ) ,
makeTestCluster ( t , snap , "tcp:geo-cache" ) ,
) ,
} )
// It also (in parallel) issues the next cluster request (which acts as an ACK
// of the version we sent)
2022-03-08 19:37:24 +00:00
envoy . SendDeltaReq ( t , xdscommon . ClusterType , nil )
2021-04-29 18:54:05 +00:00
// Check no response sent yet
assertDeltaChanBlocked ( t , envoy . deltaStream . sendCh )
{
err , ok := getError ( )
require . NoError ( t , err )
require . False ( t , ok )
}
// Now nuke the ACL token while there's no activity.
validToken . Store ( "" )
select {
case err := <- errCh :
require . Error ( t , err )
gerr , ok := status . FromError ( err )
require . Truef ( t , ok , "not a grpc status error: type='%T' value=%v" , err , err )
require . Equal ( t , codes . Unauthenticated , gerr . Code ( ) )
require . Equal ( t , "unauthenticated: ACL not found" , gerr . Message ( ) )
mgr . AssertWatchCancelled ( t , sid )
case <- time . After ( 200 * time . Millisecond ) :
t . Fatalf ( "timed out waiting for handler to finish" )
}
}
func TestServer_DeltaAggregatedResources_v3_IngressEmptyResponse ( t * testing . T ) {
aclResolve := func ( id string ) ( acl . Authorizer , error ) {
// Allow all
return acl . RootAuthorizer ( "manage" ) , nil
}
2023-01-18 18:33:21 +00:00
scenario := newTestServerDeltaScenario ( t , aclResolve , "ingress-gateway" , "" , 0 )
2021-04-29 18:54:05 +00:00
mgr , errCh , envoy := scenario . mgr , scenario . errCh , scenario . envoy
sid := structs . NewServiceID ( "ingress-gateway" , nil )
// Register the proxy to create state needed to Watch() on
mgr . RegisterProxy ( t , sid )
// Send initial cluster discover
2022-03-08 19:37:24 +00:00
envoy . SendDeltaReq ( t , xdscommon . ClusterType , nil )
2021-04-29 18:54:05 +00:00
// Check no response sent yet
assertDeltaChanBlocked ( t , envoy . deltaStream . sendCh )
// Deliver a new snapshot with no services
2022-03-07 17:47:14 +00:00
snap := proxycfg . TestConfigSnapshotIngressGateway ( t , false , "tcp" , "default" , nil , nil , nil )
2021-04-29 18:54:05 +00:00
mgr . DeliverConfig ( t , sid , snap )
// REQ: clusters
2022-03-08 19:37:24 +00:00
envoy . SendDeltaReq ( t , xdscommon . ClusterType , nil )
2021-04-29 18:54:05 +00:00
2021-09-22 18:48:50 +00:00
// RESP: cluster
2021-04-29 18:54:05 +00:00
assertDeltaResponseSent ( t , envoy . deltaStream . sendCh , & envoy_discovery_v3 . DeltaDiscoveryResponse {
2022-03-08 19:37:24 +00:00
TypeUrl : xdscommon . ClusterType ,
2021-04-29 18:54:05 +00:00
Nonce : hexString ( 1 ) ,
} )
assertDeltaChanBlocked ( t , envoy . deltaStream . sendCh )
// ACK: clusters
2022-03-08 19:37:24 +00:00
envoy . SendDeltaReqACK ( t , xdscommon . ClusterType , 1 )
2021-04-29 18:54:05 +00:00
// REQ: listeners
2022-03-08 19:37:24 +00:00
envoy . SendDeltaReq ( t , xdscommon . ListenerType , nil )
2021-04-29 18:54:05 +00:00
// RESP: listeners
assertDeltaResponseSent ( t , envoy . deltaStream . sendCh , & envoy_discovery_v3 . DeltaDiscoveryResponse {
2022-03-08 19:37:24 +00:00
TypeUrl : xdscommon . ListenerType ,
2021-04-29 18:54:05 +00:00
Nonce : hexString ( 2 ) ,
} )
assertDeltaChanBlocked ( t , envoy . deltaStream . sendCh )
envoy . Close ( )
select {
case err := <- errCh :
require . NoError ( t , err )
case <- time . After ( 50 * time . Millisecond ) :
t . Fatalf ( "timed out waiting for handler to finish" )
}
}
2022-09-09 14:02:01 +00:00
func TestServer_DeltaAggregatedResources_v3_CapacityReached ( t * testing . T ) {
aclResolve := func ( id string ) ( acl . Authorizer , error ) { return acl . ManageAll ( ) , nil }
2023-01-18 18:33:21 +00:00
scenario := newTestServerDeltaScenario ( t , aclResolve , "web-sidecar-proxy" , "" , 0 )
2022-09-09 14:02:01 +00:00
mgr , errCh , envoy := scenario . mgr , scenario . errCh , scenario . envoy
sid := structs . NewServiceID ( "web-sidecar-proxy" , nil )
mgr . RegisterProxy ( t , sid )
2023-01-18 18:33:21 +00:00
mgr . DrainStreams ( sid )
2022-09-09 14:02:01 +00:00
2023-05-23 11:55:06 +00:00
snap := newTestSnapshot ( t , nil , "" , nil )
2022-09-09 14:02:01 +00:00
envoy . SendDeltaReq ( t , xdscommon . ClusterType , & envoy_discovery_v3 . DeltaDiscoveryRequest {
InitialResourceVersions : mustMakeVersionMap ( t ,
makeTestCluster ( t , snap , "tcp:geo-cache" ) ,
) ,
} )
select {
case err := <- errCh :
require . Error ( t , err )
require . Equal ( t , codes . ResourceExhausted . String ( ) , status . Code ( err ) . String ( ) )
case <- time . After ( 50 * time . Millisecond ) :
t . Fatalf ( "timed out waiting for handler to finish" )
}
}
type capacityReachedLimiter struct { }
func ( capacityReachedLimiter ) BeginSession ( ) ( limiter . Session , error ) {
return nil , limiter . ErrCapacityReached
}
func TestServer_DeltaAggregatedResources_v3_StreamDrained ( t * testing . T ) {
aclResolve := func ( id string ) ( acl . Authorizer , error ) { return acl . ManageAll ( ) , nil }
2023-01-18 18:33:21 +00:00
scenario := newTestServerDeltaScenario ( t , aclResolve , "web-sidecar-proxy" , "" , 0 )
2022-09-09 14:02:01 +00:00
mgr , errCh , envoy := scenario . mgr , scenario . errCh , scenario . envoy
sid := structs . NewServiceID ( "web-sidecar-proxy" , nil )
mgr . RegisterProxy ( t , sid )
testutil . RunStep ( t , "successful request/response" , func ( t * testing . T ) {
2023-05-23 11:55:06 +00:00
snap := newTestSnapshot ( t , nil , "" , nil )
2022-09-09 14:02:01 +00:00
envoy . SendDeltaReq ( t , xdscommon . ClusterType , & envoy_discovery_v3 . DeltaDiscoveryRequest {
InitialResourceVersions : mustMakeVersionMap ( t ,
makeTestCluster ( t , snap , "tcp:geo-cache" ) ,
) ,
} )
mgr . DeliverConfig ( t , sid , snap )
assertDeltaResponseSent ( t , envoy . deltaStream . sendCh , & envoy_discovery_v3 . DeltaDiscoveryResponse {
TypeUrl : xdscommon . ClusterType ,
Nonce : hexString ( 1 ) ,
Resources : makeTestResources ( t ,
makeTestCluster ( t , snap , "tcp:local_app" ) ,
makeTestCluster ( t , snap , "tcp:db" ) ,
) ,
} )
} )
testutil . RunStep ( t , "terminate limiter session" , func ( t * testing . T ) {
2023-01-18 18:33:21 +00:00
mgr . DrainStreams ( sid )
2022-09-09 14:02:01 +00:00
select {
case err := <- errCh :
require . Error ( t , err )
require . Equal ( t , codes . ResourceExhausted . String ( ) , status . Code ( err ) . String ( ) )
case <- time . After ( 50 * time . Millisecond ) :
t . Fatalf ( "timed out waiting for handler to finish" )
}
} )
2022-10-12 19:17:58 +00:00
testutil . RunStep ( t , "check drain counter incremented" , func ( t * testing . T ) {
2022-09-09 14:02:01 +00:00
data := scenario . sink . Data ( )
require . Len ( t , data , 1 )
item := data [ 0 ]
require . Len ( t , item . Counters , 1 )
val , ok := item . Counters [ "consul.xds.test.xds.server.streamDrained" ]
require . True ( t , ok )
require . Equal ( t , 1 , val . Count )
} )
2022-10-12 19:17:58 +00:00
testutil . RunStep ( t , "check streamStart metric recorded" , func ( t * testing . T ) {
data := scenario . sink . Data ( )
require . Len ( t , data , 1 )
item := data [ 0 ]
2023-01-16 17:31:56 +00:00
require . Len ( t , item . Samples , 1 )
2022-10-12 19:17:58 +00:00
val , ok := item . Samples [ "consul.xds.test.xds.server.streamStart" ]
require . True ( t , ok )
require . Equal ( t , 1 , val . Count )
} )
2022-09-09 14:02:01 +00:00
}
2021-04-29 18:54:05 +00:00
func assertDeltaChanBlocked ( t * testing . T , ch chan * envoy_discovery_v3 . DeltaDiscoveryResponse ) {
t . Helper ( )
select {
case r := <- ch :
t . Fatalf ( "chan should block but received: %v" , r )
case <- time . After ( 10 * time . Millisecond ) :
return
}
}
func assertDeltaResponseSent ( t * testing . T , ch chan * envoy_discovery_v3 . DeltaDiscoveryResponse , want * envoy_discovery_v3 . DeltaDiscoveryResponse ) {
t . Helper ( )
select {
case got := <- ch :
assertDeltaResponse ( t , got , want )
case <- time . After ( 50 * time . Millisecond ) :
t . Fatalf ( "no response received after 50ms" )
}
}
// assertDeltaResponse is a helper to test a envoy.DeltaDiscoveryResponse matches the
// expected value. We use JSON during comparison here because the responses use protobuf
// Any type which includes binary protobuf encoding.
func assertDeltaResponse ( t * testing . T , got , want * envoy_discovery_v3 . DeltaDiscoveryResponse ) {
t . Helper ( )
gotJSON := protoToSortedJSON ( t , got )
wantJSON := protoToSortedJSON ( t , want )
require . JSONEqf ( t , wantJSON , gotJSON , "got:\n%s" , gotJSON )
}
func mustMakeVersionMap ( t * testing . T , resources ... proto . Message ) map [ string ] string {
m := make ( map [ string ] string )
for _ , res := range resources {
2023-02-03 06:18:10 +00:00
name := xdscommon . GetResourceName ( res )
2021-04-29 18:54:05 +00:00
m [ name ] = mustHashResource ( t , res )
}
return m
}
2023-01-31 20:50:30 +00:00
func requireExtensionMetrics (
t * testing . T ,
scenario * testServerScenario ,
extName string ,
sid structs . ServiceID ,
err error ,
) {
data := scenario . sink . Data ( )
require . Len ( t , data , 1 )
item := data [ 0 ]
expectLabels := [ ] metrics . Label {
{ Name : "extension" , Value : extName } ,
{ Name : "version" , Value : "builtin/" + version . Version } ,
{ Name : "service" , Value : sid . ID } ,
{ Name : "partition" , Value : sid . PartitionOrDefault ( ) } ,
{ Name : "namespace" , Value : sid . NamespaceOrDefault ( ) } ,
{ Name : "error" , Value : strconv . FormatBool ( err != nil ) } ,
}
for _ , s := range [ ] string {
"consul.xds.test.envoy_extension.validate_arguments;" ,
"consul.xds.test.envoy_extension.validate;" ,
"consul.xds.test.envoy_extension.extend;" ,
} {
foundLabel := false
for k , v := range item . Samples {
if strings . HasPrefix ( k , s ) {
foundLabel = true
require . ElementsMatch ( t , expectLabels , v . Labels )
}
}
require . True ( t , foundLabel )
}
}
2023-06-08 14:26:11 +00:00
2023-07-31 19:24:33 +00:00
func Test_validateAndApplyEnvoyExtension_Validations ( t * testing . T ) {
2023-06-08 14:26:11 +00:00
type testCase struct {
name string
runtimeConfig extensioncommon . RuntimeConfig
err bool
errString string
}
envoyVersion , _ := goversion . NewVersion ( "1.25.0" )
consulVersion , _ := goversion . NewVersion ( "1.16.0" )
svc := api . CompoundServiceName {
Name : "s1" ,
Partition : "ap1" ,
Namespace : "ns1" ,
}
makeRuntimeConfig := func ( required bool , consulVersion string , envoyVersion string , args map [ string ] interface { } ) extensioncommon . RuntimeConfig {
if args == nil {
args = map [ string ] interface { } {
"ARN" : "arn:aws:lambda:us-east-1:111111111111:function:lambda-1234" ,
}
}
return extensioncommon . RuntimeConfig {
EnvoyExtension : api . EnvoyExtension {
Name : api . BuiltinAWSLambdaExtension ,
Required : required ,
ConsulVersion : consulVersion ,
EnvoyVersion : envoyVersion ,
Arguments : args ,
} ,
ServiceName : svc ,
}
}
cases := [ ] testCase {
{
name : "invalid consul version constraint - required" ,
runtimeConfig : makeRuntimeConfig ( true , "bad" , ">= 1.0" , nil ) ,
err : true ,
errString : "failed to parse Consul version constraint for extension" ,
} ,
{
name : "invalid consul version constraint - not required" ,
runtimeConfig : makeRuntimeConfig ( false , "bad" , ">= 1.0" , nil ) ,
err : false ,
} ,
{
name : "invalid envoy version constraint - required" ,
runtimeConfig : makeRuntimeConfig ( true , ">= 1.0" , "bad" , nil ) ,
err : true ,
errString : "failed to parse Envoy version constraint for extension" ,
} ,
{
name : "invalid envoy version constraint - not required" ,
runtimeConfig : makeRuntimeConfig ( false , ">= 1.0" , "bad" , nil ) ,
err : false ,
} ,
{
name : "no envoy version constraint match" ,
runtimeConfig : makeRuntimeConfig ( false , "" , ">= 2.0.0" , nil ) ,
err : false ,
} ,
{
name : "no consul version constraint match" ,
runtimeConfig : makeRuntimeConfig ( false , ">= 2.0.0" , "" , nil ) ,
err : false ,
} ,
{
name : "invalid extension arguments - required" ,
runtimeConfig : makeRuntimeConfig ( true , ">= 1.15.0" , ">= 1.25.0" , map [ string ] interface { } { "bad" : "args" } ) ,
err : true ,
errString : "failed to construct extension" ,
} ,
{
name : "invalid extension arguments - not required" ,
runtimeConfig : makeRuntimeConfig ( false , ">= 1.15.0" , ">= 1.25.0" , map [ string ] interface { } { "bad" : "args" } ) ,
err : false ,
} ,
2023-07-21 21:48:25 +00:00
{
name : "valid everything - no resources and required" ,
runtimeConfig : makeRuntimeConfig ( true , ">= 1.15.0" , ">= 1.25.0" , nil ) ,
err : true ,
errString : "failed to patch xDS resources in" ,
} ,
2023-06-08 14:26:11 +00:00
{
name : "valid everything" ,
runtimeConfig : makeRuntimeConfig ( false , ">= 1.15.0" , ">= 1.25.0" , nil ) ,
err : false ,
} ,
}
for _ , tc := range cases {
t . Run ( tc . name , func ( t * testing . T ) {
snap := proxycfg . ConfigSnapshot {
ProxyID : proxycfg . ProxyID {
ServiceID : structs . NewServiceID ( "s1" , nil ) ,
} ,
}
2023-07-31 19:24:33 +00:00
resources , err := validateAndApplyEnvoyExtension ( hclog . NewNullLogger ( ) , & snap , nil , tc . runtimeConfig , envoyVersion , consulVersion )
2023-06-08 14:26:11 +00:00
if tc . err {
require . Error ( t , err )
require . Contains ( t , err . Error ( ) , tc . errString )
} else {
require . NoError ( t , err )
2023-07-31 19:24:33 +00:00
require . Nil ( t , resources )
2023-06-08 14:26:11 +00:00
}
} )
}
}
2023-07-31 19:24:33 +00:00
func Test_applyEnvoyExtension_CanApply ( t * testing . T ) {
type testCase struct {
name string
canApply bool
}
cases := [ ] testCase {
{
name : "cannot apply: is not applied" ,
canApply : false ,
} ,
{
name : "can apply: is applied" ,
canApply : true ,
} ,
}
for _ , tc := range cases {
t . Run ( tc . name , func ( t * testing . T ) {
extender := extensioncommon . BasicEnvoyExtender {
Extension : & maybeCanApplyExtension {
canApply : tc . canApply ,
} ,
}
config := & extensioncommon . RuntimeConfig {
Kind : api . ServiceKindConnectProxy ,
ServiceName : api . CompoundServiceName { Name : "api" } ,
Upstreams : map [ api . CompoundServiceName ] * extensioncommon . UpstreamData { } ,
IsSourcedFromUpstream : false ,
EnvoyExtension : api . EnvoyExtension {
Name : "maybeCanApplyExtension" ,
Required : false ,
} ,
}
listener := & envoy_listener_v3 . Listener {
Name : xdscommon . OutboundListenerName ,
IgnoreGlobalConnLimit : false ,
}
indexedResources := xdscommon . IndexResources ( testutil . Logger ( t ) , map [ string ] [ ] proto . Message {
xdscommon . ListenerType : {
listener ,
} ,
} )
result , err := applyEnvoyExtension ( & extender , indexedResources , config )
require . NoError ( t , err )
resultListener := result . Index [ xdscommon . ListenerType ] [ xdscommon . OutboundListenerName ] . ( * envoy_listener_v3 . Listener )
require . Equal ( t , tc . canApply , resultListener . IgnoreGlobalConnLimit )
} )
}
}
func Test_applyEnvoyExtension_PartialApplicationDisallowed ( t * testing . T ) {
type testCase struct {
name string
fail bool
returnOnFailure bool
}
cases := [ ] testCase {
{
name : "failure: returns nothing" ,
fail : true ,
returnOnFailure : false ,
} ,
// Not expected, but cover to be sure.
{
name : "failure: returns values" ,
fail : true ,
returnOnFailure : true ,
} ,
// Ensure that under normal circumstances, the extension would succeed in
// modifying resources.
{
name : "success: resources modified" ,
fail : false ,
} ,
}
for _ , tc := range cases {
for _ , indexType := range [ ] string {
xdscommon . ListenerType ,
xdscommon . ClusterType ,
} {
typeShortName := indexType [ strings . LastIndex ( indexType , "." ) + 1 : ]
t . Run ( fmt . Sprintf ( "%s: %s" , tc . name , typeShortName ) , func ( t * testing . T ) {
extender := extensioncommon . BasicEnvoyExtender {
Extension : & partialFailureExtension {
returnOnFailure : tc . returnOnFailure ,
// Alternate which resource fails so that we can test for
// partial modification independent of patch order.
failListener : tc . fail && indexType == xdscommon . ListenerType ,
failCluster : tc . fail && indexType == xdscommon . ClusterType ,
} ,
}
config := & extensioncommon . RuntimeConfig {
Kind : api . ServiceKindConnectProxy ,
ServiceName : api . CompoundServiceName { Name : "api" } ,
Upstreams : map [ api . CompoundServiceName ] * extensioncommon . UpstreamData { } ,
IsSourcedFromUpstream : false ,
EnvoyExtension : api . EnvoyExtension {
Name : "partialFailureExtension" ,
Required : false ,
} ,
}
cluster := & envoy_cluster_v3 . Cluster {
Name : xdscommon . LocalAppClusterName ,
RespectDnsTtl : false ,
}
listener := & envoy_listener_v3 . Listener {
Name : xdscommon . OutboundListenerName ,
IgnoreGlobalConnLimit : false ,
}
indexedResources := xdscommon . IndexResources ( testutil . Logger ( t ) , map [ string ] [ ] proto . Message {
xdscommon . ClusterType : {
cluster ,
} ,
xdscommon . ListenerType : {
listener ,
} ,
} )
result , err := applyEnvoyExtension ( & extender , indexedResources , config )
if tc . fail {
require . Error ( t , err )
} else {
require . NoError ( t , err )
}
resultListener := result . Index [ xdscommon . ListenerType ] [ xdscommon . OutboundListenerName ] . ( * envoy_listener_v3 . Listener )
resultCluster := result . Index [ xdscommon . ClusterType ] [ xdscommon . LocalAppClusterName ] . ( * envoy_cluster_v3 . Cluster )
require . Equal ( t , ! tc . fail , resultListener . IgnoreGlobalConnLimit )
require . Equal ( t , ! tc . fail , resultCluster . RespectDnsTtl )
// Regardless of success, original values should not be modified.
originalListener := indexedResources . Index [ xdscommon . ListenerType ] [ xdscommon . OutboundListenerName ] . ( * envoy_listener_v3 . Listener )
originalCluster := indexedResources . Index [ xdscommon . ClusterType ] [ xdscommon . LocalAppClusterName ] . ( * envoy_cluster_v3 . Cluster )
require . False ( t , originalListener . IgnoreGlobalConnLimit )
require . False ( t , originalCluster . RespectDnsTtl )
} )
}
}
}
func Test_applyEnvoyExtension_HandlesPanics ( t * testing . T ) {
type testCase struct {
name string
panicOnCanApply bool
panicOnPatch bool
}
cases := [ ] testCase {
{
name : "panic: CanApply" ,
panicOnCanApply : true ,
} ,
{
name : "panic: Extend" ,
panicOnPatch : true ,
} ,
}
for _ , tc := range cases {
t . Run ( tc . name , func ( t * testing . T ) {
extension := & maybePanicExtension {
panicOnCanApply : tc . panicOnCanApply ,
panicOnPatch : tc . panicOnPatch ,
}
extender := extensioncommon . BasicEnvoyExtender {
Extension : extension ,
}
config := & extensioncommon . RuntimeConfig {
Kind : api . ServiceKindConnectProxy ,
ServiceName : api . CompoundServiceName { Name : "api" } ,
Upstreams : map [ api . CompoundServiceName ] * extensioncommon . UpstreamData { } ,
IsSourcedFromUpstream : false ,
EnvoyExtension : api . EnvoyExtension {
Name : "maybePanicExtension" ,
Required : false ,
} ,
}
listener := & envoy_listener_v3 . Listener {
Name : xdscommon . OutboundListenerName ,
IgnoreGlobalConnLimit : false ,
}
indexedResources := xdscommon . IndexResources ( testutil . Logger ( t ) , map [ string ] [ ] proto . Message {
xdscommon . ListenerType : {
listener ,
} ,
} )
_ , err := applyEnvoyExtension ( & extender , indexedResources , config )
// We did not panic, good.
// First assert our test is valid by forcing a panic, then check the error message that was returned.
if tc . panicOnCanApply {
require . PanicsWithError ( t , "this is an expected failure in CanApply" , func ( ) {
extension . CanApply ( config )
} )
require . ErrorContains ( t , err , "attempt to apply Envoy extension \"maybePanicExtension\" caused an unexpected panic: this is an expected failure in CanApply" )
}
if tc . panicOnPatch {
require . PanicsWithError ( t , "this is an expected failure in PatchListener" , func ( ) {
_ , _ , _ = extension . PatchListener ( config . GetListenerPayload ( listener ) )
} )
require . ErrorContains ( t , err , "attempt to apply Envoy extension \"maybePanicExtension\" caused an unexpected panic: this is an expected failure in PatchListener" )
}
} )
}
}
type maybeCanApplyExtension struct {
extensioncommon . BasicExtensionAdapter
canApply bool
}
var _ extensioncommon . BasicExtension = ( * maybeCanApplyExtension ) ( nil )
func ( m * maybeCanApplyExtension ) CanApply ( _ * extensioncommon . RuntimeConfig ) bool {
return m . canApply
}
func ( m * maybeCanApplyExtension ) PatchListener ( payload extensioncommon . ListenerPayload ) ( * envoy_listener_v3 . Listener , bool , error ) {
payload . Message . IgnoreGlobalConnLimit = true
return payload . Message , true , nil
}
type partialFailureExtension struct {
extensioncommon . BasicExtensionAdapter
returnOnFailure bool
failCluster bool
failListener bool
}
var _ extensioncommon . BasicExtension = ( * partialFailureExtension ) ( nil )
func ( p * partialFailureExtension ) CanApply ( _ * extensioncommon . RuntimeConfig ) bool {
return true
}
func ( p * partialFailureExtension ) PatchListener ( payload extensioncommon . ListenerPayload ) ( * envoy_listener_v3 . Listener , bool , error ) {
// Modify original input message
payload . Message . IgnoreGlobalConnLimit = true
err := fmt . Errorf ( "oops - listener patch failed" )
if ! p . failListener {
err = nil
}
returnMsg := payload . Message
if err != nil && ! p . returnOnFailure {
returnMsg = nil
}
patched := err == nil || p . returnOnFailure
return returnMsg , patched , err
}
func ( p * partialFailureExtension ) PatchCluster ( payload extensioncommon . ClusterPayload ) ( * envoy_cluster_v3 . Cluster , bool , error ) {
// Modify original input message
payload . Message . RespectDnsTtl = true
err := fmt . Errorf ( "oops - cluster patch failed" )
if ! p . failCluster {
err = nil
}
returnMsg := payload . Message
if err != nil && ! p . returnOnFailure {
returnMsg = nil
}
patched := err == nil || p . returnOnFailure
return returnMsg , patched , err
}
type maybePanicExtension struct {
extensioncommon . BasicExtensionAdapter
panicOnCanApply bool
panicOnPatch bool
}
var _ extensioncommon . BasicExtension = ( * maybePanicExtension ) ( nil )
func ( m * maybePanicExtension ) CanApply ( _ * extensioncommon . RuntimeConfig ) bool {
if m . panicOnCanApply {
panic ( fmt . Errorf ( "this is an expected failure in CanApply" ) )
}
return true
}
func ( m * maybePanicExtension ) PatchListener ( payload extensioncommon . ListenerPayload ) ( * envoy_listener_v3 . Listener , bool , error ) {
if m . panicOnPatch {
panic ( fmt . Errorf ( "this is an expected failure in PatchListener" ) )
}
payload . Message . IgnoreGlobalConnLimit = true
return payload . Message , true , nil
}