From f115cdb1d5c7cc10c3bd73c7d1cb6ec32b3b7abd Mon Sep 17 00:00:00 2001 From: John Murret Date: Tue, 7 Nov 2023 08:00:08 -0700 Subject: [PATCH] NET-6385 - Static routes that are inlined in listener filters are also created as a resource. (#19459) * cover all protocols in local_app golden tests * fix xds tests * updating latest * fix broken test * add sorting of routers to TestBuildLocalApp to get rid of the flaking * cover all protocols in local_app golden tests * cover all protocols in local_app golden tests * cover all protocols in local_app golden tests * process envoy resource by walking the map. use a map rather than array for envoy resource to prevent duplication. * cleanup. doc strings. * update to latest * fix broken test * update tests after adding sorting of routers in local_app builder tests * do not make endpoints for local_app * fix catalog destinations only by creating clusters for any cluster not already created by walking the graph. * Configure TestAllResourcesFromSnapshot to run V2 tests * wip * fix processing of failover groups * add endpoints and clusters for any clusters that were not created from walking the listener -> path * fix xds v2 golden files for clusters to include failover group clusters --- agent/xds/clusters_test.go | 2 +- agent/xds/endpoints_test.go | 14 - agent/xds/proxystateconverter/clusters.go | 3 + .../proxystateconverter/failover_policy.go | 16 - agent/xds/resources_test.go | 152 +++++++-- ...xy-catalog-destinations-only.latest.golden | 104 ++++++ ...roxy-dial-instances-directly.latest.golden | 131 ++++++++ ...roxy-with-chain-and-failover.latest.golden | 115 +++++++ ...roxy-with-chain-and-failover.latest.golden | 5 + ...xy-catalog-destinations-only.latest.golden | 5 + ...roxy-dial-instances-directly.latest.golden | 5 + ...roxy-with-chain-and-failover.latest.golden | 5 + ...xy-catalog-destinations-only.latest.golden | 5 + ...roxy-dial-instances-directly.latest.golden | 5 + agent/xdsv2/cluster_resources.go | 65 ++-- agent/xdsv2/endpoint_resources.go | 15 - agent/xdsv2/listener_resources.go | 16 +- agent/xdsv2/resources.go | 78 +++-- agent/xdsv2/route_resources.go | 48 +-- .../mixed-multi-destination.golden | 303 ++++++++++++------ .../routes/source/l7-expose-paths.golden | 48 --- .../local-and-inbound-connections.golden | 72 ----- ...kload-addresses-with-specific-ports.golden | 71 ---- ...le-workload-addresses-without-ports.golden | 71 ---- ...kload-addresses-with-specific-ports.golden | 48 --- ...le-workload-addresses-without-ports.golden | 71 ---- ...ngle-workload-address-without-ports.golden | 71 ---- 27 files changed, 821 insertions(+), 723 deletions(-) create mode 100644 agent/xds/testdata/endpoints/transparent-proxy-catalog-destinations-only.latest.golden create mode 100644 agent/xds/testdata/endpoints/transparent-proxy-dial-instances-directly.latest.golden create mode 100644 agent/xds/testdata/listeners/connect-proxy-with-chain-and-failover.latest.golden create mode 100644 agent/xds/testdata/routes/connect-proxy-with-chain-and-failover.latest.golden create mode 100644 agent/xds/testdata/routes/transparent-proxy-catalog-destinations-only.latest.golden create mode 100644 agent/xds/testdata/routes/transparent-proxy-dial-instances-directly.latest.golden create mode 100644 agent/xds/testdata/secrets/connect-proxy-with-chain-and-failover.latest.golden create mode 100644 agent/xds/testdata/secrets/transparent-proxy-catalog-destinations-only.latest.golden create mode 100644 agent/xds/testdata/secrets/transparent-proxy-dial-instances-directly.latest.golden diff --git a/agent/xds/clusters_test.go b/agent/xds/clusters_test.go index 1e03353379..ff6b0f70fc 100644 --- a/agent/xds/clusters_test.go +++ b/agent/xds/clusters_test.go @@ -90,7 +90,7 @@ func makeClusterDiscoChainTests(enterprise bool) []clusterTestCase { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover", enterprise, nil, nil) }, // TODO(proxystate): requires routes work - alsoRunTestForV2: false, + alsoRunTestForV2: true, }, { name: "connect-proxy-with-tcp-chain-failover-through-remote-gateway", diff --git a/agent/xds/endpoints_test.go b/agent/xds/endpoints_test.go index c5293d29b8..f16327f88b 100644 --- a/agent/xds/endpoints_test.go +++ b/agent/xds/endpoints_test.go @@ -247,20 +247,6 @@ type endpointTestCase struct { func makeEndpointDiscoChainTests(enterprise bool) []endpointTestCase { return []endpointTestCase{ - { - name: "connect-proxy-with-chain-and-overrides", - create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "simple-with-overrides", enterprise, nil, nil) - }, - alsoRunTestForV2: true, - }, - { - name: "connect-proxy-with-chain-and-failover", - create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover", enterprise, nil, nil) - }, - alsoRunTestForV2: true, - }, { name: "connect-proxy-with-tcp-chain-failover-through-remote-gateway", create: func(t testinf.T) *proxycfg.ConfigSnapshot { diff --git a/agent/xds/proxystateconverter/clusters.go b/agent/xds/proxystateconverter/clusters.go index b8857bee29..fa3a5eadd8 100644 --- a/agent/xds/proxystateconverter/clusters.go +++ b/agent/xds/proxystateconverter/clusters.go @@ -864,6 +864,7 @@ func (s *Converter) makeUpstreamClustersForDiscoveryChain( failoverGroup = &pbproxystate.FailoverGroup{ Config: &pbproxystate.FailoverGroupConfig{ ConnectTimeout: durationpb.New(node.Resolver.ConnectTimeout), + UseAltStatName: true, }, } } @@ -927,6 +928,7 @@ func (s *Converter) makeUpstreamClustersForDiscoveryChain( Group: &pbproxystate.EndpointGroup_Dynamic{ Dynamic: dynamic, }, + Name: groupedTarget.ClusterName, } endpointGroups = append(endpointGroups, eg) } else { @@ -940,6 +942,7 @@ func (s *Converter) makeUpstreamClustersForDiscoveryChain( }, }, }, + Name: mappedTargets.baseClusterName, } out[mappedTargets.baseClusterName] = cluster diff --git a/agent/xds/proxystateconverter/failover_policy.go b/agent/xds/proxystateconverter/failover_policy.go index ae1565458f..2b0c0617c6 100644 --- a/agent/xds/proxystateconverter/failover_policy.go +++ b/agent/xds/proxystateconverter/failover_policy.go @@ -124,22 +124,6 @@ func (s *Converter) mapDiscoChainTargets(cfgSnap *proxycfg.ConfigSnapshot, chain Service: target.Service, }.URI().String()} } - //commonTLSContext := makeCommonTLSContext( - // cfgSnap.Leaf(), - // rootPEMs, - // makeTLSParametersFromProxyTLSConfig(cfgSnap.MeshConfigTLSOutgoing()), - //) - // - //err := injectSANMatcher(commonTLSContext, spiffeIDs...) - //if err != nil { - // return failoverTargets, fmt.Errorf("failed to inject SAN matcher rules for cluster %q: %v", sni, err) - //} - - //tlsContext := &envoy_tls_v3.UpstreamTlsContext{ - // CommonTlsContext: commonTLSContext, - // Sni: sni, - //} - //ti.TLSContext = tlsContext failoverTargets.targets = append(failoverTargets.targets, ti) } diff --git a/agent/xds/resources_test.go b/agent/xds/resources_test.go index 4ab495ea65..f5a4acca38 100644 --- a/agent/xds/resources_test.go +++ b/agent/xds/resources_test.go @@ -4,6 +4,10 @@ package xds import ( + "fmt" + "github.com/hashicorp/consul/agent/xds/proxystateconverter" + "github.com/hashicorp/consul/agent/xdsv2" + "google.golang.org/protobuf/proto" "path/filepath" "sort" "testing" @@ -46,6 +50,7 @@ type goldenTestCase struct { setup func(snap *proxycfg.ConfigSnapshot) overrideGoldenName string generatorSetup func(*ResourceGenerator) + alsoRunTestForV2 bool } func TestAllResourcesFromSnapshot(t *testing.T) { @@ -77,6 +82,33 @@ func TestAllResourcesFromSnapshot(t *testing.T) { tt.setup(snap) } + typeUrls := []string{ + xdscommon.ListenerType, + xdscommon.RouteType, + xdscommon.ClusterType, + xdscommon.EndpointType, + xdscommon.SecretType, + } + + resourceSortingFunc := func(items []proto.Message, typeURL string) func(i, j int) bool { + return func(i, j int) bool { + switch typeURL { + case xdscommon.ListenerType: + return items[i].(*envoy_listener_v3.Listener).Name < items[j].(*envoy_listener_v3.Listener).Name + case xdscommon.RouteType: + return items[i].(*envoy_route_v3.RouteConfiguration).Name < items[j].(*envoy_route_v3.RouteConfiguration).Name + case xdscommon.ClusterType: + return items[i].(*envoy_cluster_v3.Cluster).Name < items[j].(*envoy_cluster_v3.Cluster).Name + case xdscommon.EndpointType: + return items[i].(*envoy_endpoint_v3.ClusterLoadAssignment).ClusterName < items[j].(*envoy_endpoint_v3.ClusterLoadAssignment).ClusterName + case xdscommon.SecretType: + return items[i].(*envoy_tls_v3.Secret).Name < items[j].(*envoy_tls_v3.Secret).Name + default: + panic("not possible") + } + } + } + // Need server just for logger dependency g := NewResourceGenerator(testutil.Logger(t), nil, false) g.ProxyFeatures = sf @@ -87,37 +119,15 @@ func TestAllResourcesFromSnapshot(t *testing.T) { resources, err := g.AllResourcesFromSnapshot(snap) require.NoError(t, err) - typeUrls := []string{ - xdscommon.ListenerType, - xdscommon.RouteType, - xdscommon.ClusterType, - xdscommon.EndpointType, - xdscommon.SecretType, - } require.Len(t, resources, len(typeUrls)) for _, typeUrl := range typeUrls { prettyName := testTypeUrlToPrettyName[typeUrl] - t.Run(prettyName, func(t *testing.T) { + t.Run(fmt.Sprintf("xdsv1-%s", prettyName), func(t *testing.T) { items, ok := resources[typeUrl] require.True(t, ok) - sort.Slice(items, func(i, j int) bool { - switch typeUrl { - case xdscommon.ListenerType: - return items[i].(*envoy_listener_v3.Listener).Name < items[j].(*envoy_listener_v3.Listener).Name - case xdscommon.RouteType: - return items[i].(*envoy_route_v3.RouteConfiguration).Name < items[j].(*envoy_route_v3.RouteConfiguration).Name - case xdscommon.ClusterType: - return items[i].(*envoy_cluster_v3.Cluster).Name < items[j].(*envoy_cluster_v3.Cluster).Name - case xdscommon.EndpointType: - return items[i].(*envoy_endpoint_v3.ClusterLoadAssignment).ClusterName < items[j].(*envoy_endpoint_v3.ClusterLoadAssignment).ClusterName - case xdscommon.SecretType: - return items[i].(*envoy_tls_v3.Secret).Name < items[j].(*envoy_tls_v3.Secret).Name - default: - panic("not possible") - } - }) + sort.Slice(items, resourceSortingFunc(items, typeUrl)) r, err := response.CreateResponse(typeUrl, "00000001", "00000001", items) require.NoError(t, err) @@ -133,6 +143,43 @@ func TestAllResourcesFromSnapshot(t *testing.T) { require.JSONEq(t, expectedJSON, gotJSON) }) } + + if tt.alsoRunTestForV2 { + generator := xdsv2.NewResourceGenerator(testutil.Logger(t)) + + converter := proxystateconverter.NewConverter(testutil.Logger(t), &mockCfgFetcher{addressLan: "10.10.10.10"}) + proxyState, err := converter.ProxyStateFromSnapshot(snap) + require.NoError(t, err) + + v2Resources, err := generator.AllResourcesFromIR(proxyState) + require.NoError(t, err) + require.Len(t, v2Resources, len(typeUrls)-1) // secrets are not currently implemented in V2. + for _, typeUrl := range typeUrls { + prettyName := testTypeUrlToPrettyName[typeUrl] + t.Run(fmt.Sprintf("xdsv2-%s", prettyName), func(t *testing.T) { + if typeUrl == xdscommon.SecretType { + t.Skip("skipping. secrets are not yet implemented in xdsv2") + } + items, ok := v2Resources[typeUrl] + require.True(t, ok) + + sort.Slice(items, resourceSortingFunc(items, typeUrl)) + + r, err := response.CreateResponse(typeUrl, "00000001", "00000001", items) + require.NoError(t, err) + + gotJSON := protoToJSON(t, r) + + gName := tt.name + if tt.overrideGoldenName != "" { + gName = tt.overrideGoldenName + } + + expectedJSON := goldenEnvoy(t, filepath.Join(prettyName, gName), envoyVersion, latestEnvoyVersion, gotJSON) + require.JSONEq(t, expectedJSON, gotJSON) + }) + } + } } tests := []testcase{ @@ -141,18 +188,28 @@ func TestAllResourcesFromSnapshot(t *testing.T) { create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshot(t, nil, nil) }, + alsoRunTestForV2: true, }, { name: "connect-proxy-with-chain", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "simple", false, nil, nil) }, + alsoRunTestForV2: true, }, { name: "connect-proxy-with-chain-external-sni", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "external-sni", false, nil, nil) }, + alsoRunTestForV2: true, + }, + { + name: "connect-proxy-with-chain-and-failover", + create: func(t testinf.T) *proxycfg.ConfigSnapshot { + return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover", false, nil, nil) + }, + alsoRunTestForV2: true, }, { name: "connect-proxy-exported-to-peers", @@ -168,36 +225,49 @@ func TestAllResourcesFromSnapshot(t *testing.T) { }, }) }, + alsoRunTestForV2: true, }, { name: "transparent-proxy", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotTransparentProxy(t) }, + alsoRunTestForV2: true, }, { name: "connect-proxy-with-peered-upstreams", create: proxycfg.TestConfigSnapshotPeering, + // TODO(proxystate): peering will come at a later date. + alsoRunTestForV2: false, }, { name: "connect-proxy-with-peered-upstreams-escape-overrides", create: proxycfg.TestConfigSnapshotPeeringWithEscapeOverrides, + // TODO(proxystate): peering will come at a later date. + alsoRunTestForV2: false, }, { name: "connect-proxy-with-peered-upstreams-http2", create: proxycfg.TestConfigSnapshotPeeringWithHTTP2, + // TODO(proxystate): peering will come at a later date. + alsoRunTestForV2: false, }, { name: "transparent-proxy-with-peered-upstreams", create: proxycfg.TestConfigSnapshotPeeringTProxy, + // TODO(proxystate): peering will come at a later date. + alsoRunTestForV2: false, }, { name: "local-mesh-gateway-with-peered-upstreams", create: proxycfg.TestConfigSnapshotPeeringLocalMeshGateway, + // TODO(proxystate): mesh gateways and peering will come at a later date. + alsoRunTestForV2: false, }, { - name: "telemetry-collector", - create: proxycfg.TestConfigSnapshotTelemetryCollector, + name: "telemetry-collector", + create: proxycfg.TestConfigSnapshotTelemetryCollector, + alsoRunTestForV2: false, }, } tests = append(tests, getConnectProxyTransparentProxyGoldenTestCases()...) @@ -225,18 +295,24 @@ func getConnectProxyTransparentProxyGoldenTestCases() []goldenTestCase { { name: "transparent-proxy-destination", create: proxycfg.TestConfigSnapshotTransparentProxyDestination, + // TODO(proxystate): currently failing. should work. possible issue in converter. + alsoRunTestForV2: false, }, { name: "transparent-proxy-destination-http", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotTransparentProxyDestinationHTTP(t, nil) }, + // TODO(proxystate): currently failing. should work. possible issue in converter. + alsoRunTestForV2: false, }, { name: "transparent-proxy-terminating-gateway-destinations-only", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotTerminatingGatewayDestinations(t, true, nil) }, + // TODO(proxystate): terminating gateways will come at a later date. + alsoRunTestForV2: false, }, } } @@ -248,24 +324,32 @@ func getMeshGatewayPeeringGoldenTestCases() []goldenTestCase { create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotPeeredMeshGateway(t, "default-services-tcp", nil, nil) }, + // TODO(proxystate): mesh gateways will come at a later date. + alsoRunTestForV2: false, }, { name: "mesh-gateway-with-exported-peered-services-http", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotPeeredMeshGateway(t, "default-services-http", nil, nil) }, + // TODO(proxystate): mesh gateways will come at a later date. + alsoRunTestForV2: false, }, { name: "mesh-gateway-with-exported-peered-services-http-with-router", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotPeeredMeshGateway(t, "chain-and-l7-stuff", nil, nil) }, + // TODO(proxystate): mesh gateways will come at a later date. + alsoRunTestForV2: false, }, { name: "mesh-gateway-peering-control-plane", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotPeeredMeshGateway(t, "control-plane", nil, nil) }, + // TODO(proxystate): mesh gateways will come at a later date. + alsoRunTestForV2: false, }, { name: "mesh-gateway-with-imported-peered-services", @@ -276,12 +360,16 @@ func getMeshGatewayPeeringGoldenTestCases() []goldenTestCase { } }, nil) }, + // TODO(proxystate): mesh gateways will come at a later date. + alsoRunTestForV2: false, }, { name: "mesh-gateway-with-peer-through-mesh-gateway-enabled", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotPeeredMeshGateway(t, "peer-through-mesh-gateway", nil, nil) }, + // TODO(proxystate): mesh gateways will come at a later date. + alsoRunTestForV2: false, }, } } @@ -293,12 +381,16 @@ func getTrafficControlPeeringGoldenTestCases(enterprise bool) []goldenTestCase { create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-to-cluster-peer", enterprise, nil, nil) }, + // TODO(proxystate): peering will come at a later date. + alsoRunTestForV2: false, }, { name: "connect-proxy-with-chain-and-redirect-to-cluster-peer", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "redirect-to-cluster-peer", enterprise, nil, nil) }, + // TODO(proxystate): peering will come at a later date. + alsoRunTestForV2: false, }, } @@ -465,6 +557,8 @@ func getAPIGatewayGoldenTestCases(t *testing.T) []goldenTestCase { Certificate: gatewayTestCertificate, }}, nil) }, + // TODO(proxystate): api gateways will come at a later date. + alsoRunTestForV2: false, }, { name: "api-gateway-with-multiple-inline-certificates", @@ -538,6 +632,8 @@ func getAPIGatewayGoldenTestCases(t *testing.T) []goldenTestCase { }, }, nil) }, + // TODO(proxystate): api gateways will come at a later date. + alsoRunTestForV2: false, }, { name: "api-gateway-with-http-route", @@ -619,6 +715,8 @@ func getAPIGatewayGoldenTestCases(t *testing.T) []goldenTestCase { }, }}) }, + // TODO(proxystate): api gateways will come at a later date. + alsoRunTestForV2: false, }, { name: "api-gateway-with-http-route-timeoutfilter-one-set", @@ -693,6 +791,8 @@ func getAPIGatewayGoldenTestCases(t *testing.T) []goldenTestCase { }, }}) }, + // TODO(proxystate): api gateways will come at a later date. + alsoRunTestForV2: false, }, } } diff --git a/agent/xds/testdata/endpoints/transparent-proxy-catalog-destinations-only.latest.golden b/agent/xds/testdata/endpoints/transparent-proxy-catalog-destinations-only.latest.golden new file mode 100644 index 0000000000..ebf0eb91d9 --- /dev/null +++ b/agent/xds/testdata/endpoints/transparent-proxy-catalog-destinations-only.latest.golden @@ -0,0 +1,104 @@ +{ + "nonce": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", + "clusterName": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "10.10.1.1", + "portValue": 8080 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + }, + { + "endpoint": { + "address": { + "socketAddress": { + "address": "10.10.1.2", + "portValue": 8080 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + }, + { + "@type": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", + "clusterName": "geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "10.10.1.1", + "portValue": 8080 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + }, + { + "endpoint": { + "address": { + "socketAddress": { + "address": "10.20.1.2", + "portValue": 8080 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + }, + { + "@type": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", + "clusterName": "google.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "9.9.9.9", + "portValue": 9090 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + }, + { + "@type": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", + "clusterName": "no-endpoints.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + {} + ] + } + ], + "typeUrl": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", + "versionInfo": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/endpoints/transparent-proxy-dial-instances-directly.latest.golden b/agent/xds/testdata/endpoints/transparent-proxy-dial-instances-directly.latest.golden new file mode 100644 index 0000000000..202693781e --- /dev/null +++ b/agent/xds/testdata/endpoints/transparent-proxy-dial-instances-directly.latest.golden @@ -0,0 +1,131 @@ +{ + "nonce": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", + "clusterName": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "10.10.1.1", + "portValue": 8080 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + }, + { + "endpoint": { + "address": { + "socketAddress": { + "address": "10.10.1.2", + "portValue": 8080 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + }, + { + "@type": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", + "clusterName": "geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "10.10.1.1", + "portValue": 8080 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + }, + { + "endpoint": { + "address": { + "socketAddress": { + "address": "10.20.1.2", + "portValue": 8080 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + }, + { + "@type": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", + "clusterName": "kafka.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "9.9.9.9", + "portValue": 9092 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + }, + { + "@type": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", + "clusterName": "mongo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "10.10.10.10", + "portValue": 27017 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + }, + { + "endpoint": { + "address": { + "socketAddress": { + "address": "10.10.10.12", + "portValue": 27017 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + } + ], + "typeUrl": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", + "versionInfo": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/listeners/connect-proxy-with-chain-and-failover.latest.golden b/agent/xds/testdata/listeners/connect-proxy-with-chain-and-failover.latest.golden new file mode 100644 index 0000000000..0254a224d5 --- /dev/null +++ b/agent/xds/testdata/listeners/connect-proxy-with-chain-and-failover.latest.golden @@ -0,0 +1,115 @@ +{ + "nonce": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", + "address": { + "socketAddress": { + "address": "127.0.0.1", + "portValue": 9191 + } + }, + "filterChains": [ + { + "filters": [ + { + "name": "envoy.filters.network.tcp_proxy", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", + "cluster": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "statPrefix": "upstream.db.default.default.dc1" + } + } + ] + } + ], + "name": "db:127.0.0.1:9191", + "trafficDirection": "OUTBOUND" + }, + { + "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", + "address": { + "socketAddress": { + "address": "127.10.10.10", + "portValue": 8181 + } + }, + "filterChains": [ + { + "filters": [ + { + "name": "envoy.filters.network.tcp_proxy", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", + "cluster": "geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul", + "statPrefix": "upstream.prepared_query_geo-cache" + } + } + ] + } + ], + "name": "prepared_query:geo-cache:127.10.10.10:8181", + "trafficDirection": "OUTBOUND" + }, + { + "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", + "address": { + "socketAddress": { + "address": "0.0.0.0", + "portValue": 9999 + } + }, + "filterChains": [ + { + "filters": [ + { + "name": "envoy.filters.network.rbac", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.rbac.v3.RBAC", + "rules": {}, + "statPrefix": "connect_authz" + } + }, + { + "name": "envoy.filters.network.tcp_proxy", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", + "cluster": "local_app", + "statPrefix": "public_listener" + } + } + ], + "transportSocket": { + "name": "tls", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext", + "commonTlsContext": { + "tlsCertificates": [ + { + "certificateChain": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n" + }, + "privateKey": { + "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n" + } + } + ], + "tlsParams": {}, + "validationContext": { + "trustedCa": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n" + } + } + }, + "requireClientCertificate": true + } + } + } + ], + "name": "public_listener:0.0.0.0:9999", + "trafficDirection": "INBOUND" + } + ], + "typeUrl": "type.googleapis.com/envoy.config.listener.v3.Listener", + "versionInfo": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/routes/connect-proxy-with-chain-and-failover.latest.golden b/agent/xds/testdata/routes/connect-proxy-with-chain-and-failover.latest.golden new file mode 100644 index 0000000000..8b919343d2 --- /dev/null +++ b/agent/xds/testdata/routes/connect-proxy-with-chain-and-failover.latest.golden @@ -0,0 +1,5 @@ +{ + "nonce": "00000001", + "typeUrl": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + "versionInfo": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/routes/transparent-proxy-catalog-destinations-only.latest.golden b/agent/xds/testdata/routes/transparent-proxy-catalog-destinations-only.latest.golden new file mode 100644 index 0000000000..8b919343d2 --- /dev/null +++ b/agent/xds/testdata/routes/transparent-proxy-catalog-destinations-only.latest.golden @@ -0,0 +1,5 @@ +{ + "nonce": "00000001", + "typeUrl": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + "versionInfo": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/routes/transparent-proxy-dial-instances-directly.latest.golden b/agent/xds/testdata/routes/transparent-proxy-dial-instances-directly.latest.golden new file mode 100644 index 0000000000..8b919343d2 --- /dev/null +++ b/agent/xds/testdata/routes/transparent-proxy-dial-instances-directly.latest.golden @@ -0,0 +1,5 @@ +{ + "nonce": "00000001", + "typeUrl": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + "versionInfo": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/secrets/connect-proxy-with-chain-and-failover.latest.golden b/agent/xds/testdata/secrets/connect-proxy-with-chain-and-failover.latest.golden new file mode 100644 index 0000000000..82e4565065 --- /dev/null +++ b/agent/xds/testdata/secrets/connect-proxy-with-chain-and-failover.latest.golden @@ -0,0 +1,5 @@ +{ + "nonce": "00000001", + "typeUrl": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret", + "versionInfo": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/secrets/transparent-proxy-catalog-destinations-only.latest.golden b/agent/xds/testdata/secrets/transparent-proxy-catalog-destinations-only.latest.golden new file mode 100644 index 0000000000..82e4565065 --- /dev/null +++ b/agent/xds/testdata/secrets/transparent-proxy-catalog-destinations-only.latest.golden @@ -0,0 +1,5 @@ +{ + "nonce": "00000001", + "typeUrl": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret", + "versionInfo": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/secrets/transparent-proxy-dial-instances-directly.latest.golden b/agent/xds/testdata/secrets/transparent-proxy-dial-instances-directly.latest.golden new file mode 100644 index 0000000000..82e4565065 --- /dev/null +++ b/agent/xds/testdata/secrets/transparent-proxy-dial-instances-directly.latest.golden @@ -0,0 +1,5 @@ +{ + "nonce": "00000001", + "typeUrl": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret", + "versionInfo": "00000001" +} \ No newline at end of file diff --git a/agent/xdsv2/cluster_resources.go b/agent/xdsv2/cluster_resources.go index d32eeed2ec..6e33d64ecf 100644 --- a/agent/xdsv2/cluster_resources.go +++ b/agent/xdsv2/cluster_resources.go @@ -6,7 +6,6 @@ package xdsv2 import ( "errors" "fmt" - envoy_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" envoy_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" envoy_aggregate_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/clusters/aggregate/v3" @@ -19,43 +18,13 @@ import ( "github.com/hashicorp/consul/proto-public/pbmesh/v2beta1/pbproxystate" ) -func (pr *ProxyResources) doesEnvoyClusterAlreadyExist(name string) bool { - // TODO(proxystate): consider using a map instead of [] for this kind of lookup - for _, envoyCluster := range pr.envoyResources[xdscommon.ClusterType] { - if envoyCluster.(*envoy_cluster_v3.Cluster).Name == name { - return true - } - } - return false -} - -func (pr *ProxyResources) makeXDSClusters() ([]proto.Message, error) { - clusters := make([]proto.Message, 0) - - for clusterName := range pr.proxyState.Clusters { - protoCluster, err := pr.makeClusters(clusterName) - // TODO: aggregate errors for clusters and still return any properly formed clusters. - if err != nil { - return nil, err - } - clusters = append(clusters, protoCluster...) - } - - return clusters, nil -} - -func (pr *ProxyResources) makeClusters(name string) ([]proto.Message, error) { - clusters := make([]proto.Message, 0) +func (pr *ProxyResources) makeClusters(name string) (map[string]proto.Message, error) { + envoyClusters := make(map[string]proto.Message) proxyStateCluster, ok := pr.proxyState.Clusters[name] if !ok { return nil, fmt.Errorf("cluster %q not found", name) } - if pr.doesEnvoyClusterAlreadyExist(name) { - // don't error - return []proto.Message{}, nil - } - switch proxyStateCluster.Group.(type) { case *pbproxystate.Cluster_FailoverGroup: fg := proxyStateCluster.GetFailoverGroup() @@ -64,7 +33,7 @@ func (pr *ProxyResources) makeClusters(name string) ([]proto.Message, error) { return nil, err } for _, c := range clusters { - clusters = append(clusters, c) + envoyClusters[c.Name] = c } case *pbproxystate.Cluster_EndpointGroup: @@ -73,12 +42,12 @@ func (pr *ProxyResources) makeClusters(name string) ([]proto.Message, error) { if err != nil { return nil, err } - clusters = append(clusters, cluster) + envoyClusters[cluster.Name] = cluster default: return nil, errors.New("cluster group type should be Endpoint Group or Failover Group") } - return clusters, nil + return envoyClusters, nil } func (pr *ProxyResources) makeEnvoyCluster(name string, protocol pbproxystate.Protocol, eg *pbproxystate.EndpointGroup) (*envoy_cluster_v3.Cluster, error) { @@ -207,8 +176,8 @@ func (pr *ProxyResources) makeEnvoyPassthroughCluster(name string, protocol pbpr return cluster, nil } -func (pr *ProxyResources) makeEnvoyAggregateCluster(name string, protocol pbproxystate.Protocol, fg *pbproxystate.FailoverGroup) ([]*envoy_cluster_v3.Cluster, error) { - var clusters []*envoy_cluster_v3.Cluster +func (pr *ProxyResources) makeEnvoyAggregateCluster(name string, protocol pbproxystate.Protocol, fg *pbproxystate.FailoverGroup) (map[string]*envoy_cluster_v3.Cluster, error) { + clusters := make(map[string]*envoy_cluster_v3.Cluster) if fg != nil { var egNames []string for _, eg := range fg.EndpointGroups { @@ -217,7 +186,7 @@ func (pr *ProxyResources) makeEnvoyAggregateCluster(name string, protocol pbprox return nil, err } egNames = append(egNames, cluster.Name) - clusters = append(clusters, cluster) + clusters[cluster.Name] = cluster } aggregateClusterConfig, err := anypb.New(&envoy_aggregate_cluster_v3.ClusterConfig{ Clusters: egNames, @@ -245,7 +214,7 @@ func (pr *ProxyResources) makeEnvoyAggregateCluster(name string, protocol pbprox if err != nil { return nil, err } - clusters = append(clusters, c) + clusters[c.Name] = c } return clusters, nil } @@ -376,9 +345,19 @@ func addEnvoyLBToCluster(dynamicConfig *pbproxystate.DynamicEndpointGroupConfig, return nil } -// TODO(proxystate): In a future PR this will create clusters and add it to ProxyResources.proxyState -// Currently, we do not traverse the listener -> endpoint paths and instead just generate each resource by iterating -// through its top level map. In the future we want to traverse these paths to ensure each listener has a cluster, etc. func (pr *ProxyResources) makeEnvoyClusterFromL4Destination(l4 *pbproxystate.L4Destination) error { + switch l4.Destination.(type) { + case *pbproxystate.L4Destination_Cluster: + pr.addEnvoyClustersAndEndpointsToEnvoyResources(l4.GetCluster().GetName()) + + case *pbproxystate.L4Destination_WeightedClusters: + psWeightedClusters := l4.GetWeightedClusters() + for _, psCluster := range psWeightedClusters.GetClusters() { + pr.addEnvoyClustersAndEndpointsToEnvoyResources(psCluster.Name) + } + default: + return errors.New("cluster group type should be Endpoint Group or Failover Group") + } + return nil } diff --git a/agent/xdsv2/endpoint_resources.go b/agent/xdsv2/endpoint_resources.go index fa3d39d877..ce81d5cad0 100644 --- a/agent/xdsv2/endpoint_resources.go +++ b/agent/xdsv2/endpoint_resources.go @@ -7,9 +7,7 @@ import ( envoy_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" envoy_endpoint_v3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" "github.com/hashicorp/consul/agent/xds/response" - "github.com/hashicorp/consul/envoyextensions/xdscommon" "github.com/hashicorp/consul/proto-public/pbmesh/v2beta1/pbproxystate" - "google.golang.org/protobuf/proto" ) func makeEnvoyLbEndpoint(endpoint *pbproxystate.Endpoint) *envoy_endpoint_v3.LbEndpoint { @@ -46,16 +44,3 @@ func makeEnvoyClusterLoadAssignment(clusterName string, endpoints []*pbproxystat Endpoints: []*envoy_endpoint_v3.LocalityLbEndpoints{localityLbEndpoints}, } } - -func (pr *ProxyResources) makeXDSEndpoints() ([]proto.Message, error) { - endpoints := make([]proto.Message, 0) - - for clusterName, eps := range pr.proxyState.GetEndpoints() { - if clusterName != xdscommon.LocalAppClusterName { - protoEndpoint := makeEnvoyClusterLoadAssignment(clusterName, eps.Endpoints) - endpoints = append(endpoints, protoEndpoint) - } - } - - return endpoints, nil -} diff --git a/agent/xdsv2/listener_resources.go b/agent/xdsv2/listener_resources.go index 1ccfb42a6d..a7f3976ed5 100644 --- a/agent/xdsv2/listener_resources.go +++ b/agent/xdsv2/listener_resources.go @@ -42,18 +42,16 @@ const ( envoyHttpConnectionManagerFilterName = "envoy.filters.network.http_connection_manager" ) -func (pr *ProxyResources) makeXDSListeners() ([]proto.Message, error) { - listeners := make([]proto.Message, 0) - +func (pr *ProxyResources) makeEnvoyResourceGraphsStartingFromListeners() error { for _, l := range pr.proxyState.Listeners { protoListener, err := pr.makeListener(l) // TODO: aggregate errors for listeners and still return any properly formed listeners. if err != nil { - return nil, err + return err } - listeners = append(listeners, protoListener) + pr.envoyResources[xdscommon.ListenerType][protoListener.Name] = protoListener } - return listeners, nil + return nil } func (pr *ProxyResources) makeListener(listener *pbproxystate.Listener) (*envoy_listener_v3.Listener, error) { @@ -309,7 +307,7 @@ func (pr *ProxyResources) makeEnvoyResourcesForL4Destination(l4 *pbproxystate.Ro if err != nil { return nil, err } - envoyFilters, err := makeL4Filters(l4.L4) + envoyFilters, err := pr.makeL4Filters(l4.L4) return envoyFilters, err } @@ -334,7 +332,7 @@ func getAlpnProtocols(protocol pbproxystate.L7Protocol) []string { return alpnProtocols } -func makeL4Filters(l4 *pbproxystate.L4Destination) ([]*envoy_listener_v3.Filter, error) { +func (pr *ProxyResources) makeL4Filters(l4 *pbproxystate.L4Destination) ([]*envoy_listener_v3.Filter, error) { var envoyFilters []*envoy_listener_v3.Filter if l4 != nil { rbacFilters, err := MakeRBACNetworkFilters(l4.TrafficPermissions) @@ -442,7 +440,7 @@ func (pr *ProxyResources) makeL7Filters(l7 *pbproxystate.L7Destination) ([]*envo } } else { // Add Envoy route under the route resource since it's not inlined. - pr.envoyResources[xdscommon.RouteType] = append(pr.envoyResources[xdscommon.RouteType], routeConfig) + pr.envoyResources[xdscommon.RouteType][routeConfig.Name] = routeConfig httpConnMgr.RouteSpecifier = &envoy_http_v3.HttpConnectionManager_Rds{ Rds: &envoy_http_v3.Rds{ diff --git a/agent/xdsv2/resources.go b/agent/xdsv2/resources.go index fa5f7179e6..1c3251930c 100644 --- a/agent/xdsv2/resources.go +++ b/agent/xdsv2/resources.go @@ -20,6 +20,7 @@ type ResourceGenerator struct { ProxyFeatures xdscommon.SupportedProxyFeatures } +// NewResourceGenerator will create a new ResourceGenerator. func NewResourceGenerator( logger hclog.Logger, ) *ResourceGenerator { @@ -28,48 +29,63 @@ func NewResourceGenerator( } } +// ProxyResources is the main state used to convert proxyState resources to Envoy resources. type ProxyResources struct { - proxyState *proxytracker.ProxyState - envoyResources map[string][]proto.Message + // proxyState is the final proxyState computed by Consul controllers. + proxyState *proxytracker.ProxyState + // envoyResources is a map of each resource type (listener, endpoint, route, cluster, etc.) + // with a corresponding map of k/v pairs of resource name to envoy proto message. + // map[string]map[string]proto.Message is used over map[string][]proto.Message because + // AllResourcesFromIR() will create envoy resource by walking the object graph from listener + // to endpoint. In the process, the same resource might be referenced more than once, + // so the map is used to prevent duplicate resources being created and also will use + // an O(1) lookup to see if it exists (it actually will set the map key rather than + // checks everywhere) where as each lookup would be O(n) with a []proto structure. + envoyResources map[string]map[string]proto.Message } func (g *ResourceGenerator) AllResourcesFromIR(proxyState *proxytracker.ProxyState) (map[string][]proto.Message, error) { pr := &ProxyResources{ proxyState: proxyState, - envoyResources: make(map[string][]proto.Message), + envoyResources: make(map[string]map[string]proto.Message), } - err := pr.generateXDSResources() + pr.envoyResources[xdscommon.ListenerType] = make(map[string]proto.Message) + pr.envoyResources[xdscommon.RouteType] = make(map[string]proto.Message) + pr.envoyResources[xdscommon.ClusterType] = make(map[string]proto.Message) + pr.envoyResources[xdscommon.EndpointType] = make(map[string]proto.Message) + + err := pr.makeEnvoyResourceGraphsStartingFromListeners() if err != nil { return nil, fmt.Errorf("failed to generate xDS resources for ProxyState: %v", err) } - return pr.envoyResources, nil + + // Now account for Clusters that did not have a destination. + for name := range proxyState.Clusters { + if _, ok := pr.envoyResources[xdscommon.ClusterType][name]; !ok { + pr.addEnvoyClustersAndEndpointsToEnvoyResources(name) + } + } + + envoyResources := convertResourceMapsToResourceArrays(pr.envoyResources) + return envoyResources, nil } -func (pr *ProxyResources) generateXDSResources() error { - listeners, err := pr.makeXDSListeners() - if err != nil { - return err +// convertResourceMapsToResourceArrays will convert map[string]map[string]proto.Message, which is used to +// prevent duplicate resource being created, to map[string][]proto.Message which is used by Delta server. +func convertResourceMapsToResourceArrays(resourceMap map[string]map[string]proto.Message) map[string][]proto.Message { + resources := make(map[string][]proto.Message) + resources[xdscommon.ListenerType] = make([]proto.Message, 0) + resources[xdscommon.RouteType] = make([]proto.Message, 0) + resources[xdscommon.ClusterType] = make([]proto.Message, 0) + resources[xdscommon.EndpointType] = make([]proto.Message, 0) + + // This conversion incurs processing cost which is done once in the generating envoy resources. + // This tradeoff is preferable to doing array scan every time an envoy resource needs to be + // to pr.envoyResource to see if it already exists. + for resourceTypeName, resourceMap := range resourceMap { + for _, resource := range resourceMap { + resources[resourceTypeName] = append(resources[resourceTypeName], resource) + } } - - pr.envoyResources[xdscommon.ListenerType] = listeners - - clusters, err := pr.makeXDSClusters() - if err != nil { - return err - } - pr.envoyResources[xdscommon.ClusterType] = clusters - - endpoints, err := pr.makeXDSEndpoints() - if err != nil { - return err - } - pr.envoyResources[xdscommon.EndpointType] = endpoints - - routes, err := pr.makeXDSRoutes() - if err != nil { - return err - } - pr.envoyResources[xdscommon.RouteType] = routes - - return nil + return resources } diff --git a/agent/xdsv2/route_resources.go b/agent/xdsv2/route_resources.go index af06c17448..d2306fd436 100644 --- a/agent/xdsv2/route_resources.go +++ b/agent/xdsv2/route_resources.go @@ -16,21 +16,8 @@ import ( "github.com/hashicorp/consul/proto-public/pbmesh/v2beta1/pbproxystate" envoy_route_v3 "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" - "google.golang.org/protobuf/proto" ) -func (pr *ProxyResources) makeXDSRoutes() ([]proto.Message, error) { - routes := make([]proto.Message, 0) - - for name, r := range pr.proxyState.Routes { - protoRoute := pr.makeEnvoyRouteConfigFromProxystateRoute(name, r) - // TODO: aggregate errors for routes and still return any properly formed routes. - routes = append(routes, protoRoute) - } - - return routes, nil -} - func (pr *ProxyResources) makeEnvoyRoute(name string) (*envoy_route_v3.RouteConfiguration, error) { var route *envoy_route_v3.RouteConfiguration // TODO(proxystate): This will make routes in the future. This function should distinguish between static routes @@ -247,6 +234,20 @@ func makeEnvoyQueryParamFromProxystateQueryMatch(psMatch *pbproxystate.QueryPara return envoyQueryParamMatcher } +func (pr *ProxyResources) addEnvoyClustersAndEndpointsToEnvoyResources(clusterName string) { + clusters, _ := pr.makeClusters(clusterName) + for name, cluster := range clusters { + pr.envoyResources[xdscommon.ClusterType][name] = cluster + + if name != xdscommon.LocalAppClusterName { + if endpointList, ok := pr.proxyState.Endpoints[name]; ok { + protoEndpoint := makeEnvoyClusterLoadAssignment(name, endpointList.Endpoints) + pr.envoyResources[xdscommon.EndpointType][name] = protoEndpoint + } + } + } +} + // TODO (dans): Will this always be envoy_route_v3.Route_Route? // Definitely for connect proxies this is the only option. func (pr *ProxyResources) makeEnvoyRouteActionFromProxystateRouteDestination(psRouteDestination *pbproxystate.RouteDestination) *envoy_route_v3.Route_Route { @@ -260,16 +261,15 @@ func (pr *ProxyResources) makeEnvoyRouteActionFromProxystateRouteDestination(psR envoyRouteRoute.Route.ClusterSpecifier = &envoy_route_v3.RouteAction_Cluster{ Cluster: psCluster.GetName(), } - clusters, _ := pr.makeClusters(psCluster.Name) - pr.envoyResources[xdscommon.ClusterType] = append(pr.envoyResources[xdscommon.ClusterType], clusters...) + pr.addEnvoyClustersAndEndpointsToEnvoyResources(psCluster.Name) case *pbproxystate.RouteDestination_WeightedClusters: psWeightedClusters := psRouteDestination.GetWeightedClusters() envoyClusters := make([]*envoy_route_v3.WeightedCluster_ClusterWeight, 0, len(psWeightedClusters.GetClusters())) totalWeight := 0 for _, psCluster := range psWeightedClusters.GetClusters() { - clusters, _ := pr.makeClusters(psCluster.Name) - pr.envoyResources[xdscommon.ClusterType] = append(pr.envoyResources[xdscommon.ClusterType], clusters...) + pr.addEnvoyClustersAndEndpointsToEnvoyResources(psCluster.Name) + totalWeight += int(psCluster.Weight.GetValue()) envoyClusters = append(envoyClusters, makeEnvoyClusterWeightFromProxystateWeightedCluster(psCluster)) } @@ -318,10 +318,7 @@ func (pr *ProxyResources) makeEnvoyRouteActionFromProxystateRouteDestination(psR } func makeEnvoyClusterWeightFromProxystateWeightedCluster(cluster *pbproxystate.L7WeightedDestinationCluster) *envoy_route_v3.WeightedCluster_ClusterWeight { - envoyClusterWeight := &envoy_route_v3.WeightedCluster_ClusterWeight{ - Name: cluster.GetName(), - Weight: cluster.GetWeight(), - } + envoyClusterWeight := makeEnvoyClusterWeightFromNameAndWeight(cluster.GetName(), cluster.GetWeight()) for _, hm := range cluster.GetHeaderMutations() { injectEnvoyClusterWeightWithProxystateHeaderMutation(envoyClusterWeight, hm) @@ -330,6 +327,15 @@ func makeEnvoyClusterWeightFromProxystateWeightedCluster(cluster *pbproxystate.L return envoyClusterWeight } +func makeEnvoyClusterWeightFromNameAndWeight(name string, weight *wrapperspb.UInt32Value) *envoy_route_v3.WeightedCluster_ClusterWeight { + envoyClusterWeight := &envoy_route_v3.WeightedCluster_ClusterWeight{ + Name: name, + Weight: weight, + } + + return envoyClusterWeight +} + func injectEnvoyClusterWeightWithProxystateHeaderMutation(envoyClusterWeight *envoy_route_v3.WeightedCluster_ClusterWeight, mutation *pbproxystate.HeaderMutation) { mutation.GetAction() switch mutation.GetAction().(type) { diff --git a/agent/xdsv2/testdata/clusters/destination/mixed-multi-destination.golden b/agent/xdsv2/testdata/clusters/destination/mixed-multi-destination.golden index 280f42b858..b87031ba81 100644 --- a/agent/xdsv2/testdata/clusters/destination/mixed-multi-destination.golden +++ b/agent/xdsv2/testdata/clusters/destination/mixed-multi-destination.golden @@ -1,157 +1,270 @@ { - "versionInfo": "00000001", - "resources": [ + "versionInfo": "00000001", + "resources": [ { - "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", - "name": "http.api-2.default.dc1.internal.foo.consul", - "type": "EDS", - "edsClusterConfig": { - "edsConfig": { - "ads": {}, - "resourceApiVersion": "V3" + "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "name": "failover-target~0~http.api-1.default.dc1.internal.foo.consul", + "type": "EDS", + "edsClusterConfig": { + "edsConfig": { + "ads": {}, + "resourceApiVersion": "V3" } }, - "connectTimeout": "5s", - "commonLbConfig": { - "healthyPanicThreshold": {} + "connectTimeout": "55s", + "commonLbConfig": { + "healthyPanicThreshold": {} }, - "transportSocket": { - "name": "tls", - "typedConfig": { - "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext", - "commonTlsContext": { - "tlsParams": {}, - "tlsCertificates": [ + "transportSocket": { + "name": "tls", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext", + "commonTlsContext": { + "tlsParams": {}, + "tlsCertificates": [ { - "certificateChain": { - "inlineString": "-----BEGIN CERTIFICATE-----\nMIICDjCCAbWgAwIBAgIBAjAKBggqhkjOPQQDAjAUMRIwEAYDVQQDEwlUZXN0IENB\nIDEwHhcNMjMxMDE2MTYxMzI5WhcNMjMxMDE2MTYyMzI5WjAAMFkwEwYHKoZIzj0C\nAQYIKoZIzj0DAQcDQgAErErAIosDPheZQGbxFQ4hYC/e9Fi4MG9z/zjfCnCq/oK9\nta/bGT+5orZqTmdN/ICsKQDhykxZ2u/Xr6845zhcJaOCAQowggEGMA4GA1UdDwEB\n/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcDAgYIKwYBBQUHAwEwDAYDVR0TAQH/\nBAIwADApBgNVHQ4EIgQg3ogXVz9cqaK2B6xdiJYMa5NtT0KkYv7BA2dR7h9EcwUw\nKwYDVR0jBCQwIoAgq+C1mPlPoGa4lt7sSft1goN5qPGyBIB/3mUHJZKSFY8wbwYD\nVR0RAQH/BGUwY4Zhc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9hcC9kZWZhdWx0L25zL2RlZmF1bHQvaWRlbnRpdHkv\ndGVzdC1pZGVudGl0eTAKBggqhkjOPQQDAgNHADBEAiB6L+t5bzRrBPhiQYNeA7fF\nUCuLWrdjW4Xbv3SLg0IKMgIgfRC5hEx+DqzQxTCP4sexX3hVWMjKoWmHdwiUcg+K\n/IE=\n-----END CERTIFICATE-----\n" + "certificateChain": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICDjCCAbWgAwIBAgIBAjAKBggqhkjOPQQDAjAUMRIwEAYDVQQDEwlUZXN0IENB\nIDEwHhcNMjMxMDE2MTYxMzI5WhcNMjMxMDE2MTYyMzI5WjAAMFkwEwYHKoZIzj0C\nAQYIKoZIzj0DAQcDQgAErErAIosDPheZQGbxFQ4hYC/e9Fi4MG9z/zjfCnCq/oK9\nta/bGT+5orZqTmdN/ICsKQDhykxZ2u/Xr6845zhcJaOCAQowggEGMA4GA1UdDwEB\n/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcDAgYIKwYBBQUHAwEwDAYDVR0TAQH/\nBAIwADApBgNVHQ4EIgQg3ogXVz9cqaK2B6xdiJYMa5NtT0KkYv7BA2dR7h9EcwUw\nKwYDVR0jBCQwIoAgq+C1mPlPoGa4lt7sSft1goN5qPGyBIB/3mUHJZKSFY8wbwYD\nVR0RAQH/BGUwY4Zhc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9hcC9kZWZhdWx0L25zL2RlZmF1bHQvaWRlbnRpdHkv\ndGVzdC1pZGVudGl0eTAKBggqhkjOPQQDAgNHADBEAiB6L+t5bzRrBPhiQYNeA7fF\nUCuLWrdjW4Xbv3SLg0IKMgIgfRC5hEx+DqzQxTCP4sexX3hVWMjKoWmHdwiUcg+K\n/IE=\n-----END CERTIFICATE-----\n" }, - "privateKey": { - "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIFIFkTIL1iUV4O/RpveVHzHs7ZzhSkvYIzbdXDttz9EooAoGCCqGSM49\nAwEHoUQDQgAErErAIosDPheZQGbxFQ4hYC/e9Fi4MG9z/zjfCnCq/oK9ta/bGT+5\norZqTmdN/ICsKQDhykxZ2u/Xr6845zhcJQ==\n-----END EC PRIVATE KEY-----\n" + "privateKey": { + "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIFIFkTIL1iUV4O/RpveVHzHs7ZzhSkvYIzbdXDttz9EooAoGCCqGSM49\nAwEHoUQDQgAErErAIosDPheZQGbxFQ4hYC/e9Fi4MG9z/zjfCnCq/oK9ta/bGT+5\norZqTmdN/ICsKQDhykxZ2u/Xr6845zhcJQ==\n-----END EC PRIVATE KEY-----\n" } } ], - "validationContext": { - "trustedCa": { - "inlineString": "some-root\nsome-other-root\n" + "validationContext": { + "trustedCa": { + "inlineString": "some-root\nsome-other-root\n" }, - "matchSubjectAltNames": [ + "matchSubjectAltNames": [ { - "exact": "spiffe://foo.consul/ap/default/ns/default/identity/api2-identity" + "exact": "spiffe://foo.consul/ap/default/ns/default/identity/api1-identity" } ] }, - "alpnProtocols": [ + "alpnProtocols": [ "consul~http" ] }, - "sni": "api-2.default.dc1.internal.foo.consul" + "sni": "api-1.default.dc1.internal.foo.consul" } } }, { - "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", - "name": "null_route_cluster", - "type": "STATIC", - "connectTimeout": "10s" - }, - { - "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", - "name": "tcp.api-1.default.dc1.internal.foo.consul", - "type": "EDS", - "edsClusterConfig": { - "edsConfig": { - "ads": {}, - "resourceApiVersion": "V3" + "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "name": "failover-target~1~http.api-1.default.dc1.internal.foo.consul", + "type": "EDS", + "edsClusterConfig": { + "edsConfig": { + "ads": {}, + "resourceApiVersion": "V3" } }, - "connectTimeout": "5s", - "commonLbConfig": { - "healthyPanicThreshold": {} + "connectTimeout": "5s", + "commonLbConfig": { + "healthyPanicThreshold": {} }, - "transportSocket": { - "name": "tls", - "typedConfig": { - "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext", - "commonTlsContext": { - "tlsParams": {}, - "tlsCertificates": [ + "transportSocket": { + "name": "tls", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext", + "commonTlsContext": { + "tlsParams": {}, + "tlsCertificates": [ { - "certificateChain": { - "inlineString": "-----BEGIN CERTIFICATE-----\nMIICDjCCAbWgAwIBAgIBAjAKBggqhkjOPQQDAjAUMRIwEAYDVQQDEwlUZXN0IENB\nIDEwHhcNMjMxMDE2MTYxMzI5WhcNMjMxMDE2MTYyMzI5WjAAMFkwEwYHKoZIzj0C\nAQYIKoZIzj0DAQcDQgAErErAIosDPheZQGbxFQ4hYC/e9Fi4MG9z/zjfCnCq/oK9\nta/bGT+5orZqTmdN/ICsKQDhykxZ2u/Xr6845zhcJaOCAQowggEGMA4GA1UdDwEB\n/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcDAgYIKwYBBQUHAwEwDAYDVR0TAQH/\nBAIwADApBgNVHQ4EIgQg3ogXVz9cqaK2B6xdiJYMa5NtT0KkYv7BA2dR7h9EcwUw\nKwYDVR0jBCQwIoAgq+C1mPlPoGa4lt7sSft1goN5qPGyBIB/3mUHJZKSFY8wbwYD\nVR0RAQH/BGUwY4Zhc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9hcC9kZWZhdWx0L25zL2RlZmF1bHQvaWRlbnRpdHkv\ndGVzdC1pZGVudGl0eTAKBggqhkjOPQQDAgNHADBEAiB6L+t5bzRrBPhiQYNeA7fF\nUCuLWrdjW4Xbv3SLg0IKMgIgfRC5hEx+DqzQxTCP4sexX3hVWMjKoWmHdwiUcg+K\n/IE=\n-----END CERTIFICATE-----\n" + "certificateChain": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICDjCCAbWgAwIBAgIBAjAKBggqhkjOPQQDAjAUMRIwEAYDVQQDEwlUZXN0IENB\nIDEwHhcNMjMxMDE2MTYxMzI5WhcNMjMxMDE2MTYyMzI5WjAAMFkwEwYHKoZIzj0C\nAQYIKoZIzj0DAQcDQgAErErAIosDPheZQGbxFQ4hYC/e9Fi4MG9z/zjfCnCq/oK9\nta/bGT+5orZqTmdN/ICsKQDhykxZ2u/Xr6845zhcJaOCAQowggEGMA4GA1UdDwEB\n/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcDAgYIKwYBBQUHAwEwDAYDVR0TAQH/\nBAIwADApBgNVHQ4EIgQg3ogXVz9cqaK2B6xdiJYMa5NtT0KkYv7BA2dR7h9EcwUw\nKwYDVR0jBCQwIoAgq+C1mPlPoGa4lt7sSft1goN5qPGyBIB/3mUHJZKSFY8wbwYD\nVR0RAQH/BGUwY4Zhc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9hcC9kZWZhdWx0L25zL2RlZmF1bHQvaWRlbnRpdHkv\ndGVzdC1pZGVudGl0eTAKBggqhkjOPQQDAgNHADBEAiB6L+t5bzRrBPhiQYNeA7fF\nUCuLWrdjW4Xbv3SLg0IKMgIgfRC5hEx+DqzQxTCP4sexX3hVWMjKoWmHdwiUcg+K\n/IE=\n-----END CERTIFICATE-----\n" }, - "privateKey": { - "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIFIFkTIL1iUV4O/RpveVHzHs7ZzhSkvYIzbdXDttz9EooAoGCCqGSM49\nAwEHoUQDQgAErErAIosDPheZQGbxFQ4hYC/e9Fi4MG9z/zjfCnCq/oK9ta/bGT+5\norZqTmdN/ICsKQDhykxZ2u/Xr6845zhcJQ==\n-----END EC PRIVATE KEY-----\n" + "privateKey": { + "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIFIFkTIL1iUV4O/RpveVHzHs7ZzhSkvYIzbdXDttz9EooAoGCCqGSM49\nAwEHoUQDQgAErErAIosDPheZQGbxFQ4hYC/e9Fi4MG9z/zjfCnCq/oK9ta/bGT+5\norZqTmdN/ICsKQDhykxZ2u/Xr6845zhcJQ==\n-----END EC PRIVATE KEY-----\n" } } ], - "validationContext": { - "trustedCa": { - "inlineString": "some-root\nsome-other-root\n" + "validationContext": { + "trustedCa": { + "inlineString": "some-root\nsome-other-root\n" }, - "matchSubjectAltNames": [ + "matchSubjectAltNames": [ { - "exact": "spiffe://foo.consul/ap/default/ns/default/identity/api1-identity" + "exact": "spiffe://foo.consul/ap/default/ns/default/identity/backup1-identity" } ] }, - "alpnProtocols": [ - "consul~tcp" + "alpnProtocols": [ + "consul~http" ] }, - "sni": "api-1.default.dc1.internal.foo.consul" + "sni": "backup-1.default.dc1.internal.foo.consul" } } }, { - "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", - "name": "tcp.api-2.default.dc1.internal.foo.consul", - "type": "EDS", - "edsClusterConfig": { - "edsConfig": { - "ads": {}, - "resourceApiVersion": "V3" + "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "name": "http.api-1.default.dc1.internal.foo.consul", + "altStatName": "http.api-1.default.dc1.internal.foo.consul", + "clusterType": { + "name": "envoy.clusters.aggregate", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.clusters.aggregate.v3.ClusterConfig", + "clusters": [ + "failover-target~0~http.api-1.default.dc1.internal.foo.consul", + "failover-target~1~http.api-1.default.dc1.internal.foo.consul" + ] } }, - "connectTimeout": "5s", - "commonLbConfig": { - "healthyPanicThreshold": {} + "connectTimeout": "55s", + "lbPolicy": "CLUSTER_PROVIDED" + }, + { + "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "name": "http.api-2.default.dc1.internal.foo.consul", + "type": "EDS", + "edsClusterConfig": { + "edsConfig": { + "ads": {}, + "resourceApiVersion": "V3" + } }, - "transportSocket": { - "name": "tls", - "typedConfig": { - "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext", - "commonTlsContext": { - "tlsParams": {}, - "tlsCertificates": [ + "connectTimeout": "5s", + "commonLbConfig": { + "healthyPanicThreshold": {} + }, + "transportSocket": { + "name": "tls", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext", + "commonTlsContext": { + "tlsParams": {}, + "tlsCertificates": [ { - "certificateChain": { - "inlineString": "-----BEGIN CERTIFICATE-----\nMIICDjCCAbWgAwIBAgIBAjAKBggqhkjOPQQDAjAUMRIwEAYDVQQDEwlUZXN0IENB\nIDEwHhcNMjMxMDE2MTYxMzI5WhcNMjMxMDE2MTYyMzI5WjAAMFkwEwYHKoZIzj0C\nAQYIKoZIzj0DAQcDQgAErErAIosDPheZQGbxFQ4hYC/e9Fi4MG9z/zjfCnCq/oK9\nta/bGT+5orZqTmdN/ICsKQDhykxZ2u/Xr6845zhcJaOCAQowggEGMA4GA1UdDwEB\n/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcDAgYIKwYBBQUHAwEwDAYDVR0TAQH/\nBAIwADApBgNVHQ4EIgQg3ogXVz9cqaK2B6xdiJYMa5NtT0KkYv7BA2dR7h9EcwUw\nKwYDVR0jBCQwIoAgq+C1mPlPoGa4lt7sSft1goN5qPGyBIB/3mUHJZKSFY8wbwYD\nVR0RAQH/BGUwY4Zhc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9hcC9kZWZhdWx0L25zL2RlZmF1bHQvaWRlbnRpdHkv\ndGVzdC1pZGVudGl0eTAKBggqhkjOPQQDAgNHADBEAiB6L+t5bzRrBPhiQYNeA7fF\nUCuLWrdjW4Xbv3SLg0IKMgIgfRC5hEx+DqzQxTCP4sexX3hVWMjKoWmHdwiUcg+K\n/IE=\n-----END CERTIFICATE-----\n" + "certificateChain": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICDjCCAbWgAwIBAgIBAjAKBggqhkjOPQQDAjAUMRIwEAYDVQQDEwlUZXN0IENB\nIDEwHhcNMjMxMDE2MTYxMzI5WhcNMjMxMDE2MTYyMzI5WjAAMFkwEwYHKoZIzj0C\nAQYIKoZIzj0DAQcDQgAErErAIosDPheZQGbxFQ4hYC/e9Fi4MG9z/zjfCnCq/oK9\nta/bGT+5orZqTmdN/ICsKQDhykxZ2u/Xr6845zhcJaOCAQowggEGMA4GA1UdDwEB\n/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcDAgYIKwYBBQUHAwEwDAYDVR0TAQH/\nBAIwADApBgNVHQ4EIgQg3ogXVz9cqaK2B6xdiJYMa5NtT0KkYv7BA2dR7h9EcwUw\nKwYDVR0jBCQwIoAgq+C1mPlPoGa4lt7sSft1goN5qPGyBIB/3mUHJZKSFY8wbwYD\nVR0RAQH/BGUwY4Zhc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9hcC9kZWZhdWx0L25zL2RlZmF1bHQvaWRlbnRpdHkv\ndGVzdC1pZGVudGl0eTAKBggqhkjOPQQDAgNHADBEAiB6L+t5bzRrBPhiQYNeA7fF\nUCuLWrdjW4Xbv3SLg0IKMgIgfRC5hEx+DqzQxTCP4sexX3hVWMjKoWmHdwiUcg+K\n/IE=\n-----END CERTIFICATE-----\n" }, - "privateKey": { - "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIFIFkTIL1iUV4O/RpveVHzHs7ZzhSkvYIzbdXDttz9EooAoGCCqGSM49\nAwEHoUQDQgAErErAIosDPheZQGbxFQ4hYC/e9Fi4MG9z/zjfCnCq/oK9ta/bGT+5\norZqTmdN/ICsKQDhykxZ2u/Xr6845zhcJQ==\n-----END EC PRIVATE KEY-----\n" + "privateKey": { + "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIFIFkTIL1iUV4O/RpveVHzHs7ZzhSkvYIzbdXDttz9EooAoGCCqGSM49\nAwEHoUQDQgAErErAIosDPheZQGbxFQ4hYC/e9Fi4MG9z/zjfCnCq/oK9ta/bGT+5\norZqTmdN/ICsKQDhykxZ2u/Xr6845zhcJQ==\n-----END EC PRIVATE KEY-----\n" } } ], - "validationContext": { - "trustedCa": { - "inlineString": "some-root\nsome-other-root\n" + "validationContext": { + "trustedCa": { + "inlineString": "some-root\nsome-other-root\n" }, - "matchSubjectAltNames": [ + "matchSubjectAltNames": [ { - "exact": "spiffe://foo.consul/ap/default/ns/default/identity/api2-identity" + "exact": "spiffe://foo.consul/ap/default/ns/default/identity/api2-identity" } ] }, - "alpnProtocols": [ + "alpnProtocols": [ + "consul~http" + ] + }, + "sni": "api-2.default.dc1.internal.foo.consul" + } + } + }, + { + "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "name": "null_route_cluster", + "type": "STATIC", + "connectTimeout": "10s" + }, + { + "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "name": "tcp.api-1.default.dc1.internal.foo.consul", + "type": "EDS", + "edsClusterConfig": { + "edsConfig": { + "ads": {}, + "resourceApiVersion": "V3" + } + }, + "connectTimeout": "5s", + "commonLbConfig": { + "healthyPanicThreshold": {} + }, + "transportSocket": { + "name": "tls", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext", + "commonTlsContext": { + "tlsParams": {}, + "tlsCertificates": [ + { + "certificateChain": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICDjCCAbWgAwIBAgIBAjAKBggqhkjOPQQDAjAUMRIwEAYDVQQDEwlUZXN0IENB\nIDEwHhcNMjMxMDE2MTYxMzI5WhcNMjMxMDE2MTYyMzI5WjAAMFkwEwYHKoZIzj0C\nAQYIKoZIzj0DAQcDQgAErErAIosDPheZQGbxFQ4hYC/e9Fi4MG9z/zjfCnCq/oK9\nta/bGT+5orZqTmdN/ICsKQDhykxZ2u/Xr6845zhcJaOCAQowggEGMA4GA1UdDwEB\n/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcDAgYIKwYBBQUHAwEwDAYDVR0TAQH/\nBAIwADApBgNVHQ4EIgQg3ogXVz9cqaK2B6xdiJYMa5NtT0KkYv7BA2dR7h9EcwUw\nKwYDVR0jBCQwIoAgq+C1mPlPoGa4lt7sSft1goN5qPGyBIB/3mUHJZKSFY8wbwYD\nVR0RAQH/BGUwY4Zhc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9hcC9kZWZhdWx0L25zL2RlZmF1bHQvaWRlbnRpdHkv\ndGVzdC1pZGVudGl0eTAKBggqhkjOPQQDAgNHADBEAiB6L+t5bzRrBPhiQYNeA7fF\nUCuLWrdjW4Xbv3SLg0IKMgIgfRC5hEx+DqzQxTCP4sexX3hVWMjKoWmHdwiUcg+K\n/IE=\n-----END CERTIFICATE-----\n" + }, + "privateKey": { + "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIFIFkTIL1iUV4O/RpveVHzHs7ZzhSkvYIzbdXDttz9EooAoGCCqGSM49\nAwEHoUQDQgAErErAIosDPheZQGbxFQ4hYC/e9Fi4MG9z/zjfCnCq/oK9ta/bGT+5\norZqTmdN/ICsKQDhykxZ2u/Xr6845zhcJQ==\n-----END EC PRIVATE KEY-----\n" + } + } + ], + "validationContext": { + "trustedCa": { + "inlineString": "some-root\nsome-other-root\n" + }, + "matchSubjectAltNames": [ + { + "exact": "spiffe://foo.consul/ap/default/ns/default/identity/api1-identity" + } + ] + }, + "alpnProtocols": [ "consul~tcp" ] }, - "sni": "api-2.default.dc1.internal.foo.consul" + "sni": "api-1.default.dc1.internal.foo.consul" + } + } + }, + { + "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "name": "tcp.api-2.default.dc1.internal.foo.consul", + "type": "EDS", + "edsClusterConfig": { + "edsConfig": { + "ads": {}, + "resourceApiVersion": "V3" + } + }, + "connectTimeout": "5s", + "commonLbConfig": { + "healthyPanicThreshold": {} + }, + "transportSocket": { + "name": "tls", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext", + "commonTlsContext": { + "tlsParams": {}, + "tlsCertificates": [ + { + "certificateChain": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICDjCCAbWgAwIBAgIBAjAKBggqhkjOPQQDAjAUMRIwEAYDVQQDEwlUZXN0IENB\nIDEwHhcNMjMxMDE2MTYxMzI5WhcNMjMxMDE2MTYyMzI5WjAAMFkwEwYHKoZIzj0C\nAQYIKoZIzj0DAQcDQgAErErAIosDPheZQGbxFQ4hYC/e9Fi4MG9z/zjfCnCq/oK9\nta/bGT+5orZqTmdN/ICsKQDhykxZ2u/Xr6845zhcJaOCAQowggEGMA4GA1UdDwEB\n/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcDAgYIKwYBBQUHAwEwDAYDVR0TAQH/\nBAIwADApBgNVHQ4EIgQg3ogXVz9cqaK2B6xdiJYMa5NtT0KkYv7BA2dR7h9EcwUw\nKwYDVR0jBCQwIoAgq+C1mPlPoGa4lt7sSft1goN5qPGyBIB/3mUHJZKSFY8wbwYD\nVR0RAQH/BGUwY4Zhc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9hcC9kZWZhdWx0L25zL2RlZmF1bHQvaWRlbnRpdHkv\ndGVzdC1pZGVudGl0eTAKBggqhkjOPQQDAgNHADBEAiB6L+t5bzRrBPhiQYNeA7fF\nUCuLWrdjW4Xbv3SLg0IKMgIgfRC5hEx+DqzQxTCP4sexX3hVWMjKoWmHdwiUcg+K\n/IE=\n-----END CERTIFICATE-----\n" + }, + "privateKey": { + "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIFIFkTIL1iUV4O/RpveVHzHs7ZzhSkvYIzbdXDttz9EooAoGCCqGSM49\nAwEHoUQDQgAErErAIosDPheZQGbxFQ4hYC/e9Fi4MG9z/zjfCnCq/oK9ta/bGT+5\norZqTmdN/ICsKQDhykxZ2u/Xr6845zhcJQ==\n-----END EC PRIVATE KEY-----\n" + } + } + ], + "validationContext": { + "trustedCa": { + "inlineString": "some-root\nsome-other-root\n" + }, + "matchSubjectAltNames": [ + { + "exact": "spiffe://foo.consul/ap/default/ns/default/identity/api2-identity" + } + ] + }, + "alpnProtocols": [ + "consul~tcp" + ] + }, + "sni": "api-2.default.dc1.internal.foo.consul" } } } ], - "typeUrl": "type.googleapis.com/envoy.config.cluster.v3.Cluster", - "nonce": "00000001" + "typeUrl": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "nonce": "00000001" } \ No newline at end of file diff --git a/agent/xdsv2/testdata/routes/source/l7-expose-paths.golden b/agent/xdsv2/testdata/routes/source/l7-expose-paths.golden index 68b5239aec..306f5220e7 100644 --- a/agent/xdsv2/testdata/routes/source/l7-expose-paths.golden +++ b/agent/xdsv2/testdata/routes/source/l7-expose-paths.golden @@ -1,53 +1,5 @@ { "versionInfo": "00000001", - "resources": [ - { - "@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", - "name": "exposed_path_route_GetHealth1235", - "virtualHosts": [ - { - "name": "exposed_path_route_GetHealth1235", - "domains": [ - "*" - ], - "routes": [ - { - "match": { - "path": "GetHealth" - }, - "route": { - "cluster": "exposed_cluster_9091" - } - } - ] - } - ], - "validateClusters": true - }, - { - "@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", - "name": "exposed_path_route_health1234", - "virtualHosts": [ - { - "name": "exposed_path_route_health1234", - "domains": [ - "*" - ], - "routes": [ - { - "match": { - "path": "/health" - }, - "route": { - "cluster": "exposed_cluster_9090" - } - } - ] - } - ], - "validateClusters": true - } - ], "typeUrl": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", "nonce": "00000001" } \ No newline at end of file diff --git a/agent/xdsv2/testdata/routes/source/local-and-inbound-connections.golden b/agent/xdsv2/testdata/routes/source/local-and-inbound-connections.golden index 97c96f44bb..306f5220e7 100644 --- a/agent/xdsv2/testdata/routes/source/local-and-inbound-connections.golden +++ b/agent/xdsv2/testdata/routes/source/local-and-inbound-connections.golden @@ -1,77 +1,5 @@ { "versionInfo": "00000001", - "resources": [ - { - "@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", - "name": "exposed_path_route_GetHealth1235", - "virtualHosts": [ - { - "name": "exposed_path_route_GetHealth1235", - "domains": [ - "*" - ], - "routes": [ - { - "match": { - "path": "GetHealth" - }, - "route": { - "cluster": "exposed_cluster_9091" - } - } - ] - } - ], - "validateClusters": true - }, - { - "@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", - "name": "exposed_path_route_health1234", - "virtualHosts": [ - { - "name": "exposed_path_route_health1234", - "domains": [ - "*" - ], - "routes": [ - { - "match": { - "path": "/health" - }, - "route": { - "cluster": "exposed_cluster_9090" - } - } - ] - } - ], - "validateClusters": true - }, - { - "@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", - "name": "public_listener:port3", - "virtualHosts": [ - { - "name": "public_listener:port3", - "domains": [ - "*" - ], - "routes": [ - { - "match": { - "prefix": "/" - }, - "route": { - "cluster": "local_app:port3", - "timeout": "9s" - } - } - ] - } - ], - "validateClusters": true - } - ], "typeUrl": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", "nonce": "00000001" } \ No newline at end of file diff --git a/agent/xdsv2/testdata/routes/source/multiple-workload-addresses-with-specific-ports.golden b/agent/xdsv2/testdata/routes/source/multiple-workload-addresses-with-specific-ports.golden index c114a36cfe..306f5220e7 100644 --- a/agent/xdsv2/testdata/routes/source/multiple-workload-addresses-with-specific-ports.golden +++ b/agent/xdsv2/testdata/routes/source/multiple-workload-addresses-with-specific-ports.golden @@ -1,76 +1,5 @@ { "versionInfo": "00000001", - "resources": [ - { - "@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", - "name": "public_listener:grpc", - "virtualHosts": [ - { - "name": "public_listener:grpc", - "domains": [ - "*" - ], - "routes": [ - { - "match": { - "prefix": "/" - }, - "route": { - "cluster": "local_app:grpc" - } - } - ] - } - ], - "validateClusters": true - }, - { - "@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", - "name": "public_listener:http", - "virtualHosts": [ - { - "name": "public_listener:http", - "domains": [ - "*" - ], - "routes": [ - { - "match": { - "prefix": "/" - }, - "route": { - "cluster": "local_app:http" - } - } - ] - } - ], - "validateClusters": true - }, - { - "@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", - "name": "public_listener:http2", - "virtualHosts": [ - { - "name": "public_listener:http2", - "domains": [ - "*" - ], - "routes": [ - { - "match": { - "prefix": "/" - }, - "route": { - "cluster": "local_app:http2" - } - } - ] - } - ], - "validateClusters": true - } - ], "typeUrl": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", "nonce": "00000001" } \ No newline at end of file diff --git a/agent/xdsv2/testdata/routes/source/multiple-workload-addresses-without-ports.golden b/agent/xdsv2/testdata/routes/source/multiple-workload-addresses-without-ports.golden index c114a36cfe..306f5220e7 100644 --- a/agent/xdsv2/testdata/routes/source/multiple-workload-addresses-without-ports.golden +++ b/agent/xdsv2/testdata/routes/source/multiple-workload-addresses-without-ports.golden @@ -1,76 +1,5 @@ { "versionInfo": "00000001", - "resources": [ - { - "@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", - "name": "public_listener:grpc", - "virtualHosts": [ - { - "name": "public_listener:grpc", - "domains": [ - "*" - ], - "routes": [ - { - "match": { - "prefix": "/" - }, - "route": { - "cluster": "local_app:grpc" - } - } - ] - } - ], - "validateClusters": true - }, - { - "@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", - "name": "public_listener:http", - "virtualHosts": [ - { - "name": "public_listener:http", - "domains": [ - "*" - ], - "routes": [ - { - "match": { - "prefix": "/" - }, - "route": { - "cluster": "local_app:http" - } - } - ] - } - ], - "validateClusters": true - }, - { - "@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", - "name": "public_listener:http2", - "virtualHosts": [ - { - "name": "public_listener:http2", - "domains": [ - "*" - ], - "routes": [ - { - "match": { - "prefix": "/" - }, - "route": { - "cluster": "local_app:http2" - } - } - ] - } - ], - "validateClusters": true - } - ], "typeUrl": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", "nonce": "00000001" } \ No newline at end of file diff --git a/agent/xdsv2/testdata/routes/source/multiport-l7-multiple-workload-addresses-with-specific-ports.golden b/agent/xdsv2/testdata/routes/source/multiport-l7-multiple-workload-addresses-with-specific-ports.golden index 3b1a61403b..306f5220e7 100644 --- a/agent/xdsv2/testdata/routes/source/multiport-l7-multiple-workload-addresses-with-specific-ports.golden +++ b/agent/xdsv2/testdata/routes/source/multiport-l7-multiple-workload-addresses-with-specific-ports.golden @@ -1,53 +1,5 @@ { "versionInfo": "00000001", - "resources": [ - { - "@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", - "name": "public_listener:admin-port", - "virtualHosts": [ - { - "name": "public_listener:admin-port", - "domains": [ - "*" - ], - "routes": [ - { - "match": { - "prefix": "/" - }, - "route": { - "cluster": "local_app:admin-port" - } - } - ] - } - ], - "validateClusters": true - }, - { - "@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", - "name": "public_listener:api-port", - "virtualHosts": [ - { - "name": "public_listener:api-port", - "domains": [ - "*" - ], - "routes": [ - { - "match": { - "prefix": "/" - }, - "route": { - "cluster": "local_app:api-port" - } - } - ] - } - ], - "validateClusters": true - } - ], "typeUrl": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", "nonce": "00000001" } \ No newline at end of file diff --git a/agent/xdsv2/testdata/routes/source/multiport-l7-multiple-workload-addresses-without-ports.golden b/agent/xdsv2/testdata/routes/source/multiport-l7-multiple-workload-addresses-without-ports.golden index 7f976890c6..306f5220e7 100644 --- a/agent/xdsv2/testdata/routes/source/multiport-l7-multiple-workload-addresses-without-ports.golden +++ b/agent/xdsv2/testdata/routes/source/multiport-l7-multiple-workload-addresses-without-ports.golden @@ -1,76 +1,5 @@ { "versionInfo": "00000001", - "resources": [ - { - "@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", - "name": "public_listener:admin-port", - "virtualHosts": [ - { - "name": "public_listener:admin-port", - "domains": [ - "*" - ], - "routes": [ - { - "match": { - "prefix": "/" - }, - "route": { - "cluster": "local_app:admin-port" - } - } - ] - } - ], - "validateClusters": true - }, - { - "@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", - "name": "public_listener:api-port", - "virtualHosts": [ - { - "name": "public_listener:api-port", - "domains": [ - "*" - ], - "routes": [ - { - "match": { - "prefix": "/" - }, - "route": { - "cluster": "local_app:api-port" - } - } - ] - } - ], - "validateClusters": true - }, - { - "@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", - "name": "public_listener:grpc-port", - "virtualHosts": [ - { - "name": "public_listener:grpc-port", - "domains": [ - "*" - ], - "routes": [ - { - "match": { - "prefix": "/" - }, - "route": { - "cluster": "local_app:grpc-port" - } - } - ] - } - ], - "validateClusters": true - } - ], "typeUrl": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", "nonce": "00000001" } \ No newline at end of file diff --git a/agent/xdsv2/testdata/routes/source/single-workload-address-without-ports.golden b/agent/xdsv2/testdata/routes/source/single-workload-address-without-ports.golden index c114a36cfe..306f5220e7 100644 --- a/agent/xdsv2/testdata/routes/source/single-workload-address-without-ports.golden +++ b/agent/xdsv2/testdata/routes/source/single-workload-address-without-ports.golden @@ -1,76 +1,5 @@ { "versionInfo": "00000001", - "resources": [ - { - "@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", - "name": "public_listener:grpc", - "virtualHosts": [ - { - "name": "public_listener:grpc", - "domains": [ - "*" - ], - "routes": [ - { - "match": { - "prefix": "/" - }, - "route": { - "cluster": "local_app:grpc" - } - } - ] - } - ], - "validateClusters": true - }, - { - "@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", - "name": "public_listener:http", - "virtualHosts": [ - { - "name": "public_listener:http", - "domains": [ - "*" - ], - "routes": [ - { - "match": { - "prefix": "/" - }, - "route": { - "cluster": "local_app:http" - } - } - ] - } - ], - "validateClusters": true - }, - { - "@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", - "name": "public_listener:http2", - "virtualHosts": [ - { - "name": "public_listener:http2", - "domains": [ - "*" - ], - "routes": [ - { - "match": { - "prefix": "/" - }, - "route": { - "cluster": "local_app:http2" - } - } - ] - } - ], - "validateClusters": true - } - ], "typeUrl": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", "nonce": "00000001" } \ No newline at end of file