From ab60fec15ae5f7426ef25b5a5cb4d89aecf1387f Mon Sep 17 00:00:00 2001 From: Nathan Coleman Date: Thu, 21 Dec 2023 16:37:47 -0500 Subject: [PATCH] [NET-6426] Add gateway proxy controller that generates empty proxy state template (#19901) * NET-6426 Create ProxyStateTemplate when reconciling MeshGateway resource * Add TODO for switching fetch method based on gateway type * Use gateway-kind in workload metadata instead of owner reference * Create ProxyStateTemplate builder for gatewayproxy controller * Update to use new controller interface * Add copyright headers * Set correct name for ProxyStateTemplate identity reference * Generate empty ProxyStateTemplate by fetching MeshGateway This cheats and looks up the MeshGateway directly. In the future, we will need a Workload => xGateway mapper * Specify owner reference when writing ProxyStateTemplate * Update dependency mapper to account for multiple controllers per resource type * Regenerate v2 resource dependencies map * Add helpful trace logs, tag TODOs with ticket identifiers --- .../testdata/v2-resource-dependencies.md | 1 + internal/controller/dependencies.go | 20 ++- .../gatewayproxy/builder/builder.go | 65 +++++++++ .../controllers/gatewayproxy/controller.go | 137 ++++++++++++++++++ .../gatewayproxy/fetcher/data_fetcher.go | 60 ++++++++ .../mesh/internal/controllers/register.go | 4 + internal/mesh/internal/types/decoded.go | 1 + 7 files changed, 284 insertions(+), 4 deletions(-) create mode 100644 internal/mesh/internal/controllers/gatewayproxy/builder/builder.go create mode 100644 internal/mesh/internal/controllers/gatewayproxy/controller.go create mode 100644 internal/mesh/internal/controllers/gatewayproxy/fetcher/data_fetcher.go diff --git a/agent/consul/testdata/v2-resource-dependencies.md b/agent/consul/testdata/v2-resource-dependencies.md index 5a4e767085..9e90a24947 100644 --- a/agent/consul/testdata/v2-resource-dependencies.md +++ b/agent/consul/testdata/v2-resource-dependencies.md @@ -41,6 +41,7 @@ flowchart TD mesh/v2beta1/proxyconfiguration mesh/v2beta1/proxystatetemplate --> auth/v2beta1/computedtrafficpermissions mesh/v2beta1/proxystatetemplate --> catalog/v2beta1/service + mesh/v2beta1/proxystatetemplate --> catalog/v2beta1/serviceendpoints mesh/v2beta1/proxystatetemplate --> catalog/v2beta1/workload mesh/v2beta1/proxystatetemplate --> mesh/v2beta1/computedexplicitdestinations mesh/v2beta1/proxystatetemplate --> mesh/v2beta1/computedproxyconfiguration diff --git a/internal/controller/dependencies.go b/internal/controller/dependencies.go index a7ae0e96d6..fb3f151beb 100644 --- a/internal/controller/dependencies.go +++ b/internal/controller/dependencies.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/hashicorp/go-multierror" + "golang.org/x/exp/maps" "github.com/hashicorp/consul/internal/resource" "github.com/hashicorp/consul/proto-public/pbresource" @@ -66,12 +67,23 @@ func (m *Manager) CalculateDependencies(registrations []resource.Registration) D } for _, c := range m.controllers { - watches := make([]string, 0, len(c.watches)) - for _, w := range c.watches { - watches = append(watches, typeToString(w.watchedType)) + watches := map[string]struct{}{} + + // Extend existing watch list if one is present. This is necessary + // because there can be multiple controllers for a given type. + // ProxyStateTemplate, for example, is controlled by sidecar proxy and + // gateway proxy controllers. + if existing, ok := out[typeToString(c.managedTypeWatch.watchedType)]; ok { + for _, w := range existing { + watches[w] = struct{}{} + } } - out[typeToString(c.managedTypeWatch.watchedType)] = watches + for _, w := range c.watches { + watches[typeToString(w.watchedType)] = struct{}{} + } + + out[typeToString(c.managedTypeWatch.watchedType)] = maps.Keys(watches) } return out diff --git a/internal/mesh/internal/controllers/gatewayproxy/builder/builder.go b/internal/mesh/internal/controllers/gatewayproxy/builder/builder.go new file mode 100644 index 0000000000..98856b4191 --- /dev/null +++ b/internal/mesh/internal/controllers/gatewayproxy/builder/builder.go @@ -0,0 +1,65 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package builder + +import ( + "github.com/hashicorp/consul/internal/mesh/internal/types" + pbauth "github.com/hashicorp/consul/proto-public/pbauth/v2beta1" + meshv2beta1 "github.com/hashicorp/consul/proto-public/pbmesh/v2beta1" + "github.com/hashicorp/consul/proto-public/pbmesh/v2beta1/pbproxystate" + "github.com/hashicorp/consul/proto-public/pbresource" +) + +type proxyStateTemplateBuilder struct { + workload *types.DecodedWorkload +} + +func NewProxyStateTemplateBuilder(workload *types.DecodedWorkload) *proxyStateTemplateBuilder { + return &proxyStateTemplateBuilder{ + workload: workload, + } +} + +func (b *proxyStateTemplateBuilder) identity() *pbresource.Reference { + return &pbresource.Reference{ + Name: b.workload.Data.Identity, + Tenancy: b.workload.Id.Tenancy, + Type: pbauth.WorkloadIdentityType, + } +} + +func (b *proxyStateTemplateBuilder) listeners() []*pbproxystate.Listener { + // TODO NET-6429 + return nil +} + +func (b *proxyStateTemplateBuilder) clusters() map[string]*pbproxystate.Cluster { + // TODO NET-6430 + return nil +} + +func (b *proxyStateTemplateBuilder) endpoints() map[string]*pbproxystate.Endpoints { + // TODO NET-6431 + return nil +} + +func (b *proxyStateTemplateBuilder) routes() map[string]*pbproxystate.Route { + // TODO NET-6428 + return nil +} + +func (b *proxyStateTemplateBuilder) Build() *meshv2beta1.ProxyStateTemplate { + return &meshv2beta1.ProxyStateTemplate{ + ProxyState: &meshv2beta1.ProxyState{ + Identity: b.identity(), + Listeners: b.listeners(), + Clusters: b.clusters(), + Endpoints: b.endpoints(), + Routes: b.routes(), + }, + RequiredEndpoints: make(map[string]*pbproxystate.EndpointRef), + RequiredLeafCertificates: make(map[string]*pbproxystate.LeafCertificateRef), + RequiredTrustBundles: make(map[string]*pbproxystate.TrustBundleRef), + } +} diff --git a/internal/mesh/internal/controllers/gatewayproxy/controller.go b/internal/mesh/internal/controllers/gatewayproxy/controller.go new file mode 100644 index 0000000000..edc1e32be4 --- /dev/null +++ b/internal/mesh/internal/controllers/gatewayproxy/controller.go @@ -0,0 +1,137 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package gatewayproxy + +import ( + "context" + + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" + + "github.com/hashicorp/consul/internal/controller" + "github.com/hashicorp/consul/internal/controller/dependency" + "github.com/hashicorp/consul/internal/mesh/internal/controllers/gatewayproxy/builder" + "github.com/hashicorp/consul/internal/mesh/internal/controllers/gatewayproxy/fetcher" + "github.com/hashicorp/consul/internal/mesh/internal/controllers/sidecarproxy/cache" + "github.com/hashicorp/consul/internal/resource" + pbcatalog "github.com/hashicorp/consul/proto-public/pbcatalog/v2beta1" + pbmesh "github.com/hashicorp/consul/proto-public/pbmesh/v2beta1" + "github.com/hashicorp/consul/proto-public/pbresource" +) + +// ControllerName is the name for this controller. It's used for logging or status keys. +const ControllerName = "consul.io/gateway-proxy-controller" + +// Controller is responsible for triggering reconciler for watched resources +func Controller(cache *cache.Cache) *controller.Controller { + // TODO NET-7016 Use caching functionality in NewController being implemented at time of writing + // TODO NET-7017 Add the host of other types we should watch + return controller.NewController(ControllerName, pbmesh.ProxyStateTemplateType). + WithWatch(pbcatalog.WorkloadType, dependency.ReplaceType(pbmesh.ProxyStateTemplateType)). + WithWatch(pbmesh.ComputedProxyConfigurationType, dependency.ReplaceType(pbmesh.ProxyStateTemplateType)). + WithReconciler(&reconciler{ + cache: cache, + }) +} + +// reconciler is responsible for managing the ProxyStateTemplate for all +// gateway types: mesh, api (future) and terminating (future). +type reconciler struct { + cache *cache.Cache +} + +// Reconcile is responsible for creating and updating the pbmesh.ProxyStateTemplate +// for all gateway types. Since the ProxyStateTemplates managed here will always have +// an owner reference pointing to the corresponding pbmesh.MeshGateway, deletion is +// left to the garbage collector. +func (r *reconciler) Reconcile(ctx context.Context, rt controller.Runtime, req controller.Request) error { + rt.Logger = rt.Logger.With("resource-id", req.ID) + rt.Logger.Trace("reconciling proxy state template") + + // Instantiate a data fetcher to fetch all reconciliation data. + dataFetcher := fetcher.New(rt.Client, r.cache) + + workloadID := resource.ReplaceType(pbcatalog.WorkloadType, req.ID) + workload, err := dataFetcher.FetchWorkload(ctx, workloadID) + if err != nil { + rt.Logger.Error("error reading the associated workload", "error", err) + return err + } + + if workload == nil { + // If workload has been deleted, then return as ProxyStateTemplate should be cleaned up + // by the garbage collector because of the owner reference. + rt.Logger.Trace("workload doesn't exist; skipping reconciliation", "workload", workloadID) + return nil + } + + // If the workload is not for a xGateway, let the sidecarproxy reconciler handle it + if gatewayKind := workload.Metadata["gateway-kind"]; gatewayKind == "" { + rt.Logger.Trace("workload is not a gateway; skipping reconciliation", "workload", workloadID) + return nil + } + + // TODO NET-7014 Determine what gateway controls this workload + // For now, we cheat by knowing the MeshGateway's name, type + tenancy ahead of time + gatewayID := &pbresource.ID{ + Name: "mesh-gateway", + Type: pbmesh.MeshGatewayType, + Tenancy: resource.DefaultPartitionedTenancy(), + } + + // Check if the gateway exists. + gateway, err := dataFetcher.FetchMeshGateway(ctx, gatewayID) + if err != nil { + rt.Logger.Error("error reading the associated gateway", "error", err) + return err + } + if gateway == nil { + // If gateway has been deleted, then return as ProxyStateTemplate should be + // cleaned up by the garbage collector because of the owner reference. + rt.Logger.Trace("gateway doesn't exist; skipping reconciliation", "gateway", gatewayID) + return nil + } + + proxyStateTemplate, err := dataFetcher.FetchProxyStateTemplate(ctx, req.ID) + if err != nil { + rt.Logger.Error("error reading proxy state template", "error", err) + return nil + } + + if proxyStateTemplate == nil { + req.ID.Uid = "" + rt.Logger.Trace("proxy state template for this gateway doesn't yet exist; generating a new one") + } + + newPST := builder.NewProxyStateTemplateBuilder(workload).Build() + + proxyTemplateData, err := anypb.New(newPST) + if err != nil { + rt.Logger.Error("error creating proxy state template data", "error", err) + return err + } + rt.Logger.Trace("updating proxy state template") + + // If we're not creating a new PST and the generated one matches the existing one, nothing to do + if proxyStateTemplate != nil && proto.Equal(proxyStateTemplate.Data, newPST) { + rt.Logger.Trace("no changes to existing proxy state template") + return nil + } + + // Write the created/updated ProxyStateTemplate with MeshGateway owner + _, err = rt.Client.Write(ctx, &pbresource.WriteRequest{ + Resource: &pbresource.Resource{ + Id: req.ID, + Metadata: map[string]string{"gateway-kind": workload.Metadata["gateway-kind"]}, + Owner: workload.Resource.Id, + Data: proxyTemplateData, + }, + }) + if err != nil { + rt.Logger.Error("error writing proxy state template", "error", err) + return err + } + + return nil +} diff --git a/internal/mesh/internal/controllers/gatewayproxy/fetcher/data_fetcher.go b/internal/mesh/internal/controllers/gatewayproxy/fetcher/data_fetcher.go new file mode 100644 index 0000000000..9281109dd2 --- /dev/null +++ b/internal/mesh/internal/controllers/gatewayproxy/fetcher/data_fetcher.go @@ -0,0 +1,60 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package fetcher + +import ( + "context" + + "github.com/hashicorp/consul/internal/mesh/internal/controllers/sidecarproxy/cache" + "github.com/hashicorp/consul/internal/mesh/internal/types" + "github.com/hashicorp/consul/internal/resource" + pbcatalog "github.com/hashicorp/consul/proto-public/pbcatalog/v2beta1" + pbmesh "github.com/hashicorp/consul/proto-public/pbmesh/v2beta1" + "github.com/hashicorp/consul/proto-public/pbresource" +) + +type Fetcher struct { + client pbresource.ResourceServiceClient + cache *cache.Cache +} + +func New(client pbresource.ResourceServiceClient, cache *cache.Cache) *Fetcher { + return &Fetcher{ + client: client, + cache: cache, + } +} + +func (f *Fetcher) FetchMeshGateway(ctx context.Context, id *pbresource.ID) (*types.DecodedMeshGateway, error) { + dec, err := resource.GetDecodedResource[*pbmesh.MeshGateway](ctx, f.client, id) + if err != nil { + return nil, err + } else if dec == nil { + return nil, nil + } + + return dec, err +} + +func (f *Fetcher) FetchProxyStateTemplate(ctx context.Context, id *pbresource.ID) (*types.DecodedProxyStateTemplate, error) { + dec, err := resource.GetDecodedResource[*pbmesh.ProxyStateTemplate](ctx, f.client, id) + if err != nil { + return nil, err + } else if dec == nil { + return nil, nil + } + + return dec, err +} + +func (f *Fetcher) FetchWorkload(ctx context.Context, id *pbresource.ID) (*types.DecodedWorkload, error) { + dec, err := resource.GetDecodedResource[*pbcatalog.Workload](ctx, f.client, id) + if err != nil { + return nil, err + } else if dec == nil { + return nil, nil + } + + return dec, err +} diff --git a/internal/mesh/internal/controllers/register.go b/internal/mesh/internal/controllers/register.go index 2416c5615f..cd268c708a 100644 --- a/internal/mesh/internal/controllers/register.go +++ b/internal/mesh/internal/controllers/register.go @@ -5,6 +5,8 @@ package controllers import ( "context" + + "github.com/hashicorp/consul/internal/mesh/internal/controllers/gatewayproxy" "github.com/hashicorp/consul/internal/mesh/internal/controllers/meshconfiguration" "github.com/hashicorp/consul/agent/leafcert" @@ -46,6 +48,8 @@ func Register(mgr *controller.Manager, deps Dependencies) { sidecarproxy.Controller(cache.New(), deps.TrustDomainFetcher, deps.LocalDatacenter, deps.DefaultAllow), ) + mgr.Register(gatewayproxy.Controller(cache.New())) + mgr.Register(routes.Controller()) mgr.Register(proxyconfiguration.Controller(workloadselectionmapper.New[*pbmesh.ProxyConfiguration](pbmesh.ComputedProxyConfigurationType))) diff --git a/internal/mesh/internal/types/decoded.go b/internal/mesh/internal/types/decoded.go index be4836c066..2d5dd304c2 100644 --- a/internal/mesh/internal/types/decoded.go +++ b/internal/mesh/internal/types/decoded.go @@ -27,4 +27,5 @@ type ( DecodedDestinations = resource.DecodedResource[*pbmesh.Destinations] DecodedComputedDestinations = resource.DecodedResource[*pbmesh.ComputedExplicitDestinations] DecodedProxyStateTemplate = resource.DecodedResource[*pbmesh.ProxyStateTemplate] + DecodedMeshGateway = resource.DecodedResource[*pbmesh.MeshGateway] )