consul/agent/hcp/manager_lifecycle.go
Nick Cellino 5fb6ab6a3a
Move HCP Manager lifecycle management out of Link controller (#20401)
* Add function to get update channel for watching HCP Link

* Add MonitorHCPLink function

This function can be called in a goroutine to manage the lifecycle
of the HCP manager.

* Update HCP Manager config in link monitor before starting

This updates HCPMonitorLink so it updates the HCP manager
with an HCP client and management token when a Link is upserted.

* Let MonitorHCPManager handle lifecycle instead of link controller

* Remove cleanup from Link controller and move it to MonitorHCPLink

Previously, the Link Controller was responsible for cleaning up the
HCP-related files on the file system. This change makes it so
MonitorHCPLink handles this cleanup. As a result, we are able to remove
the PlacementEachServer placement strategy for the Link controller
because it no longer needs to do this per-node cleanup.

* Remove HCP Manager dependency from Link Controller

The Link controller does not need to have HCP Manager
as a dependency anymore, so this removes that dependency
in order to simplify the design.

* Add Linked prefix to Linked status variables

This is in preparation for adding a new status type to the
Link resource.

* Add new "validated" status type to link resource

The link resource controller will now set a "validated" status
in addition to the "linked" status. This is needed so that other
components (eg the HCP manager) know when the Link is ready to link
with HCP.

* Fix tests

* Handle new 'EndOfSnapshot' WatchList event

* Fix watch test

* Remove unnecessary config from TestAgent_scadaProvider

Since the Scada provider is now started on agent startup
regardless of whether a cloud config is provided, this removes
the cloud config override from the relevant test.

This change is not exactly related to the changes from this PR,
but rather is something small and sort of related that was noticed
while working on this PR.

* Simplify link watch test and remove sleep from link watch

This updates the link watch test so that it uses more mocks
and does not require setting up the infrastructure for the HCP Link
controller.

This also removes the time.Sleep delay in the link watcher loop in favor
of an error counter. When we receive 10 consecutive errors, we shut down
the link watcher loop.

* Add better logging for link validation. Remove EndOfSnapshot test.

* Refactor link monitor test into a table test

* Add some clarifying comments to link monitor

* Simplify link watch test

* Test a bunch more errors cases in link monitor test

* Use exponential backoff instead of errorCounter in LinkWatch

* Move link watch and link monitor into a single goroutine called from server.go

* Refactor HCP link watcher to use single go-routine.

Previously, if the WatchClient errored, we would've never recovered
because we never retry to create the stream. With this change,
we have a single goroutine that runs for the life of the server agent
and if the WatchClient stream ever errors, we retry the creation
of the stream with an exponential backoff.
2024-02-12 10:48:23 -05:00

108 lines
3.5 KiB
Go

// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: BUSL-1.1
package hcp
import (
"context"
"os"
"path/filepath"
"github.com/hashicorp/go-hclog"
"github.com/hashicorp/consul/agent/hcp/bootstrap/constants"
hcpclient "github.com/hashicorp/consul/agent/hcp/client"
"github.com/hashicorp/consul/agent/hcp/config"
hcpctl "github.com/hashicorp/consul/internal/hcp"
pbhcp "github.com/hashicorp/consul/proto-public/pbhcp/v2"
"github.com/hashicorp/consul/proto-public/pbresource"
)
// HCPManagerLifecycleFn returns a LinkEventHandler function which will appropriately
// Start and Stop the HCP Manager based on the Link event received. If a link is upserted,
// the HCP Manager is started, and if a link is deleted, the HCP manager is stopped.
func HCPManagerLifecycleFn(
m Manager,
hcpClientFn func(cfg config.CloudConfig) (hcpclient.Client, error),
loadMgmtTokenFn func(
ctx context.Context, logger hclog.Logger, hcpClient hcpclient.Client, dataDir string,
) (string, error),
cloudConfig config.CloudConfig,
dataDir string,
) LinkEventHandler {
return func(ctx context.Context, logger hclog.Logger, watchEvent *pbresource.WatchEvent) {
// This indicates that a Link was deleted
if watchEvent.GetDelete() != nil {
logger.Debug("HCP Link deleted, stopping HCP manager")
if dataDir != "" {
hcpConfigDir := filepath.Join(dataDir, constants.SubDir)
logger.Debug("deleting hcp-config dir", "dir", hcpConfigDir)
err := os.RemoveAll(hcpConfigDir)
if err != nil {
logger.Error("failed to delete hcp-config dir", "dir", hcpConfigDir, "err", err)
}
}
err := m.Stop()
if err != nil {
logger.Error("error stopping HCP manager", "error", err)
}
return
}
// This indicates that a Link was either created or updated
if watchEvent.GetUpsert() != nil {
logger.Debug("HCP Link upserted, starting manager if not already started")
res := watchEvent.GetUpsert().GetResource()
var link pbhcp.Link
if err := res.GetData().UnmarshalTo(&link); err != nil {
logger.Error("error unmarshalling link data", "error", err)
return
}
if validated, reason := hcpctl.IsValidated(res); !validated {
logger.Debug("HCP Link not validated, not starting manager", "reason", reason)
return
}
// Update the HCP manager configuration with the link values
// Merge the link data with the existing cloud config so that we only overwrite the
// fields that are provided by the link. This ensures that:
// 1. The HCP configuration (i.e., how to connect to HCP) is preserved
// 2. The Consul agent's node ID and node name are preserved
newCfg := config.CloudConfig{
ResourceID: link.ResourceId,
ClientID: link.ClientId,
ClientSecret: link.ClientSecret,
}
mergedCfg := config.Merge(cloudConfig, newCfg)
hcpClient, err := hcpClientFn(mergedCfg)
if err != nil {
logger.Error("error creating HCP client", "error", err)
return
}
// Load the management token if access is set to read-write. Read-only clusters
// will not have a management token provided by HCP.
var token string
if link.GetAccessLevel() == pbhcp.AccessLevel_ACCESS_LEVEL_GLOBAL_READ_WRITE {
token, err = loadMgmtTokenFn(ctx, logger, hcpClient, dataDir)
if err != nil {
logger.Error("error loading management token", "error", err)
return
}
}
mergedCfg.ManagementToken = token
m.UpdateConfig(hcpClient, mergedCfg)
err = m.Start(ctx)
if err != nil {
logger.Error("error starting HCP manager", "error", err)
}
}
}
}