cache: log a warning when Cache.Notify handles an error

Without these warnings, errors are silently ignored, which can make
debugging problems more challenging.

agent/cache/cache.go

@@ -25,6 +25,7 @@ import (
 
 	"github.com/armon/go-metrics"
 	"github.com/armon/go-metrics/prometheus"
+	"github.com/hashicorp/go-hclog"
 	"golang.org/x/time/rate"
 
 	"github.com/hashicorp/consul/acl"
@@ -170,6 +171,8 @@ type ResultMeta struct {
 
 // Options are options for the Cache.
 type Options struct {
+	Logger hclog.Logger
+
 	// EntryFetchMaxBurst max burst size of RateLimit for a single cache entry
 	EntryFetchMaxBurst int
 	// EntryFetchRate represents the max calls/sec for a single cache entry
@@ -189,6 +192,9 @@ func applyDefaultValuesOnOptions(options Options) Options {
 	if options.EntryFetchMaxBurst == 0 {
 		options.EntryFetchMaxBurst = DefaultEntryFetchMaxBurst
 	}
+	if options.Logger == nil {
+		options.Logger = hclog.New(nil)
+	}
 	return options
 }
 

agent/cache/watch.go

@@ -121,6 +121,12 @@ func (c *Cache) notifyBlockingQuery(ctx context.Context, r getOptions, correlati
 		} else {
 			failures++
 			wait = backOffWait(failures)
+
+			c.options.Logger.
+				With("error", err).
+				With("cache-type", r.TypeEntry.Name).
+				With("index", index).
+				Warn("handling error in Cache.Notify")
 		}
 
 		if wait > 0 {
@@ -177,6 +183,11 @@ func (c *Cache) notifyPollingQuery(ctx context.Context, r getOptions, correlatio
 			failures = 0
 		} else {
 			failures++
+			c.options.Logger.
+				With("error", err).
+				With("cache-type", r.TypeEntry.Name).
+				With("index", index).
+				Warn("handling error in Cache.Notify")
 		}
 
 		var wait time.Duration

agent/config/testdata/TestRuntimeConfig_Sanitize.golden

@@ -74,7 +74,8 @@
     "CAPath": "",
     "Cache": {
         "EntryFetchMaxBurst": 42,
-        "EntryFetchRate": 0.334
+        "EntryFetchRate": 0.334,
+        "Logger": null
     },
     "CertFile": "",
     "CheckDeregisterIntervalMin": "0s",

agent/setup.go

@@ -97,6 +97,7 @@ func NewBaseDeps(configLoader ConfigLoader, logOut io.Writer) (BaseDeps, error)
 	d.RuntimeConfig = cfg
 	d.Tokens = new(token.Store)
 
+	cfg.Cache.Logger = d.Logger.Named("cache")
 	// cache-types are not registered yet, but they won't be used until the components are started.
 	d.Cache = cache.New(cfg.Cache)
 	d.ConnPool = newConnPool(cfg, d.Logger, d.TLSConfigurator)