ae: use stale requests when performing full sync (#5873)

Read requests performed during anti antropy full sync currently target
the leader only. This generates a non-negligible load on the leader when
the DC is large enough and can be offloaded to the followers following
the "eventually consistent" policy for the agent state.
We switch the AE read calls to use stale requests with a small (2s)
MaxStaleDuration value and make sure we do not read too fast after a
write.

agent/local/state.go

@@ -22,6 +22,8 @@ import (
 	uuid "github.com/hashicorp/go-uuid"
 )
 
+const fullSyncReadMaxStale = 2 * time.Second
+
 // Config is the configuration for the State.
 type Config struct {
 	AdvertiseAddr       string
@@ -1046,9 +1048,13 @@ func (l *State) Stats() map[string]string {
 func (l *State) updateSyncState() error {
 	// Get all checks and services from the master
 	req := structs.NodeSpecificRequest{
-		Datacenter:   l.config.Datacenter,
-		Node:         l.config.NodeName,
-		QueryOptions: structs.QueryOptions{Token: l.tokens.AgentToken()},
+		Datacenter: l.config.Datacenter,
+		Node:       l.config.NodeName,
+		QueryOptions: structs.QueryOptions{
+			Token:            l.tokens.AgentToken(),
+			AllowStale:       true,
+			MaxStaleDuration: fullSyncReadMaxStale,
+		},
 	}
 
 	var out1 structs.IndexedNodeServices