From 15e80e4e76a7f207c03da11f32bf8d948ac699cb Mon Sep 17 00:00:00 2001 From: Matt Keeler Date: Fri, 26 Apr 2019 14:25:03 -0400 Subject: [PATCH] Implement bootstrapping proxy defaults from the config file (#5714) --- agent/agent.go | 8 ++++ agent/config/builder.go | 17 ++++++++ agent/config/config.go | 9 +++++ agent/config/runtime.go | 4 ++ agent/config/runtime_test.go | 32 ++++++++++++++- agent/consul/config.go | 4 ++ agent/consul/leader.go | 47 +++++++++++++++++++++++ agent/consul/leader_test.go | 35 +++++++++++++++++ agent/consul/server.go | 5 +++ agent/consul/server_test.go | 38 ++++++++++++++++++ website/source/docs/agent/options.html.md | 15 ++++++++ 11 files changed, 212 insertions(+), 2 deletions(-) diff --git a/agent/agent.go b/agent/agent.go index 4b4377a810..36ca1a2958 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -1136,6 +1136,8 @@ func (a *Agent) consulConfig() (*consul.Config, error) { return nil, fmt.Errorf("Failed to configure keyring: %v", err) } + base.ConfigEntryBootstrap = a.config.ConfigEntryBootstrap + return base, nil } @@ -3616,6 +3618,12 @@ func (a *Agent) ReloadConfig(newCfg *config.RuntimeConfig) error { } } + // this only gets used by the consulConfig function and since + // that is only ever done during init and reload here then + // an in place modification is safe as reloads cannot be + // concurrent due to both gaing a full lock on the stateLock + a.config.ConfigEntryBootstrap = newCfg.ConfigEntryBootstrap + // create the config for the rpc server/client consulCfg, err := a.consulConfig() if err != nil { diff --git a/agent/config/builder.go b/agent/config/builder.go index 18706ffe9d..5ef7e35044 100644 --- a/agent/config/builder.go +++ b/agent/config/builder.go @@ -633,6 +633,22 @@ func (b *Builder) Build() (rt RuntimeConfig, err error) { verifyOutgoing = true } + var configEntries []structs.ConfigEntry + + if len(c.ConfigEntries.Bootstrap.ProxyDefaults) > 0 { + for name, config := range c.ConfigEntries.Bootstrap.ProxyDefaults { + if name != structs.ProxyConfigGlobal { + return RuntimeConfig{}, fmt.Errorf("invalid config.proxy_defaults name (%q), only %q is supported", name, structs.ProxyConfigGlobal) + } + + configEntries = append(configEntries, &structs.ProxyConfigEntry{ + Kind: structs.ProxyDefaults, + Name: structs.ProxyConfigGlobal, + Config: config, + }) + } + } + // ---------------------------------------------------------------- // build runtime config // @@ -767,6 +783,7 @@ func (b *Builder) Build() (rt RuntimeConfig, err error) { CheckUpdateInterval: b.durationVal("check_update_interval", c.CheckUpdateInterval), Checks: checks, ClientAddrs: clientAddrs, + ConfigEntryBootstrap: configEntries, ConnectEnabled: connectEnabled, ConnectCAProvider: connectCAProvider, ConnectCAConfig: connectCAConfig, diff --git a/agent/config/config.go b/agent/config/config.go index 855ce91948..958f53f26c 100644 --- a/agent/config/config.go +++ b/agent/config/config.go @@ -184,6 +184,7 @@ type Config struct { CheckUpdateInterval *string `json:"check_update_interval,omitempty" hcl:"check_update_interval" mapstructure:"check_update_interval"` Checks []CheckDefinition `json:"checks,omitempty" hcl:"checks" mapstructure:"checks"` ClientAddr *string `json:"client_addr,omitempty" hcl:"client_addr" mapstructure:"client_addr"` + ConfigEntries ConfigEntries `json:"config_entries,omitempty" hcl:"config_entries" mapstructure:"config_entries"` Connect Connect `json:"connect,omitempty" hcl:"connect" mapstructure:"connect"` DNS DNS `json:"dns_config,omitempty" hcl:"dns_config" mapstructure:"dns_config"` DNSDomain *string `json:"domain,omitempty" hcl:"domain" mapstructure:"domain"` @@ -650,3 +651,11 @@ type Tokens struct { Default *string `json:"default,omitempty" hcl:"default" mapstructure:"default"` Agent *string `json:"agent,omitempty" hcl:"agent" mapstructure:"agent"` } + +type ConfigEntries struct { + Bootstrap ConfigEntriesBootstrap `json:"bootstrap,omitempty" hcl:"bootstrap" mapstructure:"bootstrap"` +} + +type ConfigEntriesBootstrap struct { + ProxyDefaults map[string]map[string]interface{} `json:"proxy_defaults,omitempty" hcl:"proxy_defaults" mapstructure:"proxy_defaults"` +} diff --git a/agent/config/runtime.go b/agent/config/runtime.go index ba3689a2cd..96bbe1e3e4 100644 --- a/agent/config/runtime.go +++ b/agent/config/runtime.go @@ -496,6 +496,10 @@ type RuntimeConfig struct { // flag: -client string ClientAddrs []*net.IPAddr + // ConfigEntryBootstrap contains a list of ConfigEntries to ensure are created + // If entries of the same Kind/Name exist already these will not update them. + ConfigEntryBootstrap []structs.ConfigEntry + // ConnectEnabled opts the agent into connect. It should be set on all clients // and servers in a cluster for correct connect operation. ConnectEnabled bool diff --git a/agent/config/runtime_test.go b/agent/config/runtime_test.go index 59ab47de70..0e22b95d73 100644 --- a/agent/config/runtime_test.go +++ b/agent/config/runtime_test.go @@ -3007,6 +3007,16 @@ func TestFullConfig(t *testing.T) { ], "check_update_interval": "16507s", "client_addr": "93.83.18.19", + "config_entries": { + "bootstrap": { + "proxy_defaults": { + "global": { + "foo": "bar", + "bar": 1.0 + } + } + } + }, "connect": { "ca_provider": "consul", "ca_config": { @@ -3560,6 +3570,12 @@ func TestFullConfig(t *testing.T) { ] check_update_interval = "16507s" client_addr = "93.83.18.19" + config_entries { + bootstrap proxy_defaults global { + foo = "bar" + bar = 1.0 + } + } connect { ca_provider = "consul" ca_config { @@ -4217,8 +4233,19 @@ func TestFullConfig(t *testing.T) { DeregisterCriticalServiceAfter: 13209 * time.Second, }, }, - CheckUpdateInterval: 16507 * time.Second, - ClientAddrs: []*net.IPAddr{ipAddr("93.83.18.19")}, + CheckUpdateInterval: 16507 * time.Second, + ClientAddrs: []*net.IPAddr{ipAddr("93.83.18.19")}, + ConfigEntryBootstrap: []structs.ConfigEntry{ + &structs.ProxyConfigEntry{ + Kind: structs.ProxyDefaults, + Name: structs.ProxyConfigGlobal, + Config: map[string]interface{}{ + "foo": "bar", + // has to be a float due to being a map[string]interface + "bar": float64(1), + }, + }, + }, ConnectEnabled: true, ConnectProxyBindMinPort: 2000, ConnectProxyBindMaxPort: 3000, @@ -4996,6 +5023,7 @@ func TestSanitize(t *testing.T) { "Token": "hidden" }], "ClientAddrs": [], + "ConfigEntryBootstrap": [], "ConnectCAConfig": {}, "ConnectCAProvider": "", "ConnectEnabled": false, diff --git a/agent/consul/config.go b/agent/consul/config.go index ff2b008377..baf9e63a29 100644 --- a/agent/consul/config.go +++ b/agent/consul/config.go @@ -391,6 +391,10 @@ type Config struct { // CAConfig is used to apply the initial Connect CA configuration when // bootstrapping. CAConfig *structs.CAConfiguration + + // ConfigEntryBootstrap contains a list of ConfigEntries to ensure are created + // If entries of the same Kind/Name exist already these will not update them. + ConfigEntryBootstrap []structs.ConfigEntry } func (c *Config) ToTLSUtilConfig() tlsutil.Config { diff --git a/agent/consul/leader.go b/agent/consul/leader.go index 5abf12d607..3264892095 100644 --- a/agent/consul/leader.go +++ b/agent/consul/leader.go @@ -40,6 +40,10 @@ var ( // minAutopilotVersion is the minimum Consul version in which Autopilot features // are supported. minAutopilotVersion = version.Must(version.NewVersion("0.8.0")) + + // minCentralizedConfigVersion is the minimum Consul version in which centralized + // config is supported + minCentralizedConfigVersion = version.Must(version.NewVersion("1.5.0")) ) // monitorLeadership is used to monitor if we acquire or lose our role @@ -261,6 +265,11 @@ func (s *Server) establishLeadership() error { return err } + // attempt to bootstrap config entries + if err := s.bootstrapConfigEntries(s.config.ConfigEntryBootstrap); err != nil { + return err + } + s.getOrCreateAutopilotConfig() s.autopilot.Start() @@ -893,6 +902,44 @@ func (s *Server) getOrCreateAutopilotConfig() *autopilot.Config { return config } +func (s *Server) bootstrapConfigEntries(entries []structs.ConfigEntry) error { + if s.config.PrimaryDatacenter != "" && s.config.PrimaryDatacenter != s.config.Datacenter { + // only bootstrap in the primary datacenter + return nil + } + + if len(entries) < 1 { + // nothing to initialize + return nil + } + + if !ServersMeetMinimumVersion(s.LANMembers(), minCentralizedConfigVersion) { + s.logger.Printf("[WARN] centralized config: can't initialize until all servers >= %s", minCentralizedConfigVersion.String()) + return nil + } + + state := s.fsm.State() + for _, entry := range entries { + _, existing, err := state.ConfigEntry(nil, entry.GetKind(), entry.GetName()) + if err != nil { + return fmt.Errorf("Failed to determine whether the configuration for %q / %q already exists: %v", entry.GetKind(), entry.GetName(), err) + } + + if existing == nil { + req := structs.ConfigEntryRequest{ + Op: structs.ConfigEntryUpsert, + Datacenter: s.config.Datacenter, + Entry: entry, + } + + if _, err = s.raftApply(structs.ConfigEntryRequestType, &req); err != nil { + return fmt.Errorf("Failed to apply configuration entry %q / %q: %v", entry.GetKind(), entry.GetName(), err) + } + } + } + return nil +} + // initializeCAConfig is used to initialize the CA config if necessary // when setting up the CA during establishLeadership func (s *Server) initializeCAConfig() (*structs.CAConfiguration, error) { diff --git a/agent/consul/leader_test.go b/agent/consul/leader_test.go index b6dc58bce0..c000a37ed8 100644 --- a/agent/consul/leader_test.go +++ b/agent/consul/leader_test.go @@ -1206,3 +1206,38 @@ func TestLeader_ACLUpgrade(t *testing.T) { require.Equal(t, client.ACL.Rules, token.Rules) }) } + +func TestLeader_ConfigEntryBootstrap(t *testing.T) { + t.Parallel() + global_entry_init := &structs.ProxyConfigEntry{ + Kind: structs.ProxyDefaults, + Name: structs.ProxyConfigGlobal, + Config: map[string]interface{}{ + // these are made a []uint8 and a int64 to allow the Equals test to pass + // otherwise it will fail complaining about data types + "foo": []uint8("bar"), + "bar": int64(1), + }, + } + + dir1, s1 := testServerWithConfig(t, func(c *Config) { + c.Build = "1.5.0" + c.ConfigEntryBootstrap = []structs.ConfigEntry{ + global_entry_init, + } + }) + defer os.RemoveAll(dir1) + defer s1.Shutdown() + testrpc.WaitForTestAgent(t, s1.RPC, "dc1") + + retry.Run(t, func(t *retry.R) { + _, entry, err := s1.fsm.State().ConfigEntry(nil, structs.ProxyDefaults, structs.ProxyConfigGlobal) + require.NoError(t, err) + require.NotNil(t, entry) + global, ok := entry.(*structs.ProxyConfigEntry) + require.True(t, ok) + require.Equal(t, global_entry_init.Kind, global.Kind) + require.Equal(t, global_entry_init.Name, global.Name) + require.Equal(t, global_entry_init.Config, global.Config) + }) +} diff --git a/agent/consul/server.go b/agent/consul/server.go index b1596a82ec..f19a907455 100644 --- a/agent/consul/server.go +++ b/agent/consul/server.go @@ -1132,6 +1132,11 @@ func (s *Server) GetLANCoordinate() (lib.CoordinateSet, error) { // ReloadConfig is used to have the Server do an online reload of // relevant configuration information func (s *Server) ReloadConfig(config *Config) error { + if s.IsLeader() { + // only bootstrap the config entries if we are the leader + // this will error if we lose leadership while bootstrapping here. + return s.bootstrapConfigEntries(config.ConfigEntryBootstrap) + } return nil } diff --git a/agent/consul/server_test.go b/agent/consul/server_test.go index 86505b906a..4f809b664c 100644 --- a/agent/consul/server_test.go +++ b/agent/consul/server_test.go @@ -970,3 +970,41 @@ func TestServer_RevokeLeadershipIdempotent(t *testing.T) { t.Fatal(err) } } + +func TestServer_Reload(t *testing.T) { + t.Parallel() + + global_entry_init := &structs.ProxyConfigEntry{ + Kind: structs.ProxyDefaults, + Name: structs.ProxyConfigGlobal, + Config: map[string]interface{}{ + // these are made a []uint8 and a int64 to allow the Equals test to pass + // otherwise it will fail complaining about data types + "foo": []uint8("bar"), + "bar": int64(1), + }, + } + + dir1, s := testServerWithConfig(t, func(c *Config) { + c.Build = "1.5.0" + }) + defer os.RemoveAll(dir1) + defer s.Shutdown() + + testrpc.WaitForTestAgent(t, s.RPC, "dc1") + + s.config.ConfigEntryBootstrap = []structs.ConfigEntry{ + global_entry_init, + } + + s.ReloadConfig(s.config) + + _, entry, err := s.fsm.State().ConfigEntry(nil, structs.ProxyDefaults, structs.ProxyConfigGlobal) + require.NoError(t, err) + require.NotNil(t, entry) + global, ok := entry.(*structs.ProxyConfigEntry) + require.True(t, ok) + require.Equal(t, global_entry_init.Kind, global.Kind) + require.Equal(t, global_entry_init.Name, global.Name) + require.Equal(t, global_entry_init.Config, global.Config) +} diff --git a/website/source/docs/agent/options.html.md b/website/source/docs/agent/options.html.md index e15f59b362..2aaf9cb834 100644 --- a/website/source/docs/agent/options.html.md +++ b/website/source/docs/agent/options.html.md @@ -807,6 +807,21 @@ default will automatically work with some tooling. * `client_addr` Equivalent to the [`-client` command-line flag](#_client). +* `config_entries` + This object allows setting options for centralized config entries. + + The following sub-keys are available: + + * `bootstrap` + This object allows configuring centralized config entries to be bootstrapped + by the leader. These entries will be reloaded during an agent config reload. + + The following sub-keys are available: + + * `proxy_defaults` + This object should contain a mapping of config entry names to an opaque proxy configuration mapping. + Currently the only supported name is `global` + * `connect` This object allows setting options for the Connect feature.