From 9f7b22a5eb130dbe3e9a356883589c8bcb7acf37 Mon Sep 17 00:00:00 2001 From: Matt Keeler Date: Tue, 16 Jun 2020 15:03:22 -0400 Subject: [PATCH] Agent Auto Configuration: Configuration Syntax Updates (#8003) --- agent/config/builder.go | 145 +++++++++++++ agent/config/config.go | 32 +++ agent/config/merge.go | 8 + agent/config/merge_test.go | 7 +- agent/config/runtime.go | 26 +++ agent/config/runtime_test.go | 388 ++++++++++++++++++++++++++++++++++- agent/consul/acl_endpoint.go | 3 +- agent/consul/util.go | 45 ---- agent/consul/util_test.go | 155 -------------- agent/local/state_test.go | 12 +- lib/template/hil.go | 51 +++++ lib/template/hil_test.go | 162 +++++++++++++++ 12 files changed, 819 insertions(+), 215 deletions(-) create mode 100644 lib/template/hil.go create mode 100644 lib/template/hil_test.go diff --git a/agent/config/builder.go b/agent/config/builder.go index 026fa35673..f6638cbb5c 100644 --- a/agent/config/builder.go +++ b/agent/config/builder.go @@ -17,11 +17,15 @@ import ( "github.com/hashicorp/consul/agent/checks" "github.com/hashicorp/consul/agent/connect/ca" "github.com/hashicorp/consul/agent/consul" + "github.com/hashicorp/consul/agent/consul/authmethod/ssoauth" "github.com/hashicorp/consul/agent/structs" "github.com/hashicorp/consul/ipaddr" "github.com/hashicorp/consul/lib" + libtempl "github.com/hashicorp/consul/lib/template" "github.com/hashicorp/consul/tlsutil" "github.com/hashicorp/consul/types" + "github.com/hashicorp/go-bexpr" + "github.com/hashicorp/go-hclog" "github.com/hashicorp/go-multierror" "github.com/hashicorp/go-sockaddr/template" "github.com/hashicorp/memberlist" @@ -905,6 +909,7 @@ func (b *Builder) Build() (rt RuntimeConfig, err error) { AutoEncryptDNSSAN: autoEncryptDNSSAN, AutoEncryptIPSAN: autoEncryptIPSAN, AutoEncryptAllowTLS: autoEncryptAllowTLS, + AutoConfig: b.autoConfigVal(c.AutoConfig), ConnectEnabled: connectEnabled, ConnectCAProvider: connectCAProvider, ConnectCAConfig: connectCAConfig, @@ -1285,6 +1290,10 @@ func (b *Builder) Validate(rt RuntimeConfig) error { return err } + if err := b.validateAutoConfig(rt); err != nil { + return err + } + return nil } @@ -1918,6 +1927,142 @@ func (b *Builder) isUnixAddr(a net.Addr) bool { return a != nil && ok } +func (b *Builder) autoConfigVal(raw AutoConfigRaw) AutoConfig { + var val AutoConfig + + val.Enabled = b.boolValWithDefault(raw.Enabled, false) + val.IntroToken = b.stringVal(raw.IntroToken) + val.IntroTokenFile = b.stringVal(raw.IntroTokenFile) + // These can be go-discover values and so don't have to resolve fully yet + val.ServerAddresses = b.expandAllOptionalAddrs("auto_config.server_addresses", raw.ServerAddresses) + val.DNSSANs = raw.DNSSANs + + for _, i := range raw.IPSANs { + ip := net.ParseIP(i) + if ip == nil { + b.warn(fmt.Sprintf("Cannot parse ip %q from auto_config.ip_sans", i)) + continue + } + val.IPSANs = append(val.IPSANs, ip) + } + + val.Authorizer = b.autoConfigAuthorizerVal(raw.Authorizer) + + return val +} + +func (b *Builder) autoConfigAuthorizerVal(raw AutoConfigAuthorizerRaw) AutoConfigAuthorizer { + var val AutoConfigAuthorizer + + val.Enabled = b.boolValWithDefault(raw.Enabled, false) + val.ClaimAssertions = raw.ClaimAssertions + val.AllowReuse = b.boolValWithDefault(raw.AllowReuse, false) + val.AuthMethod = structs.ACLAuthMethod{ + Name: "Auto Config Authorizer", + Type: "jwt", + // TODO (autoconf) - Configurable token TTL + MaxTokenTTL: 72 * time.Hour, + EnterpriseMeta: *structs.DefaultEnterpriseMeta(), + Config: map[string]interface{}{ + "JWTSupportedAlgs": raw.JWTSupportedAlgs, + "BoundAudiences": raw.BoundAudiences, + "ClaimMappings": raw.ClaimMappings, + "ListClaimMappings": raw.ListClaimMappings, + "OIDCDiscoveryURL": b.stringVal(raw.OIDCDiscoveryURL), + "OIDCDiscoveryCACert": b.stringVal(raw.OIDCDiscoveryCACert), + "JWKSURL": b.stringVal(raw.JWKSURL), + "JWKSCACert": b.stringVal(raw.JWKSCACert), + "JWTValidationPubKeys": raw.JWTValidationPubKeys, + "BoundIssuer": b.stringVal(raw.BoundIssuer), + "ExpirationLeeway": b.durationVal("auto_config.authorizer.expiration_leeway", raw.ExpirationLeeway), + "NotBeforeLeeway": b.durationVal("auto_config.authorizer.not_before_leeway", raw.NotBeforeLeeway), + "ClockSkewLeeway": b.durationVal("auto_config.authorizer.clock_skew_leeway", raw.ClockSkewLeeway), + }, + // should be unnecessary as we aren't using the typical login process to create tokens but this is our + // desired mode regardless so if it ever did matter its probably better to be explicit. + TokenLocality: "local", + } + + return val +} + +func (b *Builder) validateAutoConfig(rt RuntimeConfig) error { + autoconf := rt.AutoConfig + + if err := b.validateAutoConfigAuthorizer(rt); err != nil { + return err + } + + if !autoconf.Enabled { + return nil + } + + // Auto Config doesn't currently support configuring servers + if rt.ServerMode { + return fmt.Errorf("auto_config.enabled cannot be set to true for server agents.") + } + + // When both are set we will prefer the given value over the file. + if autoconf.IntroToken != "" && autoconf.IntroTokenFile != "" { + b.warn("auto_config.intro_token and auto_config.intro_token_file are both set. Using the value of auto_config.intro_token") + } else if autoconf.IntroToken == "" && autoconf.IntroTokenFile == "" { + return fmt.Errorf("one of auto_config.intro_token or auto_config.intro_token_file must be set to enable auto_config") + } + + if len(autoconf.ServerAddresses) == 0 { + // TODO (autoconf) can we/should we infer this from the join/retry join addresses. I think no, as we will potentially + // be overriding those retry join addresses with the autoconf process anyways. + return fmt.Errorf("auto_config.enabled is set without providing a list of addresses") + } + + // TODO (autoconf) should we validate the DNS and IP SANs? The IP SANs have already been parsed into IPs + return nil +} + +func (b *Builder) validateAutoConfigAuthorizer(rt RuntimeConfig) error { + authz := rt.AutoConfig.Authorizer + + if !authz.Enabled { + return nil + } + // Auto Config Authorization is only supported on servers + if !rt.ServerMode { + return fmt.Errorf("auto_config.authorizer.enabled cannot be set to true for client agents") + } + + // build out the validator to ensure that the given configuration was valid + null := hclog.NewNullLogger() + validator, err := ssoauth.NewValidator(null, &authz.AuthMethod) + + if err != nil { + return fmt.Errorf("auto_config.authorizer has invalid configuration: %v", err) + } + + // create a blank identity for use to validate the claim assertions. + blankID := validator.NewIdentity() + varMap := map[string]string{ + "node": "fake", + "segment": "fake", + } + + // validate all the claim assertions + for _, raw := range authz.ClaimAssertions { + // validate any HIL + filled, err := libtempl.InterpolateHIL(raw, varMap, true) + if err != nil { + return fmt.Errorf("auto_config.claim_assertion %q is invalid: %v", raw, err) + } + + // validate the bexpr syntax - note that for now all the keys mapped by the claim mappings + // are not validateable due to them being put inside a map. Some bexpr updates to setup keys + // from current map keys would probably be nice here. + if _, err := bexpr.CreateEvaluatorForType(filled, nil, blankID.SelectableFields); err != nil { + return fmt.Errorf("auto_config.claim_assertion %q is invalid: %v", raw, err) + } + } + return nil +} + // decodeBytes returns the encryption key decoded. func decodeBytes(key string) ([]byte, error) { return base64.StdEncoding.DecodeString(key) diff --git a/agent/config/config.go b/agent/config/config.go index c11e7fd2e4..43e001ffe9 100644 --- a/agent/config/config.go +++ b/agent/config/config.go @@ -108,6 +108,7 @@ type Config struct { AdvertiseAddrWAN *string `json:"advertise_addr_wan,omitempty" hcl:"advertise_addr_wan" mapstructure:"advertise_addr_wan"` AdvertiseAddrWANIPv4 *string `json:"advertise_addr_wan_ipv4,omitempty" hcl:"advertise_addr_wan_ipv4" mapstructure:"advertise_addr_wan_ipv4"` AdvertiseAddrWANIPv6 *string `json:"advertise_addr_wan_ipv6,omitempty" hcl:"advertise_addr_wan_ipv6" mapstructure:"advertise_addr_ipv6"` + AutoConfig AutoConfigRaw `json:"auto_config,omitempty" hcl:"auto_config" mapstructure:"auto_config"` Autopilot Autopilot `json:"autopilot,omitempty" hcl:"autopilot" mapstructure:"autopilot"` BindAddr *string `json:"bind_addr,omitempty" hcl:"bind_addr" mapstructure:"bind_addr"` Bootstrap *bool `json:"bootstrap,omitempty" hcl:"bootstrap" mapstructure:"bootstrap"` @@ -693,3 +694,34 @@ type AuditSink struct { RotateDuration *string `json:"rotate_duration,omitempty" hcl:"rotate_duration" mapstructure:"rotate_duration"` RotateMaxFiles *int `json:"rotate_max_files,omitempty" hcl:"rotate_max_files" mapstructure:"rotate_max_files"` } + +type AutoConfigRaw struct { + Enabled *bool `json:"enabled,omitempty" hcl:"enabled" mapstructure:"enabled"` + IntroToken *string `json:"intro_token,omitempty" hcl:"intro_token" mapstructure:"intro_token"` + IntroTokenFile *string `json:"intro_token_file,omitempty" hcl:"intro_token_file" mapstructure:"intro_token_file"` + ServerAddresses []string `json:"server_addresses,omitempty" hcl:"server_addresses" mapstructure:"server_addresses"` + DNSSANs []string `json:"dns_sans,omitempty" hcl:"dns_sans" mapstructure:"dns_sans"` + IPSANs []string `json:"ip_sans,omitempty" hcl:"ip_sans" mapstructure:"ip_sans"` + Authorizer AutoConfigAuthorizerRaw `json:"authorizer,omitempty" hcl:"authorizer" mapstructure:"authorizer"` +} + +type AutoConfigAuthorizerRaw struct { + Enabled *bool `json:"enabled,omitempty" hcl:"enabled" mapstructure:"enabled"` + ClaimAssertions []string `json:"claim_assertions,omitempty" hcl:"claim_assertions" mapstructure:"claim_assertions"` + AllowReuse *bool `json:"allow_reuse,omitempty" hcl:"allow_reuse" mapstructure:"allow_reuse"` + + // Fields to be shared with the JWT Auth Method + JWTSupportedAlgs []string `json:"jwt_supported_algs,omitempty" hcl:"jwt_supported_algs" mapstructure:"jwt_supported_algs"` + BoundAudiences []string `json:"bound_audiences,omitempty" hcl:"bound_audiences" mapstructure:"bound_audiences"` + ClaimMappings map[string]string `json:"claim_mappings,omitempty" hcl:"claim_mappings" mapstructure:"claim_mappings"` + ListClaimMappings map[string]string `json:"list_claim_mappings,omitempty" hcl:"list_claim_mappings" mapstructure:"list_claim_mappings"` + OIDCDiscoveryURL *string `json:"oidc_discovery_url,omitempty" hcl:"oidc_discovery_url" mapstructure:"oidc_discovery_url"` + OIDCDiscoveryCACert *string `json:"oidc_discovery_ca_cert,omitempty" hcl:"oidc_discovery_ca_cert" mapstructure:"oidc_discovery_ca_cert"` + JWKSURL *string `json:"jwks_url,omitempty" hcl:"jwks_url" mapstructure:"jwks_url"` + JWKSCACert *string `json:"jwks_ca_cert,omitempty" hcl:"jwks_ca_cert" mapstructure:"jwks_ca_cert"` + JWTValidationPubKeys []string `json:"jwt_validation_pub_keys,omitempty" hcl:"jwt_validation_pub_keys" mapstructure:"jwt_validation_pub_keys"` + BoundIssuer *string `json:"bound_issuer,omitempty" hcl:"bound_issuer" mapstructure:"bound_issuer"` + ExpirationLeeway *string `json:"expiration_leeway,omitempty" hcl:"expiration_leeway" mapstructure:"expiration_leeway"` + NotBeforeLeeway *string `json:"not_before_leeway,omitempty" hcl:"not_before_leeway" mapstructure:"not_before_leeway"` + ClockSkewLeeway *string `json:"clock_skew_leeway,omitempty" hcl:"clock_skew_leeway" mapstructure:"clock_skew_leeway"` +} diff --git a/agent/config/merge.go b/agent/config/merge.go index f3e6d3e1e0..a743891f4a 100644 --- a/agent/config/merge.go +++ b/agent/config/merge.go @@ -28,6 +28,14 @@ func merge(a, b interface{}) interface{} { func mergeValue(a, b reflect.Value) reflect.Value { switch a.Kind() { case reflect.Map: + // dont bother allocating a new map to aggregate keys in when either one + // or both of the maps to merge is the zero value - nil + if a.IsZero() { + return b + } else if b.IsZero() { + return a + } + r := reflect.MakeMap(a.Type()) for _, k := range a.MapKeys() { v := a.MapIndex(k) diff --git a/agent/config/merge_test.go b/agent/config/merge_test.go index 0bf75d91bf..4c4c7b69f4 100644 --- a/agent/config/merge_test.go +++ b/agent/config/merge_test.go @@ -39,12 +39,7 @@ func TestMerge(t *testing.T) { "a": "b", "c": "e", }, - Ports: Ports{DNS: pInt(2), HTTP: pInt(3)}, - SnapshotAgent: map[string]interface{}{}, - TaggedAddresses: map[string]string{}, - HTTPConfig: HTTPConfig{ResponseHeaders: map[string]string{}}, - DNS: DNS{ServiceTTL: map[string]string{}}, - Connect: Connect{CAConfig: map[string]interface{}{}}, + Ports: Ports{DNS: pInt(2), HTTP: pInt(3)}, }, }, } diff --git a/agent/config/runtime.go b/agent/config/runtime.go index 2d45f57882..66f28d171d 100644 --- a/agent/config/runtime.go +++ b/agent/config/runtime.go @@ -540,6 +540,10 @@ type RuntimeConfig struct { // AutoEncrypt.Sign requests. AutoEncryptAllowTLS bool + // AutoConfig is a grouping of the configurations around the agent auto configuration + // process including how servers can authorize requests. + AutoConfig AutoConfig + // ConnectEnabled opts the agent into connect. It should be set on all clients // and servers in a cluster for correct connect operation. ConnectEnabled bool @@ -1566,6 +1570,24 @@ type RuntimeConfig struct { EnterpriseRuntimeConfig } +type AutoConfig struct { + Enabled bool + IntroToken string + IntroTokenFile string + ServerAddresses []string + DNSSANs []string + IPSANs []net.IP + Authorizer AutoConfigAuthorizer +} + +type AutoConfigAuthorizer struct { + Enabled bool + AuthMethod structs.ACLAuthMethod + // AuthMethodConfig ssoauth.Config + ClaimAssertions []string + AllowReuse bool +} + func (c *RuntimeConfig) apiAddresses(maxPerType int) (unixAddrs, httpAddrs, httpsAddrs []string) { if len(c.HTTPSAddrs) > 0 { for i, addr := range c.HTTPSAddrs { @@ -1729,6 +1751,10 @@ func (c *RuntimeConfig) ToTLSUtilConfig() tlsutil.Config { // isSecret determines whether a field name represents a field which // may contain a secret. func isSecret(name string) bool { + // special cases for AuthMethod locality and intro token file + if name == "TokenLocality" || name == "IntroTokenFile" { + return false + } name = strings.ToLower(name) return strings.Contains(name, "key") || strings.Contains(name, "token") || strings.Contains(name, "secret") } diff --git a/agent/config/runtime_test.go b/agent/config/runtime_test.go index 20d9e70be7..3b63a194f0 100644 --- a/agent/config/runtime_test.go +++ b/agent/config/runtime_test.go @@ -1628,6 +1628,7 @@ func TestConfigFlagsAndEdgecases(t *testing.T) { }, warns: []string{`config key "acl_enforce_version_8" is deprecated and should be removed`}, }, + { desc: "advertise address detect fails v4", args: []string{`-data-dir=` + dataDir}, @@ -3795,6 +3796,272 @@ func TestConfigFlagsAndEdgecases(t *testing.T) { rt.RPCMaxConnsPerClient = 100 }, }, + + /////////////////////////////////// + // Auto Config related tests + { + desc: "auto config not allowed for servers", + args: []string{ + `-data-dir=` + dataDir, + }, + hcl: []string{` + server = true + auto_config { + enabled = true + intro_token = "blah" + server_addresses = ["198.18.0.1"] + } + `}, + json: []string{` + { + "server": true, + "auto_config": { + "enabled": true, + "intro_token": "blah", + "server_addresses": ["198.18.0.1"] + } + }`}, + err: "auto_config.enabled cannot be set to true for server agents", + }, + + { + desc: "auto config no intro token", + args: []string{ + `-data-dir=` + dataDir, + }, + hcl: []string{` + auto_config { + enabled = true + server_addresses = ["198.18.0.1"] + + } + `}, + json: []string{` + { + "auto_config": { + "enabled": true, + "server_addresses": ["198.18.0.1"] + } + }`}, + err: "one of auto_config.intro_token or auto_config.intro_token_file must be set to enable auto_config", + }, + + { + desc: "auto config no server addresses", + args: []string{ + `-data-dir=` + dataDir, + }, + hcl: []string{` + auto_config { + enabled = true + intro_token = "blah" + } + `}, + json: []string{` + { + "auto_config": { + "enabled": true, + "intro_token": "blah" + } + }`}, + err: "auto_config.enabled is set without providing a list of addresses", + }, + + { + desc: "auto config client", + args: []string{ + `-data-dir=` + dataDir, + }, + hcl: []string{` + auto_config { + enabled = true + intro_token = "blah" + intro_token_file = "blah" + server_addresses = ["198.18.0.1"] + dns_sans = ["foo"] + ip_sans = ["invalid", "127.0.0.1"] + } + `}, + json: []string{` + { + "auto_config": { + "enabled": true, + "intro_token": "blah", + "intro_token_file": "blah", + "server_addresses": ["198.18.0.1"], + "dns_sans": ["foo"], + "ip_sans": ["invalid", "127.0.0.1"] + } + }`}, + warns: []string{ + "Cannot parse ip \"invalid\" from auto_config.ip_sans", + "auto_config.intro_token and auto_config.intro_token_file are both set. Using the value of auto_config.intro_token", + }, + patch: func(rt *RuntimeConfig) { + rt.AutoConfig.Enabled = true + rt.AutoConfig.IntroToken = "blah" + rt.AutoConfig.IntroTokenFile = "blah" + rt.AutoConfig.ServerAddresses = []string{"198.18.0.1"} + rt.AutoConfig.DNSSANs = []string{"foo"} + rt.AutoConfig.IPSANs = []net.IP{net.IPv4(127, 0, 0, 1)} + rt.DataDir = dataDir + }, + }, + + { + desc: "auto config authorizer client not allowed", + args: []string{ + `-data-dir=` + dataDir, + }, + hcl: []string{` + auto_config { + authorizer { + enabled = true + } + } + `}, + json: []string{` + { + "auto_config": { + "authorizer": { + "enabled": true + } + } + }`}, + err: "auto_config.authorizer.enabled cannot be set to true for client agents", + }, + + { + desc: "auto config authorizer invalid config", + args: []string{ + `-data-dir=` + dataDir, + `-server`, + }, + hcl: []string{` + auto_config { + authorizer { + enabled = true + } + } + `}, + json: []string{` + { + "auto_config": { + "authorizer": { + "enabled": true + } + } + }`}, + err: `auto_config.authorizer has invalid configuration: exactly one of 'JWTValidationPubKeys', 'JWKSURL', or 'OIDCDiscoveryURL' must be set for type "jwt"`, + }, + + { + desc: "auto config authorizer invalid config 2", + args: []string{ + `-data-dir=` + dataDir, + `-server`, + }, + hcl: []string{` + auto_config { + authorizer { + enabled = true + jwks_url = "https://fake.uri.local" + oidc_discovery_url = "https://fake.uri.local" + } + } + `}, + json: []string{` + { + "auto_config": { + "authorizer": { + "enabled": true, + "jwks_url": "https://fake.uri.local", + "oidc_discovery_url": "https://fake.uri.local" + } + } + }`}, + err: `auto_config.authorizer has invalid configuration: exactly one of 'JWTValidationPubKeys', 'JWKSURL', or 'OIDCDiscoveryURL' must be set for type "jwt"`, + }, + + { + desc: "auto config authorizer invalid claim assertion", + args: []string{ + `-data-dir=` + dataDir, + `-server`, + }, + hcl: []string{` + auto_config { + authorizer { + enabled = true + jwt_validation_pub_keys = ["-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAERVchfCZng4mmdvQz1+sJHRN40snC\nYt8NjYOnbnScEXMkyoUmASr88gb7jaVAVt3RYASAbgBjB2Z+EUizWkx5Tg==\n-----END PUBLIC KEY-----"] + claim_assertions = [ + "values.node == ${node}" + ] + } + } + `}, + json: []string{` + { + "auto_config": { + "authorizer": { + "enabled": true, + "jwt_validation_pub_keys": ["-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAERVchfCZng4mmdvQz1+sJHRN40snC\nYt8NjYOnbnScEXMkyoUmASr88gb7jaVAVt3RYASAbgBjB2Z+EUizWkx5Tg==\n-----END PUBLIC KEY-----"], + "claim_assertions": [ + "values.node == ${node}" + ] + } + } + }`}, + err: `auto_config.claim_assertion "values.node == ${node}" is invalid: Selector "values" is not valid`, + }, + { + desc: "auto config authorizer ok", + args: []string{ + `-data-dir=` + dataDir, + `-server`, + }, + hcl: []string{` + auto_config { + authorizer { + enabled = true + jwt_validation_pub_keys = ["-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAERVchfCZng4mmdvQz1+sJHRN40snC\nYt8NjYOnbnScEXMkyoUmASr88gb7jaVAVt3RYASAbgBjB2Z+EUizWkx5Tg==\n-----END PUBLIC KEY-----"] + claim_assertions = [ + "value.node == ${node}" + ] + claim_mappings = { + node = "node" + } + } + } + `}, + json: []string{` + { + "auto_config": { + "authorizer": { + "enabled": true, + "jwt_validation_pub_keys": ["-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAERVchfCZng4mmdvQz1+sJHRN40snC\nYt8NjYOnbnScEXMkyoUmASr88gb7jaVAVt3RYASAbgBjB2Z+EUizWkx5Tg==\n-----END PUBLIC KEY-----"], + "claim_assertions": [ + "value.node == ${node}" + ], + "claim_mappings": { + "node": "node" + } + } + } + }`}, + patch: func(rt *RuntimeConfig) { + rt.AutoConfig.Authorizer.Enabled = true + rt.AutoConfig.Authorizer.AuthMethod.Config["ClaimMappings"] = map[string]string{ + "node": "node", + } + rt.AutoConfig.Authorizer.AuthMethod.Config["JWTValidationPubKeys"] = []string{"-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAERVchfCZng4mmdvQz1+sJHRN40snC\nYt8NjYOnbnScEXMkyoUmASr88gb7jaVAVt3RYASAbgBjB2Z+EUizWkx5Tg==\n-----END PUBLIC KEY-----"} + rt.AutoConfig.Authorizer.ClaimAssertions = []string{"value.node == ${node}"} + rt.DataDir = dataDir + rt.LeaveOnTerm = false + rt.ServerMode = true + rt.SkipLeaveOnInt = true + }, + }, } testConfig(t, tests, dataDir) @@ -4027,6 +4294,28 @@ func TestFullConfig(t *testing.T) { "audit": { "enabled": false }, + "auto_config": { + "enabled": false, + "intro_token": "OpBPGRwt", + "intro_token_file": "gFvAXwI8", + "dns_sans": ["6zdaWg9J"], + "ip_sans": ["198.18.99.99"], + "server_addresses": ["198.18.100.1"], + "authorizer": { + "enabled": true, + "allow_reuse": true, + "claim_mappings": { + "node": "node" + }, + "list_claim_mappings": { + "foo": "bar" + }, + "bound_issuer": "consul", + "bound_audiences": ["consul-cluster-1"], + "claim_assertions": ["value.node == \"${node}\""], + "jwt_validation_pub_keys": ["-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAERVchfCZng4mmdvQz1+sJHRN40snC\nYt8NjYOnbnScEXMkyoUmASr88gb7jaVAVt3RYASAbgBjB2Z+EUizWkx5Tg==\n-----END PUBLIC KEY-----"] + } + }, "autopilot": { "cleanup_dead_servers": true, "disable_upgrade_migration": true, @@ -4663,6 +4952,28 @@ func TestFullConfig(t *testing.T) { audit = { enabled = false } + auto_config = { + enabled = false + intro_token = "OpBPGRwt" + intro_token_file = "gFvAXwI8" + dns_sans = ["6zdaWg9J"] + ip_sans = ["198.18.99.99"] + server_addresses = ["198.18.100.1"] + authorizer = { + enabled = true + allow_reuse = true + claim_mappings = { + node = "node" + } + list_claim_mappings = { + foo = "bar" + } + bound_issuer = "consul" + bound_audiences = ["consul-cluster-1"] + claim_assertions = ["value.node == \"${node}\""] + jwt_validation_pub_keys = ["-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAERVchfCZng4mmdvQz1+sJHRN40snC\nYt8NjYOnbnScEXMkyoUmASr88gb7jaVAVt3RYASAbgBjB2Z+EUizWkx5Tg==\n-----END PUBLIC KEY-----"] + } + } autopilot = { cleanup_dead_servers = true disable_upgrade_migration = true @@ -5496,10 +5807,49 @@ func TestFullConfig(t *testing.T) { }, }, }, - AutoEncryptTLS: false, - AutoEncryptDNSSAN: []string{"a.com", "b.com"}, - AutoEncryptIPSAN: []net.IP{net.ParseIP("192.168.4.139"), net.ParseIP("192.168.4.140")}, - AutoEncryptAllowTLS: true, + AutoEncryptTLS: false, + AutoEncryptDNSSAN: []string{"a.com", "b.com"}, + AutoEncryptIPSAN: []net.IP{net.ParseIP("192.168.4.139"), net.ParseIP("192.168.4.140")}, + AutoEncryptAllowTLS: true, + AutoConfig: AutoConfig{ + Enabled: false, + IntroToken: "OpBPGRwt", + IntroTokenFile: "gFvAXwI8", + DNSSANs: []string{"6zdaWg9J"}, + IPSANs: []net.IP{net.IPv4(198, 18, 99, 99)}, + ServerAddresses: []string{"198.18.100.1"}, + Authorizer: AutoConfigAuthorizer{ + Enabled: true, + AllowReuse: true, + ClaimAssertions: []string{"value.node == \"${node}\""}, + AuthMethod: structs.ACLAuthMethod{ + Name: "Auto Config Authorizer", + Type: "jwt", + MaxTokenTTL: 72 * time.Hour, + EnterpriseMeta: *structs.DefaultEnterpriseMeta(), + Config: map[string]interface{}{ + "JWTValidationPubKeys": []string{"-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAERVchfCZng4mmdvQz1+sJHRN40snC\nYt8NjYOnbnScEXMkyoUmASr88gb7jaVAVt3RYASAbgBjB2Z+EUizWkx5Tg==\n-----END PUBLIC KEY-----"}, + "ClaimMappings": map[string]string{ + "node": "node", + }, + "BoundIssuer": "consul", + "BoundAudiences": []string{"consul-cluster-1"}, + "ListClaimMappings": map[string]string{ + "foo": "bar", + }, + "OIDCDiscoveryURL": "", + "OIDCDiscoveryCACert": "", + "JWKSURL": "", + "JWKSCACert": "", + "ExpirationLeeway": 0 * time.Second, + "NotBeforeLeeway": 0 * time.Second, + "ClockSkewLeeway": 0 * time.Second, + "JWTSupportedAlgs": []string(nil), + }, + TokenLocality: "local", + }, + }, + }, ConnectEnabled: true, ConnectSidecarMinPort: 8888, ConnectSidecarMaxPort: 9999, @@ -6621,7 +6971,35 @@ func TestSanitize(t *testing.T) { "AllowWriteHTTPFrom": [ "127.0.0.0/8", "::1/128" - ] + ], + "AutoConfig": { + "Authorizer": { + "Enabled": false, + "AllowReuse": false, + "AuthMethod": { + "ACLAuthMethodEnterpriseFields": {}, + "Config": {}, + "Description": "", + "DisplayName": "", + "EnterpriseMeta": {}, + "MaxTokenTTL": "0s", + "Name": "", + "RaftIndex": { + "CreateIndex": 0, + "ModifyIndex": 0 + }, + "Type": "", + "TokenLocality": "" + }, + "ClaimAssertions": [] + }, + "Enabled": false, + "DNSSANs": [], + "IntroToken": "hidden", + "IntroTokenFile": "", + "IPSANs": [], + "ServerAddresses": [] + } }` b, err := json.MarshalIndent(rt.Sanitized(), "", " ") if err != nil { diff --git a/agent/consul/acl_endpoint.go b/agent/consul/acl_endpoint.go index 8dc5d3c71a..f4d29ab9f7 100644 --- a/agent/consul/acl_endpoint.go +++ b/agent/consul/acl_endpoint.go @@ -17,6 +17,7 @@ import ( "github.com/hashicorp/consul/agent/consul/state" "github.com/hashicorp/consul/agent/structs" "github.com/hashicorp/consul/lib" + "github.com/hashicorp/consul/lib/template" "github.com/hashicorp/go-bexpr" "github.com/hashicorp/go-hclog" memdb "github.com/hashicorp/go-memdb" @@ -706,7 +707,7 @@ func validateBindingRuleBindName(bindType, bindName string, availableFields []st // - If the computed name is not valid for the type ("INVALID_NAME", false, nil) is returned. // - If the computed name is valid for the type ("VALID_NAME", true, nil) is returned. func computeBindingRuleBindName(bindType, bindName string, projectedVars map[string]string) (string, bool, error) { - bindName, err := InterpolateHIL(bindName, projectedVars, true) + bindName, err := template.InterpolateHIL(bindName, projectedVars, true) if err != nil { return "", false, err } diff --git a/agent/consul/util.go b/agent/consul/util.go index 79b5e935f6..8bbc247d58 100644 --- a/agent/consul/util.go +++ b/agent/consul/util.go @@ -6,13 +6,10 @@ import ( "net" "runtime" "strconv" - "strings" "github.com/hashicorp/consul/agent/metadata" "github.com/hashicorp/consul/agent/structs" "github.com/hashicorp/go-version" - "github.com/hashicorp/hil" - "github.com/hashicorp/hil/ast" "github.com/hashicorp/serf/serf" ) @@ -440,45 +437,3 @@ func ServersGetACLMode(provider checkServersProvider, leaderAddr string, datacen return state.found, state.mode, state.leaderMode } - -// InterpolateHIL processes the string as if it were HIL and interpolates only -// the provided string->string map as possible variables. -func InterpolateHIL(s string, vars map[string]string, lowercase bool) (string, error) { - if strings.Index(s, "${") == -1 { - // Skip going to the trouble of parsing something that has no HIL. - return s, nil - } - - tree, err := hil.Parse(s) - if err != nil { - return "", err - } - - vm := make(map[string]ast.Variable) - for k, v := range vars { - if lowercase { - v = strings.ToLower(v) - } - vm[k] = ast.Variable{ - Type: ast.TypeString, - Value: v, - } - } - - config := &hil.EvalConfig{ - GlobalScope: &ast.BasicScope{ - VarMap: vm, - }, - } - - result, err := hil.Eval(tree, config) - if err != nil { - return "", err - } - - if result.Type != hil.TypeString { - return "", fmt.Errorf("generated unexpected hil type: %s", result.Type) - } - - return result.Value.(string), nil -} diff --git a/agent/consul/util_test.go b/agent/consul/util_test.go index 3307ada8ba..3fbea831c3 100644 --- a/agent/consul/util_test.go +++ b/agent/consul/util_test.go @@ -442,161 +442,6 @@ func TestServersInDCMeetMinimumVersion(t *testing.T) { } } -func TestInterpolateHIL(t *testing.T) { - for name, test := range map[string]struct { - in string - vars map[string]string - exp string // when lower=false - expLower string // when lower=true - ok bool - }{ - // valid HIL - "empty": { - "", - map[string]string{}, - "", - "", - true, - }, - "no vars": { - "nothing", - map[string]string{}, - "nothing", - "nothing", - true, - }, - "just lowercase var": { - "${item}", - map[string]string{"item": "value"}, - "value", - "value", - true, - }, - "just uppercase var": { - "${item}", - map[string]string{"item": "VaLuE"}, - "VaLuE", - "value", - true, - }, - "lowercase var in middle": { - "before ${item}after", - map[string]string{"item": "value"}, - "before valueafter", - "before valueafter", - true, - }, - "uppercase var in middle": { - "before ${item}after", - map[string]string{"item": "VaLuE"}, - "before VaLuEafter", - "before valueafter", - true, - }, - "two vars": { - "before ${item}after ${more}", - map[string]string{"item": "value", "more": "xyz"}, - "before valueafter xyz", - "before valueafter xyz", - true, - }, - "missing map val": { - "${item}", - map[string]string{"item": ""}, - "", - "", - true, - }, - // "weird" HIL, but not technically invalid - "just end": { - "}", - map[string]string{}, - "}", - "}", - true, - }, - "var without start": { - " item }", - map[string]string{"item": "value"}, - " item }", - " item }", - true, - }, - "two vars missing second start": { - "before ${ item }after more }", - map[string]string{"item": "value", "more": "xyz"}, - "before valueafter more }", - "before valueafter more }", - true, - }, - // invalid HIL - "just start": { - "${", - map[string]string{}, - "", - "", - false, - }, - "backwards": { - "}${", - map[string]string{}, - "", - "", - false, - }, - "no varname": { - "${}", - map[string]string{}, - "", - "", - false, - }, - "missing map key": { - "${item}", - map[string]string{}, - "", - "", - false, - }, - "var without end": { - "${ item ", - map[string]string{"item": "value"}, - "", - "", - false, - }, - "two vars missing first end": { - "before ${ item after ${ more }", - map[string]string{"item": "value", "more": "xyz"}, - "", - "", - false, - }, - } { - test := test - t.Run(name+" lower=false", func(t *testing.T) { - out, err := InterpolateHIL(test.in, test.vars, false) - if test.ok { - require.NoError(t, err) - require.Equal(t, test.exp, out) - } else { - require.NotNil(t, err) - require.Equal(t, out, "") - } - }) - t.Run(name+" lower=true", func(t *testing.T) { - out, err := InterpolateHIL(test.in, test.vars, true) - if test.ok { - require.NoError(t, err) - require.Equal(t, test.expLower, out) - } else { - require.NotNil(t, err) - require.Equal(t, out, "") - } - }) - } -} - func TestServersGetACLMode(t *testing.T) { t.Parallel() makeServer := func(datacenter string, acls structs.ACLMode, status serf.MemberStatus, addr net.IP) metadata.Server { diff --git a/agent/local/state_test.go b/agent/local/state_test.go index 872876d8de..053c5420fc 100644 --- a/agent/local/state_test.go +++ b/agent/local/state_test.go @@ -22,6 +22,12 @@ import ( "github.com/stretchr/testify/require" ) +func unNilMap(in map[string]string) map[string]string { + if in == nil { + return make(map[string]string) + } + return in +} func TestAgentAntiEntropy_Services(t *testing.T) { t.Parallel() a := agent.NewTestAgent(t, "") @@ -170,7 +176,7 @@ func TestAgentAntiEntropy_Services(t *testing.T) { delete(meta, structs.MetaSegmentKey) // Added later, not in config. assert.Equal(t, a.Config.NodeID, id) assert.Equal(t, a.Config.TaggedAddresses, addrs) - assert.Equal(t, a.Config.NodeMeta, meta) + assert.Equal(t, unNilMap(a.Config.NodeMeta), meta) // We should have 6 services (consul included) if len(services.NodeServices.Services) != 6 { @@ -1045,7 +1051,7 @@ func TestAgentAntiEntropy_Checks(t *testing.T) { delete(meta, structs.MetaSegmentKey) // Added later, not in config. assert.Equal(t, a.Config.NodeID, id) assert.Equal(t, a.Config.TaggedAddresses, addrs) - assert.Equal(t, a.Config.NodeMeta, meta) + assert.Equal(t, unNilMap(a.Config.NodeMeta), meta) } }) retry.Run(t, func(r *retry.R) { @@ -1686,7 +1692,7 @@ func TestAgentAntiEntropy_NodeInfo(t *testing.T) { delete(meta, structs.MetaSegmentKey) // Added later, not in config. require.Equal(t, a.Config.NodeID, id) require.Equal(t, a.Config.TaggedAddresses, addrs) - require.Equal(t, a.Config.NodeMeta, meta) + assert.Equal(t, unNilMap(a.Config.NodeMeta), meta) // Blow away the catalog version of the node info if err := a.RPC("Catalog.Register", args, &out); err != nil { diff --git a/lib/template/hil.go b/lib/template/hil.go new file mode 100644 index 0000000000..f3b22f5b1c --- /dev/null +++ b/lib/template/hil.go @@ -0,0 +1,51 @@ +package template + +import ( + "fmt" + "strings" + + "github.com/hashicorp/hil" + "github.com/hashicorp/hil/ast" +) + +// InterpolateHIL processes the string as if it were HIL and interpolates only +// the provided string->string map as possible variables. +func InterpolateHIL(s string, vars map[string]string, lowercase bool) (string, error) { + if strings.Index(s, "${") == -1 { + // Skip going to the trouble of parsing something that has no HIL. + return s, nil + } + + tree, err := hil.Parse(s) + if err != nil { + return "", err + } + + vm := make(map[string]ast.Variable) + for k, v := range vars { + if lowercase { + v = strings.ToLower(v) + } + vm[k] = ast.Variable{ + Type: ast.TypeString, + Value: v, + } + } + + config := &hil.EvalConfig{ + GlobalScope: &ast.BasicScope{ + VarMap: vm, + }, + } + + result, err := hil.Eval(tree, config) + if err != nil { + return "", err + } + + if result.Type != hil.TypeString { + return "", fmt.Errorf("generated unexpected hil type: %s", result.Type) + } + + return result.Value.(string), nil +} diff --git a/lib/template/hil_test.go b/lib/template/hil_test.go new file mode 100644 index 0000000000..3147ab3097 --- /dev/null +++ b/lib/template/hil_test.go @@ -0,0 +1,162 @@ +package template + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestInterpolateHIL(t *testing.T) { + for name, test := range map[string]struct { + in string + vars map[string]string + exp string // when lower=false + expLower string // when lower=true + ok bool + }{ + // valid HIL + "empty": { + "", + map[string]string{}, + "", + "", + true, + }, + "no vars": { + "nothing", + map[string]string{}, + "nothing", + "nothing", + true, + }, + "just lowercase var": { + "${item}", + map[string]string{"item": "value"}, + "value", + "value", + true, + }, + "just uppercase var": { + "${item}", + map[string]string{"item": "VaLuE"}, + "VaLuE", + "value", + true, + }, + "lowercase var in middle": { + "before ${item}after", + map[string]string{"item": "value"}, + "before valueafter", + "before valueafter", + true, + }, + "uppercase var in middle": { + "before ${item}after", + map[string]string{"item": "VaLuE"}, + "before VaLuEafter", + "before valueafter", + true, + }, + "two vars": { + "before ${item}after ${more}", + map[string]string{"item": "value", "more": "xyz"}, + "before valueafter xyz", + "before valueafter xyz", + true, + }, + "missing map val": { + "${item}", + map[string]string{"item": ""}, + "", + "", + true, + }, + // "weird" HIL, but not technically invalid + "just end": { + "}", + map[string]string{}, + "}", + "}", + true, + }, + "var without start": { + " item }", + map[string]string{"item": "value"}, + " item }", + " item }", + true, + }, + "two vars missing second start": { + "before ${ item }after more }", + map[string]string{"item": "value", "more": "xyz"}, + "before valueafter more }", + "before valueafter more }", + true, + }, + // invalid HIL + "just start": { + "${", + map[string]string{}, + "", + "", + false, + }, + "backwards": { + "}${", + map[string]string{}, + "", + "", + false, + }, + "no varname": { + "${}", + map[string]string{}, + "", + "", + false, + }, + "missing map key": { + "${item}", + map[string]string{}, + "", + "", + false, + }, + "var without end": { + "${ item ", + map[string]string{"item": "value"}, + "", + "", + false, + }, + "two vars missing first end": { + "before ${ item after ${ more }", + map[string]string{"item": "value", "more": "xyz"}, + "", + "", + false, + }, + } { + test := test + t.Run(name+" lower=false", func(t *testing.T) { + out, err := InterpolateHIL(test.in, test.vars, false) + if test.ok { + require.NoError(t, err) + require.Equal(t, test.exp, out) + } else { + require.NotNil(t, err) + require.Equal(t, out, "") + } + }) + t.Run(name+" lower=true", func(t *testing.T) { + out, err := InterpolateHIL(test.in, test.vars, true) + if test.ok { + require.NoError(t, err) + require.Equal(t, test.expLower, out) + } else { + require.NotNil(t, err) + require.Equal(t, out, "") + } + }) + } +}