From 8c545b520623267aa5a67c63fe4e9c9d9962aec9 Mon Sep 17 00:00:00 2001 From: Matt Keeler Date: Mon, 27 Apr 2020 16:52:32 -0400 Subject: [PATCH 1/4] Update mapstructure to v1.2.3 This release contains a fix to prevent duplicate keys in the Metadata after decoding where the output value contains pointer fields. --- go.mod | 2 +- go.sum | 2 + .../mitchellh/mapstructure/.travis.yml | 3 +- .../mitchellh/mapstructure/CHANGELOG.md | 23 ++ .../github.com/mitchellh/mapstructure/go.mod | 2 + .../mitchellh/mapstructure/mapstructure.go | 231 +++++++++++++++--- vendor/modules.txt | 2 +- 7 files changed, 227 insertions(+), 38 deletions(-) diff --git a/go.mod b/go.mod index 5596d754c5..193158f5bd 100644 --- a/go.mod +++ b/go.mod @@ -65,7 +65,7 @@ require ( github.com/mitchellh/copystructure v1.0.0 github.com/mitchellh/go-testing-interface v1.14.0 github.com/mitchellh/hashstructure v0.0.0-20170609045927-2bca23e0e452 - github.com/mitchellh/mapstructure v1.1.2 + github.com/mitchellh/mapstructure v1.2.3 github.com/mitchellh/reflectwalk v1.0.1 github.com/pascaldekloe/goe v0.1.0 github.com/pkg/errors v0.8.1 diff --git a/go.sum b/go.sum index d4f638fa7d..ce597bd20a 100644 --- a/go.sum +++ b/go.sum @@ -336,6 +336,8 @@ github.com/mitchellh/hashstructure v0.0.0-20170609045927-2bca23e0e452/go.mod h1: github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.2.3 h1:f/MjBEBDLttYCGfRaKBbKSRVF5aV2O6fnBpzknuE3jU= +github.com/mitchellh/mapstructure v1.2.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mitchellh/reflectwalk v1.0.1 h1:FVzMWA5RllMAKIdUSC8mdWo3XtwoecrH79BY70sEEpE= github.com/mitchellh/reflectwalk v1.0.1/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= diff --git a/vendor/github.com/mitchellh/mapstructure/.travis.yml b/vendor/github.com/mitchellh/mapstructure/.travis.yml index 1689c7d735..5e31a95a8b 100644 --- a/vendor/github.com/mitchellh/mapstructure/.travis.yml +++ b/vendor/github.com/mitchellh/mapstructure/.travis.yml @@ -1,8 +1,9 @@ language: go go: - - "1.11.x" + - "1.14.x" - tip script: - go test + - go test -bench . -benchmem diff --git a/vendor/github.com/mitchellh/mapstructure/CHANGELOG.md b/vendor/github.com/mitchellh/mapstructure/CHANGELOG.md index 3b3cb723f8..60816288b1 100644 --- a/vendor/github.com/mitchellh/mapstructure/CHANGELOG.md +++ b/vendor/github.com/mitchellh/mapstructure/CHANGELOG.md @@ -1,3 +1,26 @@ +## 1.2.3 + +* Fix duplicate entries in Keys list with pointer values. [GH-185] + +## 1.2.2 + +* Do not add unsettable (unexported) values to the unused metadata key + or "remain" value. [GH-150] + +## 1.2.1 + +* Go modules checksum mismatch fix + +## 1.2.0 + +* Added support to capture unused values in a field using the `",remain"` value + in the mapstructure tag. There is an example to showcase usage. +* Added `DecoderConfig` option to always squash embedded structs +* `json.Number` can decode into `uint` types +* Empty slices are preserved and not replaced with nil slices +* Fix panic that can occur in when decoding a map into a nil slice of structs +* Improved package documentation for godoc + ## 1.1.2 * Fix error when decode hook decodes interface implementation into interface diff --git a/vendor/github.com/mitchellh/mapstructure/go.mod b/vendor/github.com/mitchellh/mapstructure/go.mod index d2a7125620..a03ae97308 100644 --- a/vendor/github.com/mitchellh/mapstructure/go.mod +++ b/vendor/github.com/mitchellh/mapstructure/go.mod @@ -1 +1,3 @@ module github.com/mitchellh/mapstructure + +go 1.14 diff --git a/vendor/github.com/mitchellh/mapstructure/mapstructure.go b/vendor/github.com/mitchellh/mapstructure/mapstructure.go index 256ee63fbf..e05351044f 100644 --- a/vendor/github.com/mitchellh/mapstructure/mapstructure.go +++ b/vendor/github.com/mitchellh/mapstructure/mapstructure.go @@ -1,10 +1,109 @@ -// Package mapstructure exposes functionality to convert an arbitrary -// map[string]interface{} into a native Go structure. +// Package mapstructure exposes functionality to convert one arbitrary +// Go type into another, typically to convert a map[string]interface{} +// into a native Go structure. // // The Go structure can be arbitrarily complex, containing slices, // other structs, etc. and the decoder will properly decode nested // maps and so on into the proper structures in the native Go struct. // See the examples to see what the decoder is capable of. +// +// The simplest function to start with is Decode. +// +// Field Tags +// +// When decoding to a struct, mapstructure will use the field name by +// default to perform the mapping. For example, if a struct has a field +// "Username" then mapstructure will look for a key in the source value +// of "username" (case insensitive). +// +// type User struct { +// Username string +// } +// +// You can change the behavior of mapstructure by using struct tags. +// The default struct tag that mapstructure looks for is "mapstructure" +// but you can customize it using DecoderConfig. +// +// Renaming Fields +// +// To rename the key that mapstructure looks for, use the "mapstructure" +// tag and set a value directly. For example, to change the "username" example +// above to "user": +// +// type User struct { +// Username string `mapstructure:"user"` +// } +// +// Embedded Structs and Squashing +// +// Embedded structs are treated as if they're another field with that name. +// By default, the two structs below are equivalent when decoding with +// mapstructure: +// +// type Person struct { +// Name string +// } +// +// type Friend struct { +// Person +// } +// +// type Friend struct { +// Person Person +// } +// +// This would require an input that looks like below: +// +// map[string]interface{}{ +// "person": map[string]interface{}{"name": "alice"}, +// } +// +// If your "person" value is NOT nested, then you can append ",squash" to +// your tag value and mapstructure will treat it as if the embedded struct +// were part of the struct directly. Example: +// +// type Friend struct { +// Person `mapstructure:",squash"` +// } +// +// Now the following input would be accepted: +// +// map[string]interface{}{ +// "name": "alice", +// } +// +// DecoderConfig has a field that changes the behavior of mapstructure +// to always squash embedded structs. +// +// Remainder Values +// +// If there are any unmapped keys in the source value, mapstructure by +// default will silently ignore them. You can error by setting ErrorUnused +// in DecoderConfig. If you're using Metadata you can also maintain a slice +// of the unused keys. +// +// You can also use the ",remain" suffix on your tag to collect all unused +// values in a map. The field with this tag MUST be a map type and should +// probably be a "map[string]interface{}" or "map[interface{}]interface{}". +// See example below: +// +// type Friend struct { +// Name string +// Other map[string]interface{} `mapstructure:",remain"` +// } +// +// Given the input below, Other would be populated with the other +// values that weren't used (everything but "name"): +// +// map[string]interface{}{ +// "name": "bob", +// "address": "123 Maple St.", +// } +// +// Other Configuration +// +// mapstructure is highly configurable. See the DecoderConfig struct +// for other features and options that are supported. package mapstructure import ( @@ -80,6 +179,14 @@ type DecoderConfig struct { // WeaklyTypedInput bool + // Squash will squash embedded structs. A squash tag may also be + // added to an individual struct field using a tag. For example: + // + // type Parent struct { + // Child `mapstructure:",squash"` + // } + Squash bool + // Metadata is the struct that will contain extra metadata about // the decoding. If this is nil, then no metadata will be tracked. Metadata *Metadata @@ -271,6 +378,7 @@ func (d *Decoder) decode(name string, input interface{}, outVal reflect.Value) e var err error outputKind := getKind(outVal) + addMetaKey := true switch outputKind { case reflect.Bool: err = d.decodeBool(name, input, outVal) @@ -289,7 +397,7 @@ func (d *Decoder) decode(name string, input interface{}, outVal reflect.Value) e case reflect.Map: err = d.decodeMap(name, input, outVal) case reflect.Ptr: - err = d.decodePtr(name, input, outVal) + addMetaKey, err = d.decodePtr(name, input, outVal) case reflect.Slice: err = d.decodeSlice(name, input, outVal) case reflect.Array: @@ -303,7 +411,7 @@ func (d *Decoder) decode(name string, input interface{}, outVal reflect.Value) e // If we reached here, then we successfully decoded SOMETHING, so // mark the key as used if we're tracking metainput. - if d.config.Metadata != nil && name != "" { + if addMetaKey && d.config.Metadata != nil && name != "" { d.config.Metadata.Keys = append(d.config.Metadata.Keys, name) } @@ -438,6 +546,7 @@ func (d *Decoder) decodeInt(name string, data interface{}, val reflect.Value) er func (d *Decoder) decodeUint(name string, data interface{}, val reflect.Value) error { dataVal := reflect.Indirect(reflect.ValueOf(data)) dataKind := getKind(dataVal) + dataType := dataVal.Type() switch { case dataKind == reflect.Int: @@ -469,6 +578,18 @@ func (d *Decoder) decodeUint(name string, data interface{}, val reflect.Value) e } else { return fmt.Errorf("cannot parse '%s' as uint: %s", name, err) } + case dataType.PkgPath() == "encoding/json" && dataType.Name() == "Number": + jn := data.(json.Number) + i, err := jn.Int64() + if err != nil { + return fmt.Errorf( + "error decoding json.Number into %s: %s", name, err) + } + if i < 0 && !d.config.WeaklyTypedInput { + return fmt.Errorf("cannot parse '%s', %d overflows uint", + name, i) + } + val.SetUint(uint64(i)) default: return fmt.Errorf( "'%s' expected type '%s', got unconvertible type '%s'", @@ -689,16 +810,19 @@ func (d *Decoder) decodeMapFromStruct(name string, dataVal reflect.Value, val re keyName = tagParts[0] } + // If Squash is set in the config, we squash the field down. + squash := d.config.Squash && v.Kind() == reflect.Struct // If "squash" is specified in the tag, we squash the field down. - squash := false - for _, tag := range tagParts[1:] { - if tag == "squash" { - squash = true - break + if !squash { + for _, tag := range tagParts[1:] { + if tag == "squash" { + squash = true + break + } + } + if squash && v.Kind() != reflect.Struct { + return fmt.Errorf("cannot squash non-struct type '%s'", v.Type()) } - } - if squash && v.Kind() != reflect.Struct { - return fmt.Errorf("cannot squash non-struct type '%s'", v.Type()) } switch v.Kind() { @@ -738,7 +862,7 @@ func (d *Decoder) decodeMapFromStruct(name string, dataVal reflect.Value, val re return nil } -func (d *Decoder) decodePtr(name string, data interface{}, val reflect.Value) error { +func (d *Decoder) decodePtr(name string, data interface{}, val reflect.Value) (bool, error) { // If the input data is nil, then we want to just set the output // pointer to be nil as well. isNil := data == nil @@ -759,7 +883,7 @@ func (d *Decoder) decodePtr(name string, data interface{}, val reflect.Value) er val.Set(nilValue) } - return nil + return true, nil } // Create an element of the concrete (non pointer) type and decode @@ -773,16 +897,16 @@ func (d *Decoder) decodePtr(name string, data interface{}, val reflect.Value) er } if err := d.decode(name, data, reflect.Indirect(realVal)); err != nil { - return err + return false, err } val.Set(realVal) } else { if err := d.decode(name, data, reflect.Indirect(val)); err != nil { - return err + return false, err } } - return nil + return false, nil } func (d *Decoder) decodeFunc(name string, data interface{}, val reflect.Value) error { @@ -805,8 +929,8 @@ func (d *Decoder) decodeSlice(name string, data interface{}, val reflect.Value) valElemType := valType.Elem() sliceType := reflect.SliceOf(valElemType) - valSlice := val - if valSlice.IsNil() || d.config.ZeroFields { + // If we have a non array/slice type then we first attempt to convert. + if dataValKind != reflect.Array && dataValKind != reflect.Slice { if d.config.WeaklyTypedInput { switch { // Slice and array we use the normal logic @@ -833,18 +957,17 @@ func (d *Decoder) decodeSlice(name string, data interface{}, val reflect.Value) } } - // Check input type - if dataValKind != reflect.Array && dataValKind != reflect.Slice { - return fmt.Errorf( - "'%s': source data must be an array or slice, got %s", name, dataValKind) + return fmt.Errorf( + "'%s': source data must be an array or slice, got %s", name, dataValKind) + } - } - - // If the input value is empty, then don't allocate since non-nil != nil - if dataVal.Len() == 0 { - return nil - } + // If the input value is nil, then don't allocate since empty != nil + if dataVal.IsNil() { + return nil + } + valSlice := val + if valSlice.IsNil() || d.config.ZeroFields { // Make a new slice to hold our result, same size as the original data. valSlice = reflect.MakeSlice(sliceType, dataVal.Len(), dataVal.Len()) } @@ -1005,6 +1128,11 @@ func (d *Decoder) decodeStructFromMap(name string, dataVal, val reflect.Value) e field reflect.StructField val reflect.Value } + + // remainField is set to a valid field set with the "remain" tag if + // we are keeping track of remaining values. + var remainField *field + fields := []field{} for len(structs) > 0 { structVal := structs[0] @@ -1017,13 +1145,21 @@ func (d *Decoder) decodeStructFromMap(name string, dataVal, val reflect.Value) e fieldKind := fieldType.Type.Kind() // If "squash" is specified in the tag, we squash the field down. - squash := false + squash := d.config.Squash && fieldKind == reflect.Struct + remain := false + + // We always parse the tags cause we're looking for other tags too tagParts := strings.Split(fieldType.Tag.Get(d.config.TagName), ",") for _, tag := range tagParts[1:] { if tag == "squash" { squash = true break } + + if tag == "remain" { + remain = true + break + } } if squash { @@ -1036,8 +1172,14 @@ func (d *Decoder) decodeStructFromMap(name string, dataVal, val reflect.Value) e continue } - // Normal struct field, store it away - fields = append(fields, field{fieldType, structVal.Field(i)}) + // Build our field + fieldCurrent := field{fieldType, structVal.Field(i)} + if remain { + remainField = &fieldCurrent + } else { + // Normal struct field, store it away + fields = append(fields, field{fieldType, structVal.Field(i)}) + } } } @@ -1078,9 +1220,6 @@ func (d *Decoder) decodeStructFromMap(name string, dataVal, val reflect.Value) e } } - // Delete the key we're using from the unused map so we stop tracking - delete(dataValKeysUnused, rawMapKey.Interface()) - if !fieldValue.IsValid() { // This should never happen panic("field is not valid") @@ -1092,6 +1231,9 @@ func (d *Decoder) decodeStructFromMap(name string, dataVal, val reflect.Value) e continue } + // Delete the key we're using from the unused map so we stop tracking + delete(dataValKeysUnused, rawMapKey.Interface()) + // If the name is empty string, then we're at the root, and we // don't dot-join the fields. if name != "" { @@ -1103,6 +1245,25 @@ func (d *Decoder) decodeStructFromMap(name string, dataVal, val reflect.Value) e } } + // If we have a "remain"-tagged field and we have unused keys then + // we put the unused keys directly into the remain field. + if remainField != nil && len(dataValKeysUnused) > 0 { + // Build a map of only the unused values + remain := map[interface{}]interface{}{} + for key := range dataValKeysUnused { + remain[key] = dataVal.MapIndex(reflect.ValueOf(key)).Interface() + } + + // Decode it as-if we were just decoding this map onto our map. + if err := d.decodeMap(name, remain, remainField.val); err != nil { + errors = appendErrors(errors, err) + } + + // Set the map to nil so we have none so that the next check will + // not error (ErrorUnused) + dataValKeysUnused = nil + } + if d.config.ErrorUnused && len(dataValKeysUnused) > 0 { keys := make([]string, 0, len(dataValKeysUnused)) for rawKey := range dataValKeysUnused { diff --git a/vendor/modules.txt b/vendor/modules.txt index 6ebe422f58..6e7c665012 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -303,7 +303,7 @@ github.com/mitchellh/go-homedir github.com/mitchellh/go-testing-interface # github.com/mitchellh/hashstructure v0.0.0-20170609045927-2bca23e0e452 github.com/mitchellh/hashstructure -# github.com/mitchellh/mapstructure v1.1.2 +# github.com/mitchellh/mapstructure v1.2.3 github.com/mitchellh/mapstructure # github.com/mitchellh/reflectwalk v1.0.1 github.com/mitchellh/reflectwalk From bec3fb7c18a00d73861c01c143f94e788dd01c8a Mon Sep 17 00:00:00 2001 From: Matt Keeler Date: Tue, 28 Apr 2020 09:42:46 -0400 Subject: [PATCH 2/4] Some boilerplate to allow for ACL Bootstrap disabling configurability --- agent/agent.go | 4 ++-- agent/agent_oss.go | 5 +++++ agent/config/config.go | 1 + agent/config/config_oss.go | 4 ++++ agent/consul/acl_endpoint_legacy.go | 4 ++++ 5 files changed, 16 insertions(+), 2 deletions(-) diff --git a/agent/agent.go b/agent/agent.go index d46a4bcdd9..805cdd1e48 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -1412,8 +1412,8 @@ func (a *Agent) consulConfig() (*consul.Config, error) { } base.ConfigEntryBootstrap = a.config.ConfigEntryBootstrap - - return base, nil + + return a.enterpriseConsulConfig(base) } // Setup the serf and memberlist config for any defined network segments. diff --git a/agent/agent_oss.go b/agent/agent_oss.go index ef6f69f1c8..f5be5e59ef 100644 --- a/agent/agent_oss.go +++ b/agent/agent_oss.go @@ -32,6 +32,11 @@ func (a *Agent) reloadEnterprise(conf *config.RuntimeConfig) error { return nil } +// enterpriseConsulConfig is a noop stub for the func defined in agent_ent.go +func (a *Agent) enterpriseConsulConfig(base *consul.Config) (*consul.Config, error) { + return base, nil +} + // WriteEvent is a noop stub for the func defined agent_ent.go func (a *Agent) WriteEvent(eventType string, payload interface{}) { } diff --git a/agent/config/config.go b/agent/config/config.go index 77b698dba2..4da3834afd 100644 --- a/agent/config/config.go +++ b/agent/config/config.go @@ -713,6 +713,7 @@ type ACL struct { Tokens Tokens `json:"tokens,omitempty" hcl:"tokens" mapstructure:"tokens"` DisabledTTL *string `json:"disabled_ttl,omitempty" hcl:"disabled_ttl" mapstructure:"disabled_ttl"` EnableTokenPersistence *bool `json:"enable_token_persistence" hcl:"enable_token_persistence" mapstructure:"enable_token_persistence"` + EnterpriseACLConfig `hcl:",squash" mapstructure:",squash"` } type Tokens struct { diff --git a/agent/config/config_oss.go b/agent/config/config_oss.go index 86dd7b2210..c6d44b908d 100644 --- a/agent/config/config_oss.go +++ b/agent/config/config_oss.go @@ -14,4 +14,8 @@ func (_ *EnterpriseMeta) ToStructs() structs.EnterpriseMeta { return *structs.DefaultEnterpriseMeta() } +// EnterpriseDNSConfig OSS stub type EnterpriseDNSConfig struct{} + +// EnterpriseACLConfig OSS stub +type EnterpriseACLConfig struct{} diff --git a/agent/consul/acl_endpoint_legacy.go b/agent/consul/acl_endpoint_legacy.go index 36b0fb2cc3..890699b639 100644 --- a/agent/consul/acl_endpoint_legacy.go +++ b/agent/consul/acl_endpoint_legacy.go @@ -24,6 +24,10 @@ func (a *ACL) Bootstrap(args *structs.DCSpecificRequest, reply *structs.ACL) err return acl.ErrDisabled } + if err := a.srv.aclBootstrapAllowed(); err != nil { + return err + } + // By doing some pre-checks we can head off later bootstrap attempts // without having to run them through Raft, which should curb abuse. state := a.srv.fsm.State() From 7a4c73acaf98d47a3d68a8e1eeee54735b14a21b Mon Sep 17 00:00:00 2001 From: Matt Keeler Date: Tue, 28 Apr 2020 09:44:26 -0400 Subject: [PATCH 3/4] Updates to allow for using an enterprise specific token as the agents token This is needed to allow for managed Consul instances to register themselves in the catalog with one of the managed service provider tokens. --- agent/agent.go | 2 +- agent/token/store.go | 4 ++++ agent/token/store_oss.go | 5 +++++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/agent/agent.go b/agent/agent.go index 805cdd1e48..c95910e7a5 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -1412,7 +1412,7 @@ func (a *Agent) consulConfig() (*consul.Config, error) { } base.ConfigEntryBootstrap = a.config.ConfigEntryBootstrap - + return a.enterpriseConsulConfig(base) } diff --git a/agent/token/store.go b/agent/token/store.go index 72a772904a..1d83cfa7e2 100644 --- a/agent/token/store.go +++ b/agent/token/store.go @@ -113,6 +113,10 @@ func (t *Store) AgentToken() string { t.l.RLock() defer t.l.RUnlock() + if tok := t.enterpriseAgentToken(); tok != "" { + return tok + } + if t.agentToken != "" { return t.agentToken } diff --git a/agent/token/store_oss.go b/agent/token/store_oss.go index 31744dee9f..0a182d8265 100644 --- a/agent/token/store_oss.go +++ b/agent/token/store_oss.go @@ -5,3 +5,8 @@ package token // Stub for enterpriseTokens type enterpriseTokens struct { } + +// enterpriseAgentToken OSS stub +func (s *Store) enterpriseAgentToken() string { + return "" +} From cbe3a70f5658df5b51d6a2e0ebf055b08732de9f Mon Sep 17 00:00:00 2001 From: Matt Keeler Date: Tue, 28 Apr 2020 09:45:33 -0400 Subject: [PATCH 4/4] Update enterprise configurations to be in OSS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This will emit warnings about the configs not doing anything but still allow them to be parsed. This also added the warnings for enterprise fields that we already had in OSS but didn’t change their enforcement behavior. For example, attempting to use a network segment will cause a hard error in OSS. --- agent/config/builder.go | 5 +- agent/config/builder_oss.go | 66 +++++++++++++ agent/config/builder_oss_test.go | 159 +++++++++++++++++++++++++++++++ agent/config/config.go | 82 ++++++++++------ agent/config/config_oss.go | 6 -- agent/config/default.go | 2 +- agent/config/runtime_oss_test.go | 10 ++ agent/config/runtime_test.go | 14 ++- agent/config/segment_oss_test.go | 9 ++ 9 files changed, 309 insertions(+), 44 deletions(-) create mode 100644 agent/config/builder_oss_test.go diff --git a/agent/config/builder.go b/agent/config/builder.go index f10221d671..e3b22e292f 100644 --- a/agent/config/builder.go +++ b/agent/config/builder.go @@ -278,11 +278,14 @@ func (b *Builder) Build() (rt RuntimeConfig, err error) { if s.Name == "" || s.Data == "" { continue } - c2, err := Parse(s.Data, s.Format) + c2, keys, err := Parse(s.Data, s.Format) if err != nil { return RuntimeConfig{}, fmt.Errorf("Error parsing %s: %s", s.Name, err) } + // for now this is a soft failure that will cause warnings but not actual problems + b.validateEnterpriseConfigKeys(&c2, keys) + // if we have a single 'check' or 'service' we need to add them to the // list of checks and services first since we cannot merge them // generically and later values would clobber earlier ones. diff --git a/agent/config/builder_oss.go b/agent/config/builder_oss.go index 796a6fac98..9a6538e4aa 100644 --- a/agent/config/builder_oss.go +++ b/agent/config/builder_oss.go @@ -2,6 +2,72 @@ package config +import ( + "fmt" + + "github.com/hashicorp/go-multierror" +) + +var ( + enterpriseConfigMap map[string]func(*Config) = map[string]func(c *Config){ + "non_voting_server": func(c *Config) { + // to maintain existing compatibility we don't nullify the value + }, + "segment": func(c *Config) { + // to maintain existing compatibility we don't nullify the value + }, + "segments": func(c *Config) { + // to maintain existing compatibility we don't nullify the value + }, + "autopilot.redundancy_zone_tag": func(c *Config) { + // to maintain existing compatibility we don't nullify the value + }, + "autopilot.upgrade_version_tag": func(c *Config) { + // to maintain existing compatibility we don't nullify the value + }, + "autopilot.disable_upgrade_migration": func(c *Config) { + // to maintain existing compatibility we don't nullify the value + }, + "dns_config.prefer_namespace": func(c *Config) { + c.DNS.PreferNamespace = nil + }, + "acl.msp_disable_bootstrap": func(c *Config) { + c.ACL.MSPDisableBootstrap = nil + }, + "acl.tokens.managed_service_provider": func(c *Config) { + c.ACL.Tokens.ManagedServiceProvider = nil + }, + } +) + +type enterpriseConfigKeyError struct { + key string +} + +func (e enterpriseConfigKeyError) Error() string { + return fmt.Sprintf("%q is a Consul Enterprise configuration and will have no effect", e.key) +} + func (_ *Builder) BuildEnterpriseRuntimeConfig(_ *Config) (EnterpriseRuntimeConfig, error) { return EnterpriseRuntimeConfig{}, nil } + +// validateEnterpriseConfig is a function to validate the enterprise specific +// configuration items after Parsing but before merging into the overall +// configuration. The original intent is to use it to ensure that we warn +// for enterprise configurations used in OSS. +func (b *Builder) validateEnterpriseConfigKeys(config *Config, keys []string) error { + var err error + + for _, k := range keys { + if unset, ok := enterpriseConfigMap[k]; ok { + keyErr := enterpriseConfigKeyError{key: k} + + b.warn(keyErr.Error()) + err = multierror.Append(err, keyErr) + unset(config) + } + } + + return err +} diff --git a/agent/config/builder_oss_test.go b/agent/config/builder_oss_test.go new file mode 100644 index 0000000000..d7a94a9821 --- /dev/null +++ b/agent/config/builder_oss_test.go @@ -0,0 +1,159 @@ +// +build !consulent + +package config + +import ( + "testing" + + "github.com/hashicorp/go-multierror" + "github.com/stretchr/testify/require" +) + +func TestBuilder_validateEnterpriseConfigKeys(t *testing.T) { + // ensure that all the enterprise configurations + type testCase struct { + config Config + keys []string + badKeys []string + check func(t *testing.T, c *Config) + } + + boolVal := true + stringVal := "string" + + cases := map[string]testCase{ + "non_voting_server": { + config: Config{ + NonVotingServer: &boolVal, + }, + keys: []string{"non_voting_server"}, + badKeys: []string{"non_voting_server"}, + }, + "segment": { + config: Config{ + SegmentName: &stringVal, + }, + keys: []string{"segment"}, + badKeys: []string{"segment"}, + }, + "segments": { + config: Config{ + Segments: []Segment{ + {Name: &stringVal}, + }, + }, + keys: []string{"segments"}, + badKeys: []string{"segments"}, + }, + "autopilot.redundancy_zone_tag": { + config: Config{ + Autopilot: Autopilot{ + RedundancyZoneTag: &stringVal, + }, + }, + keys: []string{"autopilot.redundancy_zone_tag"}, + badKeys: []string{"autopilot.redundancy_zone_tag"}, + }, + "autopilot.upgrade_version_tag": { + config: Config{ + Autopilot: Autopilot{ + UpgradeVersionTag: &stringVal, + }, + }, + keys: []string{"autopilot.upgrade_version_tag"}, + badKeys: []string{"autopilot.upgrade_version_tag"}, + }, + "autopilot.disable_upgrade_migration": { + config: Config{ + Autopilot: Autopilot{ + DisableUpgradeMigration: &boolVal, + }, + }, + keys: []string{"autopilot.disable_upgrade_migration"}, + badKeys: []string{"autopilot.disable_upgrade_migration"}, + }, + "dns_config.prefer_namespace": { + config: Config{ + DNS: DNS{ + PreferNamespace: &boolVal, + }, + }, + keys: []string{"dns_config.prefer_namespace"}, + badKeys: []string{"dns_config.prefer_namespace"}, + check: func(t *testing.T, c *Config) { + require.Nil(t, c.DNS.PreferNamespace) + }, + }, + "acl.msp_disable_bootstrap": { + config: Config{ + ACL: ACL{ + MSPDisableBootstrap: &boolVal, + }, + }, + keys: []string{"acl.msp_disable_bootstrap"}, + badKeys: []string{"acl.msp_disable_bootstrap"}, + check: func(t *testing.T, c *Config) { + require.Nil(t, c.ACL.MSPDisableBootstrap) + }, + }, + "acl.tokens.managed_service_provider": { + config: Config{ + ACL: ACL{ + Tokens: Tokens{ + ManagedServiceProvider: []ServiceProviderToken{ + { + AccessorID: &stringVal, + SecretID: &stringVal, + }, + }, + }, + }, + }, + keys: []string{"acl.tokens.managed_service_provider"}, + badKeys: []string{"acl.tokens.managed_service_provider"}, + check: func(t *testing.T, c *Config) { + require.Empty(t, c.ACL.Tokens.ManagedServiceProvider) + require.Nil(t, c.ACL.Tokens.ManagedServiceProvider) + }, + }, + "multi": { + config: Config{ + NonVotingServer: &boolVal, + SegmentName: &stringVal, + }, + keys: []string{"non_voting_server", "segment", "acl.tokens.agent_master"}, + badKeys: []string{"non_voting_server", "segment"}, + }, + } + + for name, tcase := range cases { + t.Run(name, func(t *testing.T) { + b := &Builder{} + + err := b.validateEnterpriseConfigKeys(&tcase.config, tcase.keys) + if len(tcase.badKeys) > 0 { + require.Error(t, err) + + multiErr, ok := err.(*multierror.Error) + require.True(t, ok) + + var badKeys []string + for _, e := range multiErr.Errors { + if keyErr, ok := e.(enterpriseConfigKeyError); ok { + badKeys = append(badKeys, keyErr.key) + require.Contains(t, b.Warnings, keyErr.Error()) + } + } + + require.ElementsMatch(t, tcase.badKeys, badKeys) + + if tcase.check != nil { + tcase.check(t, &tcase.config) + } + + } else { + require.NoError(t, err) + } + }) + } +} diff --git a/agent/config/config.go b/agent/config/config.go index 4da3834afd..2bcbfea1ca 100644 --- a/agent/config/config.go +++ b/agent/config/config.go @@ -34,7 +34,7 @@ func FormatFrom(name string) string { } // Parse parses a config fragment in either JSON or HCL format. -func Parse(data string, format string) (c Config, err error) { +func Parse(data string, format string) (c Config, keys []string, err error) { var raw map[string]interface{} switch format { case "json": @@ -45,7 +45,7 @@ func Parse(data string, format string) (c Config, err error) { err = fmt.Errorf("invalid format: %s", format) } if err != nil { - return Config{}, err + return Config{}, nil, err } // We want to be able to report fields which we cannot map as an @@ -136,15 +136,20 @@ func Parse(data string, format string) (c Config, err error) { Result: &c, }) if err != nil { - return Config{}, err + return Config{}, nil, err } if err := d.Decode(m); err != nil { - return Config{}, err + return Config{}, nil, err } for _, k := range md.Unused { err = multierror.Append(err, fmt.Errorf("invalid config key %s", k)) } + + // Don't check these here. The builder can emit warnings for fields it + // doesn't like + keys = md.Keys + return } @@ -245,7 +250,6 @@ type Config struct { NodeID *string `json:"node_id,omitempty" hcl:"node_id" mapstructure:"node_id"` NodeMeta map[string]string `json:"node_meta,omitempty" hcl:"node_meta" mapstructure:"node_meta"` NodeName *string `json:"node_name,omitempty" hcl:"node_name" mapstructure:"node_name"` - NonVotingServer *bool `json:"non_voting_server,omitempty" hcl:"non_voting_server" mapstructure:"non_voting_server"` Performance Performance `json:"performance,omitempty" hcl:"performance" mapstructure:"performance"` PidFile *string `json:"pid_file,omitempty" hcl:"pid_file" mapstructure:"pid_file"` Ports Ports `json:"ports,omitempty" hcl:"ports" mapstructure:"ports"` @@ -266,8 +270,6 @@ type Config struct { RetryJoinMaxAttemptsLAN *int `json:"retry_max,omitempty" hcl:"retry_max" mapstructure:"retry_max"` RetryJoinMaxAttemptsWAN *int `json:"retry_max_wan,omitempty" hcl:"retry_max_wan" mapstructure:"retry_max_wan"` RetryJoinWAN []string `json:"retry_join_wan,omitempty" hcl:"retry_join_wan" mapstructure:"retry_join_wan"` - SegmentName *string `json:"segment,omitempty" hcl:"segment" mapstructure:"segment"` - Segments []Segment `json:"segments,omitempty" hcl:"segments" mapstructure:"segments"` SerfBindAddrLAN *string `json:"serf_lan,omitempty" hcl:"serf_lan" mapstructure:"serf_lan"` SerfBindAddrWAN *string `json:"serf_wan,omitempty" hcl:"serf_wan" mapstructure:"serf_wan"` ServerMode *bool `json:"server,omitempty" hcl:"server" mapstructure:"server"` @@ -317,6 +319,13 @@ type Config struct { Version *string `json:"version,omitempty" hcl:"version" mapstructure:"version"` VersionPrerelease *string `json:"version_prerelease,omitempty" hcl:"version_prerelease" mapstructure:"version_prerelease"` + // Enterprise Only + NonVotingServer *bool `json:"non_voting_server,omitempty" hcl:"non_voting_server" mapstructure:"non_voting_server"` + // Enterprise Only + SegmentName *string `json:"segment,omitempty" hcl:"segment" mapstructure:"segment"` + // Enterprise Only + Segments []Segment `json:"segments,omitempty" hcl:"segments" mapstructure:"segments"` + // enterpriseConfig embeds fields that we only access in consul-enterprise builds EnterpriseConfig `hcl:",squash" mapstructure:",squash"` } @@ -372,13 +381,17 @@ type AdvertiseAddrsConfig struct { type Autopilot struct { CleanupDeadServers *bool `json:"cleanup_dead_servers,omitempty" hcl:"cleanup_dead_servers" mapstructure:"cleanup_dead_servers"` - DisableUpgradeMigration *bool `json:"disable_upgrade_migration,omitempty" hcl:"disable_upgrade_migration" mapstructure:"disable_upgrade_migration"` LastContactThreshold *string `json:"last_contact_threshold,omitempty" hcl:"last_contact_threshold" mapstructure:"last_contact_threshold"` MaxTrailingLogs *int `json:"max_trailing_logs,omitempty" hcl:"max_trailing_logs" mapstructure:"max_trailing_logs"` MinQuorum *uint `json:"min_quorum,omitempty" hcl:"min_quorum" mapstructure:"min_quorum"` - RedundancyZoneTag *string `json:"redundancy_zone_tag,omitempty" hcl:"redundancy_zone_tag" mapstructure:"redundancy_zone_tag"` ServerStabilizationTime *string `json:"server_stabilization_time,omitempty" hcl:"server_stabilization_time" mapstructure:"server_stabilization_time"` - UpgradeVersionTag *string `json:"upgrade_version_tag,omitempty" hcl:"upgrade_version_tag" mapstructure:"upgrade_version_tag"` + + // Enterprise Only + DisableUpgradeMigration *bool `json:"disable_upgrade_migration,omitempty" hcl:"disable_upgrade_migration" mapstructure:"disable_upgrade_migration"` + // Enterprise Only + RedundancyZoneTag *string `json:"redundancy_zone_tag,omitempty" hcl:"redundancy_zone_tag" mapstructure:"redundancy_zone_tag"` + // Enterprise Only + UpgradeVersionTag *string `json:"upgrade_version_tag,omitempty" hcl:"upgrade_version_tag" mapstructure:"upgrade_version_tag"` } // ServiceWeights defines the registration of weights used in DNS for a Service @@ -606,21 +619,23 @@ type SOA struct { } type DNS struct { - AllowStale *bool `json:"allow_stale,omitempty" hcl:"allow_stale" mapstructure:"allow_stale"` - ARecordLimit *int `json:"a_record_limit,omitempty" hcl:"a_record_limit" mapstructure:"a_record_limit"` - DisableCompression *bool `json:"disable_compression,omitempty" hcl:"disable_compression" mapstructure:"disable_compression"` - EnableTruncate *bool `json:"enable_truncate,omitempty" hcl:"enable_truncate" mapstructure:"enable_truncate"` - MaxStale *string `json:"max_stale,omitempty" hcl:"max_stale" mapstructure:"max_stale"` - NodeTTL *string `json:"node_ttl,omitempty" hcl:"node_ttl" mapstructure:"node_ttl"` - OnlyPassing *bool `json:"only_passing,omitempty" hcl:"only_passing" mapstructure:"only_passing"` - RecursorTimeout *string `json:"recursor_timeout,omitempty" hcl:"recursor_timeout" mapstructure:"recursor_timeout"` - ServiceTTL map[string]string `json:"service_ttl,omitempty" hcl:"service_ttl" mapstructure:"service_ttl"` - UDPAnswerLimit *int `json:"udp_answer_limit,omitempty" hcl:"udp_answer_limit" mapstructure:"udp_answer_limit"` - NodeMetaTXT *bool `json:"enable_additional_node_meta_txt,omitempty" hcl:"enable_additional_node_meta_txt" mapstructure:"enable_additional_node_meta_txt"` - SOA *SOA `json:"soa,omitempty" hcl:"soa" mapstructure:"soa"` - UseCache *bool `json:"use_cache,omitempty" hcl:"use_cache" mapstructure:"use_cache"` - CacheMaxAge *string `json:"cache_max_age,omitempty" hcl:"cache_max_age" mapstructure:"cache_max_age"` - EnterpriseDNSConfig `hcl:",squash" mapstructure:",squash"` + AllowStale *bool `json:"allow_stale,omitempty" hcl:"allow_stale" mapstructure:"allow_stale"` + ARecordLimit *int `json:"a_record_limit,omitempty" hcl:"a_record_limit" mapstructure:"a_record_limit"` + DisableCompression *bool `json:"disable_compression,omitempty" hcl:"disable_compression" mapstructure:"disable_compression"` + EnableTruncate *bool `json:"enable_truncate,omitempty" hcl:"enable_truncate" mapstructure:"enable_truncate"` + MaxStale *string `json:"max_stale,omitempty" hcl:"max_stale" mapstructure:"max_stale"` + NodeTTL *string `json:"node_ttl,omitempty" hcl:"node_ttl" mapstructure:"node_ttl"` + OnlyPassing *bool `json:"only_passing,omitempty" hcl:"only_passing" mapstructure:"only_passing"` + RecursorTimeout *string `json:"recursor_timeout,omitempty" hcl:"recursor_timeout" mapstructure:"recursor_timeout"` + ServiceTTL map[string]string `json:"service_ttl,omitempty" hcl:"service_ttl" mapstructure:"service_ttl"` + UDPAnswerLimit *int `json:"udp_answer_limit,omitempty" hcl:"udp_answer_limit" mapstructure:"udp_answer_limit"` + NodeMetaTXT *bool `json:"enable_additional_node_meta_txt,omitempty" hcl:"enable_additional_node_meta_txt" mapstructure:"enable_additional_node_meta_txt"` + SOA *SOA `json:"soa,omitempty" hcl:"soa" mapstructure:"soa"` + UseCache *bool `json:"use_cache,omitempty" hcl:"use_cache" mapstructure:"use_cache"` + CacheMaxAge *string `json:"cache_max_age,omitempty" hcl:"cache_max_age" mapstructure:"cache_max_age"` + + // Enterprise Only + PreferNamespace *bool `json:"prefer_namespace,omitempty" hcl:"prefer_namespace" mapstructure:"prefer_namespace"` } type HTTPConfig struct { @@ -713,18 +728,23 @@ type ACL struct { Tokens Tokens `json:"tokens,omitempty" hcl:"tokens" mapstructure:"tokens"` DisabledTTL *string `json:"disabled_ttl,omitempty" hcl:"disabled_ttl" mapstructure:"disabled_ttl"` EnableTokenPersistence *bool `json:"enable_token_persistence" hcl:"enable_token_persistence" mapstructure:"enable_token_persistence"` - EnterpriseACLConfig `hcl:",squash" mapstructure:",squash"` + + // Enterprise Only + MSPDisableBootstrap *bool `json:"msp_disable_bootstrap" hcl:"msp_disable_bootstrap" mapstructure:"msp_disable_bootstrap"` } type Tokens struct { - Master *string `json:"master,omitempty" hcl:"master" mapstructure:"master"` - Replication *string `json:"replication,omitempty" hcl:"replication" mapstructure:"replication"` - AgentMaster *string `json:"agent_master,omitempty" hcl:"agent_master" mapstructure:"agent_master"` - Default *string `json:"default,omitempty" hcl:"default" mapstructure:"default"` - Agent *string `json:"agent,omitempty" hcl:"agent" mapstructure:"agent"` + Master *string `json:"master,omitempty" hcl:"master" mapstructure:"master"` + Replication *string `json:"replication,omitempty" hcl:"replication" mapstructure:"replication"` + AgentMaster *string `json:"agent_master,omitempty" hcl:"agent_master" mapstructure:"agent_master"` + Default *string `json:"default,omitempty" hcl:"default" mapstructure:"default"` + Agent *string `json:"agent,omitempty" hcl:"agent" mapstructure:"agent"` + + // Enterprise Only ManagedServiceProvider []ServiceProviderToken `json:"managed_service_provider,omitempty" hcl:"managed_service_provider" mapstructure:"managed_service_provider"` } +// ServiceProviderToken groups an accessor and secret for a service provider token. Enterprise Only type ServiceProviderToken struct { AccessorID *string `json:"accessor_id,omitempty" hcl:"accessor_id" mapstructure:"accessor_id"` SecretID *string `json:"secret_id,omitempty" hcl:"secret_id" mapstructure:"secret_id"` diff --git a/agent/config/config_oss.go b/agent/config/config_oss.go index c6d44b908d..558815ce9c 100644 --- a/agent/config/config_oss.go +++ b/agent/config/config_oss.go @@ -13,9 +13,3 @@ type EnterpriseMeta struct{} func (_ *EnterpriseMeta) ToStructs() structs.EnterpriseMeta { return *structs.DefaultEnterpriseMeta() } - -// EnterpriseDNSConfig OSS stub -type EnterpriseDNSConfig struct{} - -// EnterpriseACLConfig OSS stub -type EnterpriseACLConfig struct{} diff --git a/agent/config/default.go b/agent/config/default.go index a04b0cdd85..ddb4407750 100644 --- a/agent/config/default.go +++ b/agent/config/default.go @@ -12,7 +12,7 @@ import ( func DefaultRPCProtocol() (int, error) { src := DefaultSource() - c, err := Parse(src.Data, src.Format) + c, _, err := Parse(src.Data, src.Format) if err != nil { return 0, fmt.Errorf("Error parsing default config: %s", err) } diff --git a/agent/config/runtime_oss_test.go b/agent/config/runtime_oss_test.go index 7371429923..72cee4d261 100644 --- a/agent/config/runtime_oss_test.go +++ b/agent/config/runtime_oss_test.go @@ -11,3 +11,13 @@ var entFullDNSJSONConfig = `` var entFullDNSHCLConfig = `` var entFullRuntimeConfig = EnterpriseRuntimeConfig{} + +var enterpriseNonVotingServerWarnings []string = []string{enterpriseConfigKeyError{key: "non_voting_server"}.Error()} + +var enterpriseConfigKeyWarnings []string + +func init() { + for k, _ := range enterpriseConfigMap { + enterpriseConfigKeyWarnings = append(enterpriseConfigKeyWarnings, enterpriseConfigKeyError{key: k}.Error()) + } +} diff --git a/agent/config/runtime_test.go b/agent/config/runtime_test.go index 48e81ebbcf..767d94f36d 100644 --- a/agent/config/runtime_test.go +++ b/agent/config/runtime_test.go @@ -609,6 +609,7 @@ func TestConfigFlagsAndEdgecases(t *testing.T) { rt.NonVotingServer = true rt.DataDir = dataDir }, + warns: enterpriseNonVotingServerWarnings, }, { desc: "-pid-file", @@ -3915,6 +3916,7 @@ func TestFullConfig(t *testing.T) { "role_ttl": "9876s", "token_ttl": "3321s", "enable_token_replication" : true, + "msp_disable_bootstrap": true, "tokens" : { "master" : "8a19ac27", "agent_master" : "64fd0e08", @@ -4110,7 +4112,8 @@ func TestFullConfig(t *testing.T) { }, "udp_answer_limit": 29909, "use_cache": true, - "cache_max_age": "5m"` + entFullDNSJSONConfig + ` + "cache_max_age": "5m", + "prefer_namespace": true }, "enable_acl_replication": true, "enable_agent_tls_for_checks": true, @@ -4546,6 +4549,7 @@ func TestFullConfig(t *testing.T) { role_ttl = "9876s" token_ttl = "3321s" enable_token_replication = true + msp_disable_bootstrap = true tokens = { master = "8a19ac27", agent_master = "64fd0e08", @@ -4743,7 +4747,7 @@ func TestFullConfig(t *testing.T) { udp_answer_limit = 29909 use_cache = true cache_max_age = "5m" - ` + entFullDNSHCLConfig + ` + prefer_namespace = true } enable_acl_replication = true enable_agent_tls_for_checks = true @@ -5885,6 +5889,8 @@ func TestFullConfig(t *testing.T) { `bootstrap_expect > 0: expecting 53 servers`, } + warns = append(warns, enterpriseConfigKeyWarnings...) + // ensure that all fields are set to unique non-zero values // todo(fs): This currently fails since ServiceDefinition.Check is not used // todo(fs): not sure on how to work around this. Possible options are: @@ -5947,9 +5953,7 @@ func TestFullConfig(t *testing.T) { } // check the warnings - if got, want := b.Warnings, warns; !verify.Values(t, "warnings", got, want) { - t.FailNow() - } + require.ElementsMatch(t, warns, b.Warnings, "Warnings: %v", b.Warnings) }) } } diff --git a/agent/config/segment_oss_test.go b/agent/config/segment_oss_test.go index 9de4826c51..88952e23bc 100644 --- a/agent/config/segment_oss_test.go +++ b/agent/config/segment_oss_test.go @@ -22,6 +22,9 @@ func TestSegments(t *testing.T) { json: []string{`{ "server": true, "segment": "a" }`}, hcl: []string{` server = true segment = "a" `}, err: `Network segments are not supported in this version of Consul`, + warns: []string{ + enterpriseConfigKeyError{key: "segment"}.Error(), + }, }, { desc: "segment port must be set", @@ -31,6 +34,9 @@ func TestSegments(t *testing.T) { json: []string{`{ "segments":[{ "name":"x" }] }`}, hcl: []string{`segments = [{ name = "x" }]`}, err: `Port for segment "x" cannot be <= 0`, + warns: []string{ + enterpriseConfigKeyError{key: "segments"}.Error(), + }, }, { desc: "segments not in OSS", @@ -40,6 +46,9 @@ func TestSegments(t *testing.T) { json: []string{`{ "segments":[{ "name":"x", "port": 123 }] }`}, hcl: []string{`segments = [{ name = "x" port = 123 }]`}, err: `Network segments are not supported in this version of Consul`, + warns: []string{ + enterpriseConfigKeyError{key: "segments"}.Error(), + }, }, }