status-im
/
consul
mirror of https://github.com/status-im/consul.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Add tests for node meta in prepared queries and update docs

This commit is contained in:
Kyle Havlovitz 2017-01-23 18:53:45 -05:00
parent bcf770f811
commit 3f3d7f9891
No known key found for this signature in database
GPG Key ID: 8A5E6B173056AD6C
14 changed files with 181 additions and 142 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
api/prepared_query.go
View File

@ -43,6 +43,11 @@ type ServiceQuery struct {
// this list it must be present. If the tag is preceded with "!" then
// it is disallowed.
Tags []string
// NodeMeta is a map of required node metadata fields. If a key/value
// pair is in this map it must be present on the node in order for the
// service entry to be returned.
NodeMeta map[string]string
}
// QueryTemplate carries the arguments for creating a templated query.

4
api/prepared_query_test.go
View File

@ -20,6 +20,7 @@ func TestPreparedQuery(t *testing.T) {
TaggedAddresses: map[string]string{
"wan": "127.0.0.1",
},
NodeMeta: map[string]string{"somekey": "somevalue"},
Service: &AgentService{
ID: "redis1",
Service: "redis",
@ -47,7 +48,8 @@ func TestPreparedQuery(t *testing.T) {
def := &PreparedQueryDefinition{
Name: "test",
Service: ServiceQuery{
Service: "redis",
Service: "redis",
NodeMeta: map[string]string{"somekey": "somevalue"},
},
}

50
command/agent/agent.go
View File

@ -42,26 +42,11 @@ const (
"but no reason was provided. This is a default message."
defaultServiceMaintReason = "Maintenance mode is enabled for this " +
"service, but no reason was provided. This is a default message."
// The meta key prefix reserved for Consul's internal use
metaKeyReservedPrefix = "consul-"
// The maximum number of metadata key pairs allowed to be registered
metaMaxKeyPairs = 64
// The maximum allowed length of a metadata key
metaKeyMaxLength = 128
// The maximum allowed length of a metadata value
metaValueMaxLength = 512
)
var (
// dnsNameRe checks if a name or tag is dns-compatible.
dnsNameRe = regexp.MustCompile(`^[a-zA-Z0-9\-]+$`)
// metaKeyFormat checks if a metadata key string is valid
metaKeyFormat = regexp.MustCompile(`^[a-zA-Z0-9_-]+$`).MatchString
)
/*
@ -1720,41 +1705,6 @@ func parseMetaPair(raw string) (string, string) {
}
}
// validateMeta validates a set of key/value pairs from the agent config
func validateMetadata(meta map[string]string) error {
if len(meta) > metaMaxKeyPairs {
return fmt.Errorf("Node metadata cannot contain more than %d key/value pairs", metaMaxKeyPairs)
}
for key, value := range meta {
if err := validateMetaPair(key, value); err != nil {
return fmt.Errorf("Couldn't load metadata pair ('%s', '%s'): %s", key, value, err)
}
}
return nil
}
// validateMetaPair checks that the given key/value pair is in a valid format
func validateMetaPair(key, value string) error {
if key == "" {
return fmt.Errorf("Key cannot be blank")
}
if !metaKeyFormat(key) {
return fmt.Errorf("Key contains invalid characters")
}
if len(key) > metaKeyMaxLength {
return fmt.Errorf("Key is too long (limit: %d characters)", metaKeyMaxLength)
}
if strings.HasPrefix(key, metaKeyReservedPrefix) {
return fmt.Errorf("Key prefix '%s' is reserved for internal use", metaKeyReservedPrefix)
}
if len(value) > metaValueMaxLength {
return fmt.Errorf("Value is too long (limit: %d characters)", metaValueMaxLength)
}
return nil
}
// unloadMetadata resets the local metadata state
func (a *Agent) unloadMetadata() {
a.state.Lock()

64
command/agent/agent_test.go
View File

@ -19,7 +19,6 @@ import (
"github.com/hashicorp/consul/logger"
"github.com/hashicorp/consul/testutil"
"github.com/hashicorp/raft"
"strings"
)
const (
@ -1852,69 +1851,6 @@ func TestAgent_purgeCheckState(t *testing.T) {
}
}
func TestAgent_metadata(t *testing.T) {
// Load a valid set of key/value pairs
meta := map[string]string{
"key1": "value1",
"key2": "value2",
}
// Should succeed
if err := validateMetadata(meta); err != nil {
t.Fatalf("err: %s", err)
}
// Should get error
meta = map[string]string{
"": "value1",
}
if err := validateMetadata(meta); !strings.Contains(err.Error(), "Couldn't load metadata pair") {
t.Fatalf("should have failed")
}
// Should get error
meta = make(map[string]string)
for i := 0; i < metaMaxKeyPairs+1; i++ {
meta[string(i)] = "value"
}
if err := validateMetadata(meta); !strings.Contains(err.Error(), "cannot contain more than") {
t.Fatalf("should have failed")
}
}
func TestAgent_validateMetaPair(t *testing.T) {
longKey := strings.Repeat("a", metaKeyMaxLength+1)
longValue := strings.Repeat("b", metaValueMaxLength+1)
pairs := []struct {
Key string
Value string
Error string
}{
// valid pair
{"key", "value", ""},
// invalid, blank key
{"", "value", "cannot be blank"},
// allowed special chars in key name
{"k_e-y", "value", ""},
// disallowed special chars in key name
{"(%key&)", "value", "invalid characters"},
// key too long
{longKey, "value", "Key is too long"},
// reserved prefix
{metaKeyReservedPrefix + "key", "value", "reserved for internal use"},
// value too long
{"key", longValue, "Value is too long"},
}
for _, pair := range pairs {
err := validateMetaPair(pair.Key, pair.Value)
if pair.Error == "" && err != nil {
t.Fatalf("should have succeeded: %v, %v", pair, err)
} else if pair.Error != "" && !strings.Contains(err.Error(), pair.Error) {
t.Fatalf("should have failed: %v, %v", pair, err)
}
}
}
func TestAgent_GetCoordinate(t *testing.T) {
check := func(server bool) {
config := nextConfig()

3
command/agent/command.go
View File

@ -31,6 +31,7 @@ import (
"github.com/aws/aws-sdk-go/aws/ec2metadata"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/hashicorp/consul/consul/structs"
"github.com/hashicorp/consul/lib"
"github.com/hashicorp/consul/logger"
"github.com/hashicorp/consul/watch"
@ -396,7 +397,7 @@ func (c *Command) readConfig() *Config {
}
// Verify the node metadata entries are valid
if err := validateMetadata(config.Meta); err != nil {
if err := structs.ValidateMetadata(config.Meta); err != nil {
c.Ui.Error(fmt.Sprintf("Failed to parse node metadata: %v", err))
return nil
}

4
command/agent/prepared_query_endpoint_test.go
View File

@ -90,6 +90,7 @@ func TestPreparedQuery_Create(t *testing.T) {
},
OnlyPassing: true,
Tags: []string{"foo", "bar"},
NodeMeta: map[string]string{"somekey": "somevalue"},
},
DNS: structs.QueryDNSOptions{
TTL: "10s",
@ -120,6 +121,7 @@ func TestPreparedQuery_Create(t *testing.T) {
},
"OnlyPassing": true,
"Tags": []string{"foo", "bar"},
"NodeMeta": map[string]string{"somekey": "somevalue"},
},
"DNS": map[string]interface{}{
"TTL": "10s",
@ -645,6 +647,7 @@ func TestPreparedQuery_Update(t *testing.T) {
},
OnlyPassing: true,
Tags: []string{"foo", "bar"},
NodeMeta: map[string]string{"somekey": "somevalue"},
},
DNS: structs.QueryDNSOptions{
TTL: "10s",
@ -676,6 +679,7 @@ func TestPreparedQuery_Update(t *testing.T) {
},
"OnlyPassing": true,
"Tags": []string{"foo", "bar"},
"NodeMeta": map[string]string{"somekey": "somevalue"},
},
"DNS": map[string]interface{}{
"TTL": "10s",

12
consul/prepared_query/template_test.go
View File

@ -39,9 +39,9 @@ var (
"${match(2)}",
},
NodeMeta: map[string]string{
"${name.full}": "${name.prefix}",
"${name.suffix}": "${match(0)}",
"${match(1)}": "${match(2)}",
"foo": "${name.prefix}",
"bar": "${match(0)}",
"baz": "${match(1)}",
},
},
}
@ -227,7 +227,7 @@ func TestTemplate_Render(t *testing.T) {
"${match(4)}",
"${40 + 2}",
},
NodeMeta: map[string]string{"${match(1)}": "${match(2)}"},
NodeMeta: map[string]string{"foo": "${match(1)}"},
},
}
ct, err := Compile(query)
@ -258,7 +258,7 @@ func TestTemplate_Render(t *testing.T) {
"",
"42",
},
NodeMeta: map[string]string{"hello": "foo"},
NodeMeta: map[string]string{"foo": "hello"},
},
}
if !reflect.DeepEqual(actual, expected) {
@ -289,7 +289,7 @@ func TestTemplate_Render(t *testing.T) {
"",
"42",
},
NodeMeta: map[string]string{"": ""},
NodeMeta: map[string]string{"foo": ""},
},
}
if !reflect.DeepEqual(actual, expected) {

13
consul/prepared_query/walk.go
View File

@ -35,23 +35,18 @@ func visit(path string, v reflect.Value, t reflect.Type, fn visitor) error {
}
}
case reflect.Map:
for i, key := range v.MapKeys() {
for _, key := range v.MapKeys() {
value := v.MapIndex(key)
newKey := reflect.New(key.Type()).Elem()
newKey.SetString(key.String())
newValue := reflect.New(value.Type()).Elem()
newValue.SetString(value.String())
if err := visit(fmt.Sprintf("%s.keys[%d]", path, i), newKey, newKey.Type(), fn); err != nil {
return err
}
if err := visit(fmt.Sprintf("%s[%s]", path, key.String()), newValue, newValue.Type(), fn); err != nil {
return err
}
// delete the old entry and add the new one
v.SetMapIndex(key, reflect.Value{})
v.SetMapIndex(newKey, newValue)
// overwrite the entry in case it was modified by the callback
v.SetMapIndex(key, newValue)
}
}
return nil

14
consul/prepared_query/walk_test.go
View File

@ -6,6 +6,7 @@ import (
"testing"
"github.com/hashicorp/consul/consul/structs"
"sort"
)
func TestWalk_ServiceQuery(t *testing.T) {
@ -20,25 +21,26 @@ func TestWalk_ServiceQuery(t *testing.T) {
Failover: structs.QueryDatacenterOptions{
Datacenters: []string{"dc1", "dc2"},
},
Near: "_agent",
Tags: []string{"tag1", "tag2", "tag3"},
NodeMeta: map[string]string{"role": "server"},
Near: "_agent",
Tags: []string{"tag1", "tag2", "tag3"},
NodeMeta: map[string]string{"foo": "bar", "role": "server"},
}
if err := walk(service, fn); err != nil {
t.Fatalf("err: %v", err)
}
expected := []string{
".Service:the-service",
".Failover.Datacenters[0]:dc1",
".Failover.Datacenters[1]:dc2",
".Near:_agent",
".NodeMeta[foo]:bar",
".NodeMeta[role]:server",
".Service:the-service",
".Tags[0]:tag1",
".Tags[1]:tag2",
".Tags[2]:tag3",
".NodeMeta.keys[0]:role",
".NodeMeta[role]:server",
}
sort.Strings(actual)
if !reflect.DeepEqual(actual, expected) {
t.Fatalf("bad: %#v", actual)
}

5
consul/prepared_query_endpoint.go
View File

@ -178,6 +178,11 @@ func parseService(svc *structs.ServiceQuery) error {
return fmt.Errorf("Bad NearestN '%d', must be >= 0", svc.Failover.NearestN)
}
// Make sure the metadata filters are valid
if err := structs.ValidateMetadata(svc.NodeMeta); err != nil {
return err
}
// We skip a few fields:
// - There's no validation for Datacenters; we skip any unknown entries
// at execution time.

11
consul/prepared_query_endpoint_test.go
View File

@ -604,6 +604,17 @@ func TestPreparedQuery_parseQuery(t *testing.T) {
if err := parseQuery(query, version8); err != nil {
t.Fatalf("err: %v", err)
}
query.Service.NodeMeta = map[string]string{"": "somevalue"}
err = parseQuery(query, version8)
if err == nil || !strings.Contains(err.Error(), "cannot be blank") {
t.Fatalf("bad: %v", err)
}
query.Service.NodeMeta = map[string]string{"somekey": "somevalue"}
if err := parseQuery(query, version8); err != nil {
t.Fatalf("err: %v", err)
}
}
}

56
consul/structs/structs.go
View File

@ -11,6 +11,8 @@ import (
"github.com/hashicorp/consul/types"
"github.com/hashicorp/go-msgpack/codec"
"github.com/hashicorp/serf/coordinate"
"regexp"
"strings"
)
var (
@ -67,6 +69,25 @@ const (
ServiceMaintPrefix = "_service_maintenance:"
)
const (
// The meta key prefix reserved for Consul's internal use
metaKeyReservedPrefix = "consul-"
// The maximum number of metadata key pairs allowed to be registered
metaMaxKeyPairs = 64
// The maximum allowed length of a metadata key
metaKeyMaxLength = 128
// The maximum allowed length of a metadata value
metaValueMaxLength = 512
)
var (
// metaKeyFormat checks if a metadata key string is valid
metaKeyFormat = regexp.MustCompile(`^[a-zA-Z0-9_-]+$`).MatchString
)
func ValidStatus(s string) bool {
return s == HealthPassing ||
s == HealthWarning ||
@ -289,6 +310,41 @@ type Node struct {
}
type Nodes []*Node
// ValidateMeta validates a set of key/value pairs from the agent config
func ValidateMetadata(meta map[string]string) error {
if len(meta) > metaMaxKeyPairs {
return fmt.Errorf("Node metadata cannot contain more than %d key/value pairs", metaMaxKeyPairs)
}
for key, value := range meta {
if err := validateMetaPair(key, value); err != nil {
return fmt.Errorf("Couldn't load metadata pair ('%s', '%s'): %s", key, value, err)
}
}
return nil
}
// validateMetaPair checks that the given key/value pair is in a valid format
func validateMetaPair(key, value string) error {
if key == "" {
return fmt.Errorf("Key cannot be blank")
}
if !metaKeyFormat(key) {
return fmt.Errorf("Key contains invalid characters")
}
if len(key) > metaKeyMaxLength {
return fmt.Errorf("Key is too long (limit: %d characters)", metaKeyMaxLength)
}
if strings.HasPrefix(key, metaKeyReservedPrefix) {
return fmt.Errorf("Key prefix '%s' is reserved for internal use", metaKeyReservedPrefix)
}
if len(value) > metaValueMaxLength {
return fmt.Errorf("Value is too long (limit: %d characters)", metaValueMaxLength)
}
return nil
}
// SatisfiesMetaFilters returns true if the metadata map contains the given filters
func SatisfiesMetaFilters(meta map[string]string, filters map[string]string) bool {
for key, value := range filters {

63
consul/structs/structs_test.go
View File

@ -492,3 +492,66 @@ func TestStructs_DirEntry_Clone(t *testing.T) {
t.Fatalf("clone wasn't independent of the original")
}
}
func TestStructs_ValidateMetadata(t *testing.T) {
// Load a valid set of key/value pairs
meta := map[string]string{
"key1": "value1",
"key2": "value2",
}
// Should succeed
if err := ValidateMetadata(meta); err != nil {
t.Fatalf("err: %s", err)
}
// Should get error
meta = map[string]string{
"": "value1",
}
if err := ValidateMetadata(meta); !strings.Contains(err.Error(), "Couldn't load metadata pair") {
t.Fatalf("should have failed")
}
// Should get error
meta = make(map[string]string)
for i := 0; i < metaMaxKeyPairs+1; i++ {
meta[string(i)] = "value"
}
if err := ValidateMetadata(meta); !strings.Contains(err.Error(), "cannot contain more than") {
t.Fatalf("should have failed")
}
}
func TestStructs_validateMetaPair(t *testing.T) {
longKey := strings.Repeat("a", metaKeyMaxLength+1)
longValue := strings.Repeat("b", metaValueMaxLength+1)
pairs := []struct {
Key string
Value string
Error string
}{
// valid pair
{"key", "value", ""},
// invalid, blank key
{"", "value", "cannot be blank"},
// allowed special chars in key name
{"k_e-y", "value", ""},
// disallowed special chars in key name
{"(%key&)", "value", "invalid characters"},
// key too long
{longKey, "value", "Key is too long"},
// reserved prefix
{metaKeyReservedPrefix + "key", "value", "reserved for internal use"},
// value too long
{"key", longValue, "Value is too long"},
}
for _, pair := range pairs {
err := validateMetaPair(pair.Key, pair.Value)
if pair.Error == "" && err != nil {
t.Fatalf("should have succeeded: %v, %v", pair, err)
} else if pair.Error != "" && !strings.Contains(err.Error(), pair.Error) {
t.Fatalf("should have failed: %v, %v", pair, err)
}
}
}

19
website/source/docs/agent/http/query.html.markdown
View File

@ -78,7 +78,8 @@ query, like this example:
},
"Near": "node1",
"OnlyPassing": false,
"Tags": ["primary", "!experimental"]
"Tags": ["primary", "!experimental"],
"NodeMeta": {"instance_type": "m3.large"}
},
"DNS": {
"TTL": "10s"
@ -162,6 +163,10 @@ to pass the tag filter it must have *all* of the required tags, and *none* of th
excluded tags (prefixed with `!`). The default value is an empty list, which does
no tag filtering.
`NodeMeta` provides a list of user-defined key/value pairs that will be used for
filtering the query results to nodes with the given metadata values present. This
was added in Consul 0.7.3.
`TTL` in the `DNS` structure is a duration string that can use `s` as a
suffix for seconds. It controls how the TTL is set when query results are served
over DNS. If this isn't specified, then the Consul agent configuration for the given
@ -199,7 +204,8 @@ and features. Here's an example:
"Datacenters": ["dc1", "dc2"]
},
"OnlyPassing": true,
"Tags": ["${match(2)}"]
"Tags": ["${match(2)}"],
"NodeMeta": {"instance_type": "m3.large"}
}
}
```
@ -303,7 +309,8 @@ This returns a JSON list of prepared queries, which looks like:
"Datacenters": ["dc1", "dc2"]
},
"OnlyPassing": false,
"Tags": ["primary", "!experimental"]
"Tags": ["primary", "!experimental"],
"NodeMeta": {"instance_type": "m3.large"}
},
"DNS": {
"TTL": "10s"
@ -407,7 +414,8 @@ a JSON body will be returned like this:
"TaggedAddresses": {
"lan": "10.1.10.12",
"wan": "10.1.10.12"
}
},
"NodeMeta": {"instance_type": "m3.large"}
},
"Service": {
"ID": "redis",
@ -499,7 +507,8 @@ a JSON body will be returned like this:
"Datacenters": ["dc1", "dc2"]
},
"OnlyPassing": true,
"Tags": ["primary"]
"Tags": ["primary"],
"NodeMeta": {"instance_type": "m3.large"}
}
}
```