Add agent.segment interpolation to prepared queries

agent/config.go

@@ -357,8 +357,7 @@ type NetworkSegment struct {
 	Name string `mapstructure:"name"`
 
 	// Bind is the bind address for this segment.
-	Bind      string   `mapstructure:"bind"`
+	Bind string `mapstructure:"bind"`
-	BindAddrs []string `mapstructure:"-"`
 
 	// Port is the port for this segment.
 	Port int `mapstructure:"port"`
@@ -1467,7 +1466,7 @@ func DecodeConfig(r io.Reader) (*Config, error) {
 	}
 
 	// Validate node meta fields
-	if err := structs.ValidateMetadata(result.Meta); err != nil {
+	if err := structs.ValidateMetadata(result.Meta, false); err != nil {
 		return nil, fmt.Errorf("Failed to parse node metadata: %v", err)
 	}
 

agent/config_test.go

@@ -597,11 +597,10 @@ func TestDecodeConfig(t *testing.T) {
 			c:  &Config{Segment: "thing"},
 		},
 		{
-			in: `{"server": true, "segments":[{"name": "alpha", "bind": "127.0.0.1", "port": 1234, "rpc_listener": true, "advertise": "1.1.1.1"}]}`,
+			in: `{"segments":[{"name": "alpha", "bind": "127.0.0.1", "port": 1234, "rpc_listener": true, "advertise": "1.1.1.1"}]}`,
-			c: &Config{Server: true, Segments: []NetworkSegment{{
+			c: &Config{Segments: []NetworkSegment{{
 				Name:        "alpha",
 				Bind:        "127.0.0.1",
-				BindAddrs:   []string{"127.0.0.1"},
 				Port:        1234,
 				RPCListener: true,
 				Advertise:   "1.1.1.1",

agent/consul/prepared_query/template.go

@@ -89,7 +89,7 @@ func Compile(query *structs.PreparedQuery) (*CompiledTemplate, error) {
 	// prefix it will be expected to run with. The results might not make
 	// sense and create a valid service to lookup, but it should render
 	// without any errors.
-	if _, err = ct.Render(ct.query.Name); err != nil {
+	if _, err = ct.Render(ct.query.Name, structs.QuerySource{}); err != nil {
 		return nil, err
 	}
 
@@ -99,7 +99,7 @@ func Compile(query *structs.PreparedQuery) (*CompiledTemplate, error) {
 // Render takes a compiled template and renders it for the given name. For
 // example, if the user looks up foobar.query.consul via DNS then we will call
 // this function with "foobar" on the compiled template.
-func (ct *CompiledTemplate) Render(name string) (*structs.PreparedQuery, error) {
+func (ct *CompiledTemplate) Render(name string, source structs.QuerySource) (*structs.PreparedQuery, error) {
 	// Make it "safe" to render a default structure.
 	if ct == nil {
 		return nil, fmt.Errorf("Cannot render an uncompiled template")
@@ -156,6 +156,10 @@ func (ct *CompiledTemplate) Render(name string) (*structs.PreparedQuery, error)
 					Type:  ast.TypeString,
 					Value: strings.TrimPrefix(name, query.Name),
 				},
+				"agent.segment": ast.Variable{
+					Type:  ast.TypeString,
+					Value: source.Segment,
+				},
 			},
 			FuncMap: map[string]ast.Function{
 				"match": match,

agent/consul/prepared_query/template_test.go

@@ -29,6 +29,7 @@ var (
 					"${match(0)}",
 					"${match(1)}",
 					"${match(2)}",
+					"${agent.segment}",
 				},
 			},
 			Tags: []string{
@@ -38,11 +39,13 @@ var (
 				"${match(0)}",
 				"${match(1)}",
 				"${match(2)}",
+				"${agent.segment}",
 			},
 			NodeMeta: map[string]string{
 				"foo": "${name.prefix}",
 				"bar": "${match(0)}",
 				"baz": "${match(1)}",
+				"zoo": "${agent.segment}",
 			},
 		},
 	}
@@ -83,7 +86,7 @@ func renderBench(b *testing.B, query *structs.PreparedQuery) {
 	}
 
 	for i := 0; i < b.N; i++ {
-		_, err := compiled.Render("hello-bench-mark")
+		_, err := compiled.Render("hello-bench-mark", structs.QuerySource{})
 		if err != nil {
 			b.Fatalf("err: %v", err)
 		}
@@ -121,7 +124,7 @@ func TestTemplate_Compile(t *testing.T) {
 	query.Template.Type = structs.QueryTemplateTypeNamePrefixMatch
 	query.Template.Regexp = "^(hello)there$"
 	query.Service.Service = "${name.full}"
-	query.Service.Tags = []string{"${match(1)}"}
+	query.Service.Tags = []string{"${match(1)}", "${agent.segment}"}
 	backup, err := copystructure.Copy(query)
 	if err != nil {
 		t.Fatalf("err: %v", err)
@@ -135,7 +138,7 @@ func TestTemplate_Compile(t *testing.T) {
 	}
 
 	// Do a sanity check render on it.
-	actual, err := ct.Render("hellothere")
+	actual, err := ct.Render("hellothere", structs.QuerySource{Segment: "segment-foo"})
 	if err != nil {
 		t.Fatalf("err: %v", err)
 	}
@@ -150,6 +153,7 @@ func TestTemplate_Compile(t *testing.T) {
 			Service: "hellothere",
 			Tags: []string{
 				"hello",
+				"segment-foo",
 			},
 		},
 	}
@@ -201,7 +205,7 @@ func TestTemplate_Render(t *testing.T) {
 			t.Fatalf("err: %v", err)
 		}
 
-		actual, err := ct.Render("unused")
+		actual, err := ct.Render("unused", structs.QuerySource{})
 		if err != nil {
 			t.Fatalf("err: %v", err)
 		}
@@ -218,7 +222,7 @@ func TestTemplate_Render(t *testing.T) {
 			Regexp: "^(.*?)-(.*?)-(.*)$",
 		},
 		Service: structs.ServiceQuery{
-			Service: "${name.prefix} xxx ${name.full} xxx ${name.suffix}",
+			Service: "${name.prefix} xxx ${name.full} xxx ${name.suffix} xxx ${agent.segment}",
 			Tags: []string{
 				"${match(-1)}",
 				"${match(0)}",
@@ -238,7 +242,7 @@ func TestTemplate_Render(t *testing.T) {
 
 	// Run a case that matches the regexp.
 	{
-		actual, err := ct.Render("hello-foo-bar-none")
+		actual, err := ct.Render("hello-foo-bar-none", structs.QuerySource{Segment: "segment-bar"})
 		if err != nil {
 			t.Fatalf("err: %v", err)
 		}
@@ -249,7 +253,7 @@ func TestTemplate_Render(t *testing.T) {
 				Regexp: "^(.*?)-(.*?)-(.*)$",
 			},
 			Service: structs.ServiceQuery{
-				Service: "hello- xxx hello-foo-bar-none xxx foo-bar-none",
+				Service: "hello- xxx hello-foo-bar-none xxx foo-bar-none xxx segment-bar",
 				Tags: []string{
 					"",
 					"hello-foo-bar-none",
@@ -269,7 +273,7 @@ func TestTemplate_Render(t *testing.T) {
 
 	// Run a case that doesn't match the regexp
 	{
-		actual, err := ct.Render("hello-nope")
+		actual, err := ct.Render("hello-nope", structs.QuerySource{Segment: "segment-bar"})
 		if err != nil {
 			t.Fatalf("err: %v", err)
 		}
@@ -280,7 +284,7 @@ func TestTemplate_Render(t *testing.T) {
 				Regexp: "^(.*?)-(.*?)-(.*)$",
 			},
 			Service: structs.ServiceQuery{
-				Service: "hello- xxx hello-nope xxx nope",
+				Service: "hello- xxx hello-nope xxx nope xxx segment-bar",
 				Tags: []string{
 					"",
 					"",
@@ -307,7 +311,7 @@ func TestTemplate_Render(t *testing.T) {
 			RemoveEmptyTags: true,
 		},
 		Service: structs.ServiceQuery{
-			Service: "${name.prefix} xxx ${name.full} xxx ${name.suffix}",
+			Service: "${name.prefix} xxx ${name.full} xxx ${name.suffix} xxx ${agent.segment}",
 			Tags: []string{
 				"${match(-1)}",
 				"${match(0)}",
@@ -326,7 +330,7 @@ func TestTemplate_Render(t *testing.T) {
 
 	// Run a case that matches the regexp, removing empty tags.
 	{
-		actual, err := ct.Render("hello-foo-bar-none")
+		actual, err := ct.Render("hello-foo-bar-none", structs.QuerySource{Segment: "segment-baz"})
 		if err != nil {
 			t.Fatalf("err: %v", err)
 		}
@@ -338,7 +342,7 @@ func TestTemplate_Render(t *testing.T) {
 				RemoveEmptyTags: true,
 			},
 			Service: structs.ServiceQuery{
-				Service: "hello- xxx hello-foo-bar-none xxx foo-bar-none",
+				Service: "hello- xxx hello-foo-bar-none xxx foo-bar-none xxx segment-baz",
 				Tags: []string{
 					"hello-foo-bar-none",
 					"hello",
@@ -355,7 +359,7 @@ func TestTemplate_Render(t *testing.T) {
 
 	// Run a case that doesn't match the regexp, removing empty tags.
 	{
-		actual, err := ct.Render("hello-nope")
+		actual, err := ct.Render("hello-nope", structs.QuerySource{Segment: "segment-baz"})
 		if err != nil {
 			t.Fatalf("err: %v", err)
 		}
@@ -367,7 +371,7 @@ func TestTemplate_Render(t *testing.T) {
 				RemoveEmptyTags: true,
 			},
 			Service: structs.ServiceQuery{
-				Service: "hello- xxx hello-nope xxx nope",
+				Service: "hello- xxx hello-nope xxx nope xxx segment-baz",
 				Tags: []string{
 					"42",
 				},

agent/consul/prepared_query_endpoint.go

@@ -182,7 +182,7 @@ func parseService(svc *structs.ServiceQuery) error {
 	}
 
 	// Make sure the metadata filters are valid
-	if err := structs.ValidateMetadata(svc.NodeMeta); err != nil {
+	if err := structs.ValidateMetadata(svc.NodeMeta, true); err != nil {
 		return err
 	}
 
@@ -298,7 +298,7 @@ func (p *PreparedQuery) Explain(args *structs.PreparedQueryExecuteRequest,
 
 	// Try to locate the query.
 	state := p.srv.fsm.State()
-	_, query, err := state.PreparedQueryResolve(args.QueryIDOrName)
+	_, query, err := state.PreparedQueryResolve(args.QueryIDOrName, args.Agent)
 	if err != nil {
 		return err
 	}
@@ -345,7 +345,7 @@ func (p *PreparedQuery) Execute(args *structs.PreparedQueryExecuteRequest,
 
 	// Try to locate the query.
 	state := p.srv.fsm.State()
-	_, query, err := state.PreparedQueryResolve(args.QueryIDOrName)
+	_, query, err := state.PreparedQueryResolve(args.QueryIDOrName, args.Agent)
 	if err != nil {
 		return err
 	}

agent/consul/state/prepared_query.go

@@ -256,7 +256,7 @@ func (s *Store) PreparedQueryGet(ws memdb.WatchSet, queryID string) (uint64, *st
 // PreparedQueryResolve returns the given prepared query by looking up an ID or
 // Name. If the query was looked up by name and it's a template, then the
 // template will be rendered before it is returned.
-func (s *Store) PreparedQueryResolve(queryIDOrName string) (uint64, *structs.PreparedQuery, error) {
+func (s *Store) PreparedQueryResolve(queryIDOrName string, source structs.QuerySource) (uint64, *structs.PreparedQuery, error) {
 	tx := s.db.Txn(false)
 	defer tx.Abort()
 
@@ -293,7 +293,7 @@ func (s *Store) PreparedQueryResolve(queryIDOrName string) (uint64, *structs.Pre
 	prep := func(wrapped interface{}) (uint64, *structs.PreparedQuery, error) {
 		wrapper := wrapped.(*queryWrapper)
 		if prepared_query.IsTemplate(wrapper.PreparedQuery) {
-			render, err := wrapper.ct.Render(queryIDOrName)
+			render, err := wrapper.ct.Render(queryIDOrName, source)
 			if err != nil {
 				return idx, nil, err
 			}

agent/consul/state/prepared_query_test.go

@@ -554,7 +554,7 @@ func TestStateStore_PreparedQueryResolve(t *testing.T) {
 
 	// Try to lookup a query that's not there using something that looks
 	// like a real ID.
-	idx, actual, err := s.PreparedQueryResolve(query.ID)
+	idx, actual, err := s.PreparedQueryResolve(query.ID, structs.QuerySource{})
 	if err != nil {
 		t.Fatalf("err: %s", err)
 	}
@@ -567,7 +567,7 @@ func TestStateStore_PreparedQueryResolve(t *testing.T) {
 
 	// Try to lookup a query that's not there using something that looks
 	// like a name
-	idx, actual, err = s.PreparedQueryResolve(query.Name)
+	idx, actual, err = s.PreparedQueryResolve(query.Name, structs.QuerySource{})
 	if err != nil {
 		t.Fatalf("err: %s", err)
 	}
@@ -600,7 +600,7 @@ func TestStateStore_PreparedQueryResolve(t *testing.T) {
 			ModifyIndex: 3,
 		},
 	}
-	idx, actual, err = s.PreparedQueryResolve(query.ID)
+	idx, actual, err = s.PreparedQueryResolve(query.ID, structs.QuerySource{})
 	if err != nil {
 		t.Fatalf("err: %s", err)
 	}
@@ -612,7 +612,7 @@ func TestStateStore_PreparedQueryResolve(t *testing.T) {
 	}
 
 	// Read it back using the name and verify it again.
-	idx, actual, err = s.PreparedQueryResolve(query.Name)
+	idx, actual, err = s.PreparedQueryResolve(query.Name, structs.QuerySource{})
 	if err != nil {
 		t.Fatalf("err: %s", err)
 	}
@@ -625,7 +625,7 @@ func TestStateStore_PreparedQueryResolve(t *testing.T) {
 
 	// Make sure an empty lookup is well-behaved if there are actual queries
 	// in the state store.
-	idx, actual, err = s.PreparedQueryResolve("")
+	idx, actual, err = s.PreparedQueryResolve("", structs.QuerySource{})
 	if err != ErrMissingQueryID {
 		t.Fatalf("bad: %v ", err)
 	}
@@ -681,7 +681,7 @@ func TestStateStore_PreparedQueryResolve(t *testing.T) {
 			ModifyIndex: 4,
 		},
 	}
-	idx, actual, err = s.PreparedQueryResolve("prod-mongodb")
+	idx, actual, err = s.PreparedQueryResolve("prod-mongodb", structs.QuerySource{})
 	if err != nil {
 		t.Fatalf("err: %s", err)
 	}
@@ -708,7 +708,7 @@ func TestStateStore_PreparedQueryResolve(t *testing.T) {
 			ModifyIndex: 5,
 		},
 	}
-	idx, actual, err = s.PreparedQueryResolve("prod-redis-foobar")
+	idx, actual, err = s.PreparedQueryResolve("prod-redis-foobar", structs.QuerySource{})
 	if err != nil {
 		t.Fatalf("err: %s", err)
 	}
@@ -735,7 +735,7 @@ func TestStateStore_PreparedQueryResolve(t *testing.T) {
 			ModifyIndex: 4,
 		},
 	}
-	idx, actual, err = s.PreparedQueryResolve("prod-")
+	idx, actual, err = s.PreparedQueryResolve("prod-", structs.QuerySource{})
 	if err != nil {
 		t.Fatalf("err: %s", err)
 	}
@@ -748,7 +748,7 @@ func TestStateStore_PreparedQueryResolve(t *testing.T) {
 
 	// Make sure you can't run a prepared query template by ID, since that
 	// makes no sense.
-	_, _, err = s.PreparedQueryResolve(tmpl1.ID)
+	_, _, err = s.PreparedQueryResolve(tmpl1.ID, structs.QuerySource{})
 	if err == nil || !strings.Contains(err.Error(), "prepared query templates can only be resolved up by name") {
 		t.Fatalf("bad: %v", err)
 	}
@@ -960,7 +960,7 @@ func TestStateStore_PreparedQuery_Snapshot_Restore(t *testing.T) {
 
 		// Make sure the second query, which is a template, was compiled
 		// and can be resolved.
-		_, query, err := s.PreparedQueryResolve("bob-backwards-is-bob")
+		_, query, err := s.PreparedQueryResolve("bob-backwards-is-bob", structs.QuerySource{})
 		if err != nil {
 			t.Fatalf("err: %s", err)
 		}

agent/prepared_query_endpoint.go

@@ -96,6 +96,7 @@ func (s *HTTPServer) preparedQueryExecute(id string, resp http.ResponseWriter, r
 		Agent: structs.QuerySource{
 			Node:       s.agent.config.NodeName,
 			Datacenter: s.agent.config.Datacenter,
+			Segment:    s.agent.config.Segment,
 		},
 	}
 	s.parseSource(req, &args.Source)
@@ -140,6 +141,7 @@ func (s *HTTPServer) preparedQueryExplain(id string, resp http.ResponseWriter, r
 		Agent: structs.QuerySource{
 			Node:       s.agent.config.NodeName,
 			Datacenter: s.agent.config.Datacenter,
+			Segment:    s.agent.config.Segment,
 		},
 	}
 	s.parseSource(req, &args.Source)

agent/structs/structs.go

@@ -242,6 +242,7 @@ func (r *DeregisterRequest) RequestDatacenter() string {
 // coordinates.
 type QuerySource struct {
 	Datacenter string
+	Segment    string
 	Node       string
 }
 
@@ -310,13 +311,13 @@ type Node struct {
 type Nodes []*Node
 
 // ValidateMeta validates a set of key/value pairs from the agent config
-func ValidateMetadata(meta map[string]string) error {
+func ValidateMetadata(meta map[string]string, allowConsulPrefix bool) error {
 	if len(meta) > metaMaxKeyPairs {
 		return fmt.Errorf("Node metadata cannot contain more than %d key/value pairs", metaMaxKeyPairs)
 	}
 
 	for key, value := range meta {
-		if err := validateMetaPair(key, value); err != nil {
+		if err := validateMetaPair(key, value, allowConsulPrefix); err != nil {
 			return fmt.Errorf("Couldn't load metadata pair ('%s', '%s'): %s", key, value, err)
 		}
 	}
@@ -325,7 +326,7 @@ func ValidateMetadata(meta map[string]string) error {
 }
 
 // validateMetaPair checks that the given key/value pair is in a valid format
-func validateMetaPair(key, value string) error {
+func validateMetaPair(key, value string, allowConsulPrefix bool) error {
 	if key == "" {
 		return fmt.Errorf("Key cannot be blank")
 	}
@@ -335,7 +336,7 @@ func validateMetaPair(key, value string) error {
 	if len(key) > metaKeyMaxLength {
 		return fmt.Errorf("Key is too long (limit: %d characters)", metaKeyMaxLength)
 	}
-	if strings.HasPrefix(key, metaKeyReservedPrefix) {
+	if strings.HasPrefix(key, metaKeyReservedPrefix) && !allowConsulPrefix {
 		return fmt.Errorf("Key prefix '%s' is reserved for internal use", metaKeyReservedPrefix)
 	}
 	if len(value) > metaValueMaxLength {

agent/structs/structs_test.go

@@ -482,7 +482,7 @@ func TestStructs_ValidateMetadata(t *testing.T) {
 		"key2": "value2",
 	}
 	// Should succeed
-	if err := ValidateMetadata(meta); err != nil {
+	if err := ValidateMetadata(meta, false); err != nil {
 		t.Fatalf("err: %s", err)
 	}
 
@@ -490,7 +490,7 @@ func TestStructs_ValidateMetadata(t *testing.T) {
 	meta = map[string]string{
 		"": "value1",
 	}
-	if err := ValidateMetadata(meta); !strings.Contains(err.Error(), "Couldn't load metadata pair") {
+	if err := ValidateMetadata(meta, false); !strings.Contains(err.Error(), "Couldn't load metadata pair") {
 		t.Fatalf("should have failed")
 	}
 
@@ -499,7 +499,7 @@ func TestStructs_ValidateMetadata(t *testing.T) {
 	for i := 0; i < metaMaxKeyPairs+1; i++ {
 		meta[string(i)] = "value"
 	}
-	if err := ValidateMetadata(meta); !strings.Contains(err.Error(), "cannot contain more than") {
+	if err := ValidateMetadata(meta, false); !strings.Contains(err.Error(), "cannot contain more than") {
 		t.Fatalf("should have failed")
 	}
 }
@@ -529,7 +529,7 @@ func TestStructs_validateMetaPair(t *testing.T) {
 	}
 
 	for _, pair := range pairs {
-		err := validateMetaPair(pair.Key, pair.Value)
+		err := validateMetaPair(pair.Key, pair.Value, false)
 		if pair.Error == "" && err != nil {
 			t.Fatalf("should have succeeded: %v, %v", pair, err)
 		} else if pair.Error != "" && !strings.Contains(err.Error(), pair.Error) {

command/agent.go

@@ -225,7 +225,7 @@ func (cmd *AgentCommand) readConfig() *agent.Config {
 			key, value := agent.ParseMetaPair(entry)
 			cmdCfg.Meta[key] = value
 		}
-		if err := structs.ValidateMetadata(cmdCfg.Meta); err != nil {
+		if err := structs.ValidateMetadata(cmdCfg.Meta, false); err != nil {
 			cmd.UI.Error(fmt.Sprintf("Failed to parse node metadata: %v", err))
 			return nil
 		}

website/source/api/query.html.md

@@ -87,6 +87,25 @@ populate the query before it is executed. All of the string fields inside the
   doesn't match, or an invalid index is given, then `${match(N)}` will return an
   empty string.
 
+- `${agent.segment}` has the network segment (Enterprise only) of the agent that
+  initiated the query. This can be used with the `NodeMeta` field to limit the results
+  of a query to service instances within its own network segment:
+
+    ```json
+    {
+      "Name": "",
+      "Template": {
+        "Type": "name_prefix_match"
+      },
+      "Service": {
+        "Service": "${name.full}",
+        "NodeMeta": {"consul-network-segment": "${agent.segment}"}
+      }
+    }
+    ```
+  This will map all names of the form "<service>.query.consul" over DNS to a query
+  that will select an instance of the service in the agent's own network segment.
+
 Using templates, it is possible to apply prepared query behaviors to many
 services with a single template. Here's an example template that matches any
 query and applies a failover policy to it: