From c816f79bf8fbf56ea475e274bbf66413bf5e13c9 Mon Sep 17 00:00:00 2001 From: James Phillips Date: Fri, 26 Feb 2016 00:25:44 -0800 Subject: [PATCH] Wraps the prepared query to also store the compiled template. --- consul/fsm.go | 4 +- consul/prepared_query_endpoint.go | 4 +- consul/prepared_query_endpoint_test.go | 7 +- consul/state/prepared_query.go | 102 +++++++++++++++++-------- consul/state/prepared_query_test.go | 6 +- consul/state/state_store.go | 11 ++- consul/structs/prepared_query.go | 24 ++++++ 7 files changed, 113 insertions(+), 45 deletions(-) diff --git a/consul/fsm.go b/consul/fsm.go index 9f786024ab..1d475914f3 100644 --- a/consul/fsm.go +++ b/consul/fsm.go @@ -610,9 +610,9 @@ func (s *consulSnapshot) persistPreparedQueries(sink raft.SnapshotSink, return err } - for query := queries.Next(); query != nil; query = queries.Next() { + for _, query := range queries { sink.Write([]byte{byte(structs.PreparedQueryRequestType)}) - if err := encoder.Encode(query.(*structs.PreparedQuery)); err != nil { + if err := encoder.Encode(query); err != nil { return err } } diff --git a/consul/prepared_query_endpoint.go b/consul/prepared_query_endpoint.go index 02a1a4d4bb..ebd28009ab 100644 --- a/consul/prepared_query_endpoint.go +++ b/consul/prepared_query_endpoint.go @@ -134,6 +134,8 @@ func parseQuery(query *structs.PreparedQuery) error { // transaction. Otherwise, people could "steal" queries that they don't // have proper ACL rights to change. // - Session is optional and checked for integrity during the transaction. + // - Template is checked during the transaction since that's where we + // compile it. // Token is checked when the query is executed, but we do make sure the // user hasn't accidentally pasted-in the special redacted token name, @@ -162,7 +164,7 @@ func parseQuery(query *structs.PreparedQuery) error { func parseService(svc *structs.ServiceQuery) error { // Service is required. if svc.Service == "" { - return fmt.Errorf("Must provide a service name to query") + return fmt.Errorf("Must provide a Service name to query") } // NearestN can be 0 which means "don't fail over by RTT". diff --git a/consul/prepared_query_endpoint_test.go b/consul/prepared_query_endpoint_test.go index f584552a8f..5f18c6d96b 100644 --- a/consul/prepared_query_endpoint_test.go +++ b/consul/prepared_query_endpoint_test.go @@ -530,7 +530,7 @@ func TestPreparedQuery_parseQuery(t *testing.T) { query := &structs.PreparedQuery{} err := parseQuery(query) - if err == nil || !strings.Contains(err.Error(), "Must provide a service") { + if err == nil || !strings.Contains(err.Error(), "Must provide a Service") { t.Fatalf("bad: %v", err) } @@ -1910,6 +1910,11 @@ func TestPreparedQuery_tagFilter(t *testing.T) { if ret != "node2|node6" { t.Fatalf("bad: %s", ret) } + + ret = stringify(tagFilter([]string{""}, testNodes())) + if ret != "" { + t.Fatalf("bad: %s", ret) + } } func TestPreparedQuery_Wrapper(t *testing.T) { diff --git a/consul/state/prepared_query.go b/consul/state/prepared_query.go index dad07b8962..d547394ac9 100644 --- a/consul/state/prepared_query.go +++ b/consul/state/prepared_query.go @@ -4,6 +4,7 @@ import ( "fmt" "regexp" + "github.com/hashicorp/consul/consul/prepared_query" "github.com/hashicorp/consul/consul/structs" "github.com/hashicorp/go-memdb" ) @@ -16,22 +17,54 @@ func isUUID(str string) bool { return validUUID.MatchString(str) } +// queryWrapper is an internal structure that is used to store a query alongside +// its compiled template, which can be nil. +type queryWrapper struct { + *structs.PreparedQuery + ct *prepared_query.CompiledTemplate +} + +// toPreparedQuery unwraps the internal form of a prepared query and returns +// the regular struct. +func toPreparedQuery(wrapped interface{}) *structs.PreparedQuery { + if wrapped == nil { + return nil + } + return wrapped.(*queryWrapper).PreparedQuery +} + // PreparedQueries is used to pull all the prepared queries from the snapshot. -func (s *StateSnapshot) PreparedQueries() (memdb.ResultIterator, error) { - iter, err := s.tx.Get("prepared-queries", "id") +func (s *StateSnapshot) PreparedQueries() (structs.PreparedQueries, error) { + queries, err := s.tx.Get("prepared-queries", "id") if err != nil { return nil, err } - return iter, nil + + var ret structs.PreparedQueries + for wrapped := queries.Next(); wrapped != nil; wrapped = queries.Next() { + ret = append(ret, toPreparedQuery(wrapped)) + } + return ret, nil } // PrepparedQuery is used when restoring from a snapshot. For general inserts, // use PreparedQuerySet. func (s *StateRestore) PreparedQuery(query *structs.PreparedQuery) error { - if err := s.tx.Insert("prepared-queries", query); err != nil { - return fmt.Errorf("failed restoring prepared query: %s", err) + // If this is a template, compile it, otherwise leave the compiled + // template field nil. + var ct *prepared_query.CompiledTemplate + if prepared_query.IsTemplate(query) { + var err error + ct, err = prepared_query.Compile(query) + if err != nil { + return fmt.Errorf("failed compiling template: %s", err) + } } + // Insert the wrapped query. + if err := s.tx.Insert("prepared-queries", &queryWrapper{query, ct}); err != nil { + return fmt.Errorf("failed restoring prepared query: %s", err) + } if err := indexUpdateMaxTxn(s.tx, query.ModifyIndex, "prepared-queries"); err != nil { return fmt.Errorf("failed updating index: %s", err) } @@ -62,14 +95,15 @@ func (s *StateStore) preparedQuerySetTxn(tx *memdb.Txn, idx uint64, query *struc } // Check for an existing query. - existing, err := tx.First("prepared-queries", "id", query.ID) + wrapped, err := tx.First("prepared-queries", "id", query.ID) if err != nil { return fmt.Errorf("failed prepared query lookup: %s", err) } + existing := toPreparedQuery(wrapped) // Set the indexes. if existing != nil { - query.CreateIndex = existing.(*structs.PreparedQuery).CreateIndex + query.CreateIndex = existing.CreateIndex query.ModifyIndex = idx } else { query.CreateIndex = idx @@ -79,12 +113,12 @@ func (s *StateStore) preparedQuerySetTxn(tx *memdb.Txn, idx uint64, query *struc // Verify that the query name doesn't already exist, or that we are // updating the same instance that has this name. if query.Name != "" { - alias, err := tx.First("prepared-queries", "name", query.Name) + wrapped, err := tx.First("prepared-queries", "name", query.Name) if err != nil { return fmt.Errorf("failed prepared query lookup: %s", err) } - if alias != nil && (existing == nil || - existing.(*structs.PreparedQuery).ID != alias.(*structs.PreparedQuery).ID) { + other := toPreparedQuery(wrapped) + if other != nil && (existing == nil || existing.ID != other.ID) { return fmt.Errorf("name '%s' aliases an existing query name", query.Name) } } @@ -99,11 +133,11 @@ func (s *StateStore) preparedQuerySetTxn(tx *memdb.Txn, idx uint64, query *struc // index will complain if we look up something that's not formatted // like one. if isUUID(query.Name) { - alias, err := tx.First("prepared-queries", "id", query.Name) + wrapped, err := tx.First("prepared-queries", "id", query.Name) if err != nil { return fmt.Errorf("failed prepared query lookup: %s", err) } - if alias != nil { + if wrapped != nil { return fmt.Errorf("name '%s' aliases an existing query ID", query.Name) } } @@ -123,8 +157,19 @@ func (s *StateStore) preparedQuerySetTxn(tx *memdb.Txn, idx uint64, query *struc // checked at execute time and not doing integrity checking on them // helps avoid bootstrapping chicken and egg problems. - // Insert the query. - if err := tx.Insert("prepared-queries", query); err != nil { + // If this is a template, compile it, otherwise leave the compiled + // template field nil. + var ct *prepared_query.CompiledTemplate + if prepared_query.IsTemplate(query) { + var err error + ct, err = prepared_query.Compile(query) + if err != nil { + return fmt.Errorf("failed compiling template: %s", err) + } + } + + // Insert the wrapped query. + if err := tx.Insert("prepared-queries", &queryWrapper{query, ct}); err != nil { return fmt.Errorf("failed inserting prepared query: %s", err) } if err := tx.Insert("index", &IndexEntry{"prepared-queries", idx}); err != nil { @@ -155,16 +200,16 @@ func (s *StateStore) PreparedQueryDelete(idx uint64, queryID string) error { func (s *StateStore) preparedQueryDeleteTxn(tx *memdb.Txn, idx uint64, watches *DumbWatchManager, queryID string) error { // Pull the query. - query, err := tx.First("prepared-queries", "id", queryID) + wrapped, err := tx.First("prepared-queries", "id", queryID) if err != nil { return fmt.Errorf("failed prepared query lookup: %s", err) } - if query == nil { + if wrapped == nil { return nil } // Delete the query and update the index. - if err := tx.Delete("prepared-queries", query); err != nil { + if err := tx.Delete("prepared-queries", wrapped); err != nil { return fmt.Errorf("failed prepared query delete: %s", err) } if err := tx.Insert("index", &IndexEntry{"prepared-queries", idx}); err != nil { @@ -184,14 +229,11 @@ func (s *StateStore) PreparedQueryGet(queryID string) (uint64, *structs.Prepared idx := maxIndexTxn(tx, s.getWatchTables("PreparedQueryGet")...) // Look up the query by its ID. - query, err := tx.First("prepared-queries", "id", queryID) + wrapped, err := tx.First("prepared-queries", "id", queryID) if err != nil { return 0, nil, fmt.Errorf("failed prepared query lookup: %s", err) } - if query != nil { - return idx, query.(*structs.PreparedQuery), nil - } - return idx, nil, nil + return idx, toPreparedQuery(wrapped), nil } // PreparedQueryLookup returns the given prepared query by looking up an ID or @@ -215,22 +257,22 @@ func (s *StateStore) PreparedQueryLookup(queryIDOrName string) (uint64, *structs // format before trying this because the UUID index will complain if // we look up something that's not formatted like one. if isUUID(queryIDOrName) { - query, err := tx.First("prepared-queries", "id", queryIDOrName) + wrapped, err := tx.First("prepared-queries", "id", queryIDOrName) if err != nil { return 0, nil, fmt.Errorf("failed prepared query lookup: %s", err) } - if query != nil { - return idx, query.(*structs.PreparedQuery), nil + if wrapped != nil { + return idx, toPreparedQuery(wrapped), nil } } // Then try by name. - query, err := tx.First("prepared-queries", "name", queryIDOrName) + wrapped, err := tx.First("prepared-queries", "name", queryIDOrName) if err != nil { return 0, nil, fmt.Errorf("failed prepared query lookup: %s", err) } - if query != nil { - return idx, query.(*structs.PreparedQuery), nil + if wrapped != nil { + return idx, toPreparedQuery(wrapped), nil } return idx, nil, nil @@ -252,8 +294,8 @@ func (s *StateStore) PreparedQueryList() (uint64, structs.PreparedQueries, error // Go over all of the queries and build the response. var result structs.PreparedQueries - for query := queries.Next(); query != nil; query = queries.Next() { - result = append(result, query.(*structs.PreparedQuery)) + for wrapped := queries.Next(); wrapped != nil; wrapped = queries.Next() { + result = append(result, toPreparedQuery(wrapped)) } return idx, result, nil } diff --git a/consul/state/prepared_query_test.go b/consul/state/prepared_query_test.go index 83c3625c3b..98303b8a5e 100644 --- a/consul/state/prepared_query_test.go +++ b/consul/state/prepared_query_test.go @@ -581,14 +581,10 @@ func TestStateStore_PreparedQuery_Snapshot_Restore(t *testing.T) { }, }, } - iter, err := snap.PreparedQueries() + dump, err := snap.PreparedQueries() if err != nil { t.Fatalf("err: %s", err) } - var dump structs.PreparedQueries - for query := iter.Next(); query != nil; query = iter.Next() { - dump = append(dump, query.(*structs.PreparedQuery)) - } if !reflect.DeepEqual(dump, expected) { t.Fatalf("bad: %v", dump) } diff --git a/consul/state/state_store.go b/consul/state/state_store.go index 473775caef..9f50fb2631 100644 --- a/consul/state/state_store.go +++ b/consul/state/state_store.go @@ -2151,15 +2151,14 @@ func (s *StateStore) deleteSessionTxn(tx *memdb.Txn, idx uint64, watches *DumbWa return fmt.Errorf("failed prepared query lookup: %s", err) } { - var objs []interface{} - for query := queries.Next(); query != nil; query = queries.Next() { - objs = append(objs, query) + var ids []string + for wrapped := queries.Next(); wrapped != nil; wrapped = queries.Next() { + ids = append(ids, toPreparedQuery(wrapped).ID) } // Do the delete in a separate loop so we don't trash the iterator. - for _, obj := range objs { - q := obj.(*structs.PreparedQuery) - if err := s.preparedQueryDeleteTxn(tx, idx, watches, q.ID); err != nil { + for _, id := range ids { + if err := s.preparedQueryDeleteTxn(tx, idx, watches, id); err != nil { return fmt.Errorf("failed prepared query delete: %s", err) } } diff --git a/consul/structs/prepared_query.go b/consul/structs/prepared_query.go index 6058ca50f1..f83c04fcfe 100644 --- a/consul/structs/prepared_query.go +++ b/consul/structs/prepared_query.go @@ -40,6 +40,25 @@ type ServiceQuery struct { Tags []string } +const ( + // QueryTemplateTypeNamePrefixMatch uses the Name field of the query as + // a prefix to select the template. + QueryTemplateTypeNamePrefixMatch = "name_prefix_match" +) + +// QueryTemplateOptions controls settings if this query is a template. +type QueryTemplateOptions struct { + // Type, if non-empty, means that this query is a template. This is + // set to one of the QueryTemplateType* constants above. + Type string + + // Regexp is an optional regular expression to use to parse the full + // name, once the prefix match has selected a template. This can be + // used to extract parts of the name and choose a service name, set + // tags, etc. + Regexp string +} + // PreparedQuery defines a complete prepared query, and is the structure we // maintain in the state store. type PreparedQuery struct { @@ -61,6 +80,11 @@ type PreparedQuery struct { // with management privileges, must be used to change the query later. Token string + // Template is used to configure this query as a template, which will + // respond to queries based on the Name, and then will be rendered + // before it is executed. + Template QueryTemplateOptions + // Service defines a service query (leaving things open for other types // later). Service ServiceQuery