From 998b691878c10e5531e94db7fc8a46485b7498de Mon Sep 17 00:00:00 2001 From: James Phillips Date: Fri, 26 Feb 2016 01:27:51 -0800 Subject: [PATCH] Integrates templates into state store and endpoint (sans tests). --- consul/prepared_query_endpoint.go | 15 +++++- consul/state/prepared_query.go | 72 ++++++++++++++++++++++++----- consul/state/prepared_query_test.go | 10 ++-- consul/state/schema.go | 15 ++++++ 4 files changed, 94 insertions(+), 18 deletions(-) diff --git a/consul/prepared_query_endpoint.go b/consul/prepared_query_endpoint.go index ebd28009ab..a9b9f1b7b0 100644 --- a/consul/prepared_query_endpoint.go +++ b/consul/prepared_query_endpoint.go @@ -8,6 +8,7 @@ import ( "time" "github.com/armon/go-metrics" + "github.com/hashicorp/consul/consul/prepared_query" "github.com/hashicorp/consul/consul/structs" "github.com/hashicorp/go-uuid" ) @@ -289,7 +290,7 @@ func (p *PreparedQuery) Execute(args *structs.PreparedQueryExecuteRequest, // Try to locate the query. state := p.srv.fsm.State() - _, query, err := state.PreparedQueryLookup(args.QueryIDOrName) + _, query, ct, err := state.PreparedQueryLookup(args.QueryIDOrName) if err != nil { return err } @@ -297,6 +298,18 @@ func (p *PreparedQuery) Execute(args *structs.PreparedQueryExecuteRequest, return ErrQueryNotFound } + // If this is a template then render the query first. + if prepared_query.IsTemplate(query) { + if ct == nil { + return fmt.Errorf("Missing compiled template") + } + render, err := ct.Render(args.QueryIDOrName) + if err != nil { + return err + } + query = render + } + // Execute the query for the local DC. if err := p.execute(query, reply); err != nil { return err diff --git a/consul/state/prepared_query.go b/consul/state/prepared_query.go index d547394ac9..d71f764ee2 100644 --- a/consul/state/prepared_query.go +++ b/consul/state/prepared_query.go @@ -33,6 +33,18 @@ func toPreparedQuery(wrapped interface{}) *structs.PreparedQuery { return wrapped.(*queryWrapper).PreparedQuery } +// isQueryWild returns the wild-ness of a query. See isWild for details. +func isQueryWild(query *structs.PreparedQuery) bool { + return query != nil && prepared_query.IsTemplate(query) && query.Name == "" +} + +// isWrappedWild is used to determine if the given wrapped query is a wild one, +// which means it has an empty Name and it's a template. See the comments for +// "wild" in schema.go for more details and to see where this is used. +func isWrappedWild(obj interface{}) (bool, error) { + return isQueryWild(toPreparedQuery(obj)), nil +} + // PreparedQueries is used to pull all the prepared queries from the snapshot. func (s *StateSnapshot) PreparedQueries() (structs.PreparedQueries, error) { queries, err := s.tx.Get("prepared-queries", "id") @@ -123,6 +135,19 @@ func (s *StateStore) preparedQuerySetTxn(tx *memdb.Txn, idx uint64, query *struc } } + // Similarly, if this is the wild query make sure there isn't another + // one, or that we are updating the same one. + if isQueryWild(query) { + wrapped, err := tx.First("prepared-queries", "wild", true) + if err != nil { + return fmt.Errorf("failed prepared query lookup: %s", err) + } + other := toPreparedQuery(wrapped) + if other != nil && (existing == nil || existing.ID != other.ID) { + return fmt.Errorf("a prepared query template already exists with an empty name") + } + } + // Verify that the name doesn't alias any existing ID. We allow queries // to be looked up by ID *or* name so we don't want anyone to try to // register a query with a name equal to some other query's ID in an @@ -238,7 +263,7 @@ func (s *StateStore) PreparedQueryGet(queryID string) (uint64, *structs.Prepared // PreparedQueryLookup returns the given prepared query by looking up an ID or // Name. -func (s *StateStore) PreparedQueryLookup(queryIDOrName string) (uint64, *structs.PreparedQuery, error) { +func (s *StateStore) PreparedQueryLookup(queryIDOrName string) (uint64, *structs.PreparedQuery, *prepared_query.CompiledTemplate, error) { tx := s.db.Txn(false) defer tx.Abort() @@ -250,7 +275,7 @@ func (s *StateStore) PreparedQueryLookup(queryIDOrName string) (uint64, *structs // but we check it here to be explicit about it (we'd never want to // return the results from the first query w/o a name). if queryIDOrName == "" { - return 0, nil, ErrMissingQueryID + return 0, nil, nil, ErrMissingQueryID } // Try first by ID if it looks like they gave us an ID. We check the @@ -259,23 +284,46 @@ func (s *StateStore) PreparedQueryLookup(queryIDOrName string) (uint64, *structs if isUUID(queryIDOrName) { wrapped, err := tx.First("prepared-queries", "id", queryIDOrName) if err != nil { - return 0, nil, fmt.Errorf("failed prepared query lookup: %s", err) + return 0, nil, nil, fmt.Errorf("failed prepared query lookup: %s", err) } if wrapped != nil { - return idx, toPreparedQuery(wrapped), nil + wrapper := wrapped.(*queryWrapper) + return idx, wrapper.PreparedQuery, wrapper.ct, nil } } - // Then try by name. - wrapped, err := tx.First("prepared-queries", "name", queryIDOrName) - if err != nil { - return 0, nil, fmt.Errorf("failed prepared query lookup: %s", err) - } - if wrapped != nil { - return idx, toPreparedQuery(wrapped), nil + // Then try by name. We use a prefix match but check to make sure that + // the query's name matches the whole prefix for a non-template query. + // Templates are allowed to use the partial match. It's more efficient + // to combine the two lookups here, even though the logic is a little + // less clear. + { + wrapped, err := tx.First("prepared-queries", "name_prefix", queryIDOrName) + if err != nil { + return 0, nil, nil, fmt.Errorf("failed prepared query lookup: %s", err) + } + if wrapped != nil { + wrapper := wrapped.(*queryWrapper) + query, ct := wrapper.PreparedQuery, wrapper.ct + if query.Name == queryIDOrName || prepared_query.IsTemplate(query) { + return idx, query, ct, nil + } + } } - return idx, nil, nil + // Finally, see if there's a wild template we can use. + { + wrapped, err := tx.First("prepared-queries", "wild", true) + if err != nil { + return 0, nil, nil, fmt.Errorf("failed prepared query lookup: %s", err) + } + if wrapped != nil { + wrapper := wrapped.(*queryWrapper) + return idx, wrapper.PreparedQuery, wrapper.ct, nil + } + } + + return idx, nil, nil, nil } // PreparedQueryList returns all the prepared queries. diff --git a/consul/state/prepared_query_test.go b/consul/state/prepared_query_test.go index 98303b8a5e..04412da8f4 100644 --- a/consul/state/prepared_query_test.go +++ b/consul/state/prepared_query_test.go @@ -336,7 +336,7 @@ func TestStateStore_PreparedQueryLookup(t *testing.T) { // Try to lookup a query that's not there using something that looks // like a real ID. - idx, actual, err := s.PreparedQueryLookup(query.ID) + idx, actual, _, err := s.PreparedQueryLookup(query.ID) if err != nil { t.Fatalf("err: %s", err) } @@ -349,7 +349,7 @@ func TestStateStore_PreparedQueryLookup(t *testing.T) { // Try to lookup a query that's not there using something that looks // like a name - idx, actual, err = s.PreparedQueryLookup(query.Name) + idx, actual, _, err = s.PreparedQueryLookup(query.Name) if err != nil { t.Fatalf("err: %s", err) } @@ -382,7 +382,7 @@ func TestStateStore_PreparedQueryLookup(t *testing.T) { ModifyIndex: 3, }, } - idx, actual, err = s.PreparedQueryLookup(query.ID) + idx, actual, _, err = s.PreparedQueryLookup(query.ID) if err != nil { t.Fatalf("err: %s", err) } @@ -394,7 +394,7 @@ func TestStateStore_PreparedQueryLookup(t *testing.T) { } // Read it back using the name and verify it again. - idx, actual, err = s.PreparedQueryLookup(query.Name) + idx, actual, _, err = s.PreparedQueryLookup(query.Name) if err != nil { t.Fatalf("err: %s", err) } @@ -407,7 +407,7 @@ func TestStateStore_PreparedQueryLookup(t *testing.T) { // Make sure an empty lookup is well-behaved if there are actual queries // in the state store. - idx, actual, err = s.PreparedQueryLookup("") + idx, actual, _, err = s.PreparedQueryLookup("") if err != ErrMissingQueryID { t.Fatalf("bad: %v ", err) } diff --git a/consul/state/schema.go b/consul/state/schema.go index e47939a562..53ff01f706 100644 --- a/consul/state/schema.go +++ b/consul/state/schema.go @@ -390,6 +390,21 @@ func preparedQueriesTableSchema() *memdb.TableSchema { Lowercase: true, }, }, + // This is a bit of an oddball. It's an important feature + // of prepared query templates to be able to define a + // single template that matches any query. Unfortunately, + // we can't index an empty Name field. This index lets us + // keep track of whether there is any wild template in + // existence, so there will be one "true" in here if that + // exists, and everything else will be "false". + "wild": &memdb.IndexSchema{ + Name: "wild", + AllowMissing: false, + Unique: false, + Indexer: &memdb.ConditionalIndex{ + Conditional: isWrappedWild, + }, + }, "session": &memdb.IndexSchema{ Name: "session", AllowMissing: true,