From 653f41ce419e7180284ebd1bdcb353fbd5b225ab Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 15 Apr 2021 13:26:53 -0400 Subject: [PATCH 1/2] state: use index constants for ConfigEntry indexes --- agent/consul/state/config_entry.go | 5 ++--- agent/consul/state/config_entry_intention.go | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/agent/consul/state/config_entry.go b/agent/consul/state/config_entry.go index 28d377a824..1d33580290 100644 --- a/agent/consul/state/config_entry.go +++ b/agent/consul/state/config_entry.go @@ -16,7 +16,6 @@ type ConfigEntryLinkIndex struct { } type discoveryChainConfigEntry interface { - structs.ConfigEntry // ListRelatedServices returns a list of other names of services referenced // in this config entry. ListRelatedServices() []structs.ServiceID @@ -426,7 +425,7 @@ func (s *Store) discoveryChainSourcesTxn(tx ReadTxn, ws memdb.WatchSet, dc strin queue := []structs.ServiceName{destination} for len(queue) > 0 { // The "link" index returns config entries that reference a service - iter, err := tx.Get(tableConfigEntries, "link", queue[0].ToServiceID()) + iter, err := tx.Get(tableConfigEntries, indexLink, queue[0].ToServiceID()) if err != nil { return 0, nil, err } @@ -598,7 +597,7 @@ func validateProposedConfigEntryInServiceGraph( sid := structs.NewServiceID(name, entMeta) checkChains[sid] = struct{}{} - iter, err := tx.Get(tableConfigEntries, "link", sid) + iter, err := tx.Get(tableConfigEntries, indexLink, sid) if err != nil { return err } diff --git a/agent/consul/state/config_entry_intention.go b/agent/consul/state/config_entry_intention.go index f8d91efced..98fcecbfac 100644 --- a/agent/consul/state/config_entry_intention.go +++ b/agent/consul/state/config_entry_intention.go @@ -150,7 +150,7 @@ func configIntentionGetTxn(tx ReadTxn, ws memdb.WatchSet, id string) (uint64, *s idx = 1 } - watchCh, existing, err := tx.FirstWatch(tableConfigEntries, "intention-legacy-id", id) + watchCh, existing, err := tx.FirstWatch(tableConfigEntries, indexIntentionLegacyID, id) if err != nil { return 0, nil, nil, fmt.Errorf("failed config entry lookup: %s", err) } @@ -265,7 +265,7 @@ func readSourceIntentionsFromConfigEntriesTxn(tx ReadTxn, ws memdb.WatchSet, ser func readSourceIntentionsFromConfigEntriesForServiceTxn(tx ReadTxn, ws memdb.WatchSet, serviceName string, entMeta *structs.EnterpriseMeta, results structs.Intentions) (structs.Intentions, error) { sn := structs.NewServiceName(serviceName, entMeta) - iter, err := tx.Get(tableConfigEntries, "intention-source", sn) + iter, err := tx.Get(tableConfigEntries, indexSource, sn) if err != nil { return nil, fmt.Errorf("failed config entry lookup: %s", err) } From b57b3726d29bde7fdeb62f6a200b5fa95aae93b0 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 31 Mar 2021 16:21:21 -0400 Subject: [PATCH 2/2] state: remove unnecessary kind index The query can be performed using a prefix query on the ID index. Also backport some enterprise changes to prevent conflicts. --- agent/consul/config_endpoint.go | 2 +- agent/consul/state/config_entry.go | 24 ++++++++++++++++++--- agent/consul/state/config_entry_oss.go | 24 ++++++++++++++------- agent/consul/state/config_entry_oss_test.go | 18 +++++++--------- agent/consul/state/config_entry_schema.go | 20 ----------------- agent/structs/structs_oss.go | 8 +++++++ 6 files changed, 54 insertions(+), 42 deletions(-) diff --git a/agent/consul/config_endpoint.go b/agent/consul/config_endpoint.go index 6c4017dafb..a9abb52cb7 100644 --- a/agent/consul/config_endpoint.go +++ b/agent/consul/config_endpoint.go @@ -336,7 +336,7 @@ func (c *ConfigEntry) ResolveServiceConfig(args *structs.ServiceConfigRequest, r // Pass the WatchSet to both the service and proxy config lookups. If either is updated during the // blocking query, this function will be rerun and these state store lookups will both be current. // We use the default enterprise meta to look up the global proxy defaults because they are not namespaced. - _, proxyEntry, err := state.ConfigEntry(ws, structs.ProxyDefaults, structs.ProxyConfigGlobal, structs.DefaultEnterpriseMeta()) + _, proxyEntry, err := state.ConfigEntry(ws, structs.ProxyDefaults, structs.ProxyConfigGlobal, &args.EnterpriseMeta) if err != nil { return err } diff --git a/agent/consul/state/config_entry.go b/agent/consul/state/config_entry.go index 1d33580290..b7fe4dd6e5 100644 --- a/agent/consul/state/config_entry.go +++ b/agent/consul/state/config_entry.go @@ -852,7 +852,7 @@ func readDiscoveryChainConfigEntriesTxn( sid := structs.NewServiceID(serviceName, entMeta) // Grab the proxy defaults if they exist. - idx, proxy, err := getProxyConfigEntryTxn(tx, ws, structs.ProxyConfigGlobal, overrides, structs.DefaultEnterpriseMeta()) + idx, proxy, err := getProxyConfigEntryTxn(tx, ws, structs.ProxyConfigGlobal, overrides, entMeta) if err != nil { return 0, nil, err } else if proxy != nil { @@ -1168,6 +1168,7 @@ func configEntryWithOverridesTxn( ) (uint64, structs.ConfigEntry, error) { if len(overrides) > 0 { kn := NewConfigEntryKindName(kind, name, entMeta) + kn.Normalize() entry, ok := overrides[kn] if ok { return 0, entry, nil // a nil entry implies it should act like it is erased @@ -1185,7 +1186,7 @@ func protocolForService( svc structs.ServiceName, ) (uint64, string, error) { // Get the global proxy defaults (for default protocol) - maxIdx, proxyConfig, err := configEntryTxn(tx, ws, structs.ProxyDefaults, structs.ProxyConfigGlobal, structs.DefaultEnterpriseMeta()) + maxIdx, proxyConfig, err := configEntryTxn(tx, ws, structs.ProxyDefaults, structs.ProxyConfigGlobal, &svc.EnterpriseMeta) if err != nil { return 0, "", err } @@ -1229,6 +1230,11 @@ type ConfigEntryKindName struct { structs.EnterpriseMeta } +// NewConfigEntryKindName returns a new ConfigEntryKindName. The EnterpriseMeta +// values will be normalized based on the kind. +// +// Any caller which modifies the EnterpriseMeta field must call Normalize before +// persisting or using the value as a map key. func NewConfigEntryKindName(kind, name string, entMeta *structs.EnterpriseMeta) ConfigEntryKindName { ret := ConfigEntryKindName{ Kind: kind, @@ -1239,7 +1245,7 @@ func NewConfigEntryKindName(kind, name string, entMeta *structs.EnterpriseMeta) } ret.EnterpriseMeta = *entMeta - ret.EnterpriseMeta.Normalize() + ret.Normalize() return ret } @@ -1252,3 +1258,15 @@ type ConfigEntryKindQuery struct { Kind string structs.EnterpriseMeta } + +// NamespaceOrDefault exists because structs.EnterpriseMeta uses a pointer +// receiver for this method. Remove once that is fixed. +func (q ConfigEntryKindQuery) NamespaceOrDefault() string { + return q.EnterpriseMeta.NamespaceOrDefault() +} + +// PartitionOrDefault exists because structs.EnterpriseMeta uses a pointer +// receiver for this method. Remove once that is fixed. +func (q ConfigEntryKindQuery) PartitionOrDefault() string { + return q.EnterpriseMeta.PartitionOrDefault() +} diff --git a/agent/consul/state/config_entry_oss.go b/agent/consul/state/config_entry_oss.go index 5fb2e6afc6..0afc18204e 100644 --- a/agent/consul/state/config_entry_oss.go +++ b/agent/consul/state/config_entry_oss.go @@ -12,15 +12,23 @@ import ( ) func indexFromConfigEntryKindName(arg interface{}) ([]byte, error) { - n, ok := arg.(ConfigEntryKindName) - if !ok { - return nil, fmt.Errorf("invalid type for ConfigEntryKindName query: %T", arg) + var b indexBuilder + + switch n := arg.(type) { + case *structs.EnterpriseMeta: + return nil, nil + case structs.EnterpriseMeta: + return b.Bytes(), nil + case ConfigEntryKindQuery: + b.String(strings.ToLower(n.Kind)) + return b.Bytes(), nil + case ConfigEntryKindName: + b.String(strings.ToLower(n.Kind)) + b.String(strings.ToLower(n.Name)) + return b.Bytes(), nil } - var b indexBuilder - b.String(strings.ToLower(n.Kind)) - b.String(strings.ToLower(n.Name)) - return b.Bytes(), nil + return nil, fmt.Errorf("invalid type for ConfigEntryKindName query: %T", arg) } func validateConfigEntryEnterprise(_ ReadTxn, _ structs.ConfigEntry) error { @@ -32,7 +40,7 @@ func getAllConfigEntriesWithTxn(tx ReadTxn, _ *structs.EnterpriseMeta) (memdb.Re } func getConfigEntryKindsWithTxn(tx ReadTxn, kind string, _ *structs.EnterpriseMeta) (memdb.ResultIterator, error) { - return tx.Get(tableConfigEntries, indexKind, ConfigEntryKindQuery{Kind: kind}) + return tx.Get(tableConfigEntries, indexID+"_prefix", ConfigEntryKindQuery{Kind: kind}) } func configIntentionsConvertToList(iter memdb.ResultIterator, _ *structs.EnterpriseMeta) structs.Intentions { diff --git a/agent/consul/state/config_entry_oss_test.go b/agent/consul/state/config_entry_oss_test.go index 62245d70c4..b6cb4832ce 100644 --- a/agent/consul/state/config_entry_oss_test.go +++ b/agent/consul/state/config_entry_oss_test.go @@ -18,17 +18,15 @@ func testIndexerTableConfigEntries() map[string]indexerTestCase { source: &structs.ProxyConfigEntry{Name: "NaMe"}, expected: []byte("proxy-defaults\x00name\x00"), }, - }, - indexKind: { - read: indexValue{ - source: ConfigEntryKindQuery{ - Kind: "Service-Defaults", + prefix: []indexValue{ + { + source: structs.EnterpriseMeta{}, + expected: nil, + }, + { + source: ConfigEntryKindQuery{Kind: "Proxy-Defaults"}, + expected: []byte("proxy-defaults\x00"), }, - expected: []byte("service-defaults\x00"), - }, - write: indexValue{ - source: &structs.ServiceConfigEntry{}, - expected: []byte("service-defaults\x00"), }, }, } diff --git a/agent/consul/state/config_entry_schema.go b/agent/consul/state/config_entry_schema.go index 5e32c4945a..53c03d597b 100644 --- a/agent/consul/state/config_entry_schema.go +++ b/agent/consul/state/config_entry_schema.go @@ -33,15 +33,6 @@ func configTableSchema() *memdb.TableSchema { prefixIndex: indexFromConfigEntryKindName, }, }, - indexKind: { - Name: indexKind, - AllowMissing: false, - Unique: false, - Indexer: indexerSingle{ - readIndex: indexFromConfigEntryKindQuery, - writeIndex: indexKindFromConfigEntry, - }, - }, indexLink: { Name: indexLink, AllowMissing: true, @@ -80,17 +71,6 @@ func indexFromConfigEntry(raw interface{}) ([]byte, error) { return b.Bytes(), nil } -func indexFromConfigEntryKindQuery(raw interface{}) ([]byte, error) { - q, ok := raw.(ConfigEntryKindQuery) - if !ok { - return nil, fmt.Errorf("type must be ConfigEntryKindQuery: %T", raw) - } - - var b indexBuilder - b.String(strings.ToLower(q.Kind)) - return b.Bytes(), nil -} - // indexKindFromConfigEntry indexes kinds without a namespace for any config // entries that span all namespaces. func indexKindFromConfigEntry(raw interface{}) ([]byte, error) { diff --git a/agent/structs/structs_oss.go b/agent/structs/structs_oss.go index 7e7e22930d..64305c3065 100644 --- a/agent/structs/structs_oss.go +++ b/agent/structs/structs_oss.go @@ -50,6 +50,14 @@ func (m *EnterpriseMeta) NamespaceOrEmpty() string { return "" } +func (m *EnterpriseMeta) PartitionOrDefault() string { + return "" +} + +func (m *EnterpriseMeta) PartitionOrEmpty() string { + return "" +} + func NewEnterpriseMeta(_ string) EnterpriseMeta { return emptyEnterpriseMeta }