From 8b58fa8afead0a0179f51ec6dd6c08bd9fca3ee9 Mon Sep 17 00:00:00 2001 From: freddygv Date: Mon, 23 May 2022 10:16:39 -0600 Subject: [PATCH 1/3] Update assumptions around exported-service config Given that the exported-services config entry can use wildcards, the precedence for wildcards is handled as with intentions. The most exact match is the match that applies for any given service. We do not take the union of all that apply. Another update that was made was to reflect that only one exported-services config entry applies to any given service in a partition. This is a pre-existing constraint that gets enforced by the Normalize() method on that config entry type. --- agent/consul/state/config_entry.go | 67 +-------- agent/consul/state/config_entry_oss_test.go | 154 ++++++++++---------- agent/consul/state/peering.go | 123 ++++++++++------ agent/consul/state/peering_test.go | 79 +++++----- agent/rpc/peering/service_test.go | 90 ++++++------ agent/structs/peering.go | 6 - 6 files changed, 240 insertions(+), 279 deletions(-) diff --git a/agent/consul/state/config_entry.go b/agent/consul/state/config_entry.go index 195cc3b913..e2cd8600f1 100644 --- a/agent/consul/state/config_entry.go +++ b/agent/consul/state/config_entry.go @@ -603,6 +603,10 @@ func validateProposedConfigEntryInServiceGraph( wildcardEntMeta := kindName.WithWildcardNamespace() switch kindName.Kind { + case structs.ExportedServices, structs.MeshConfig: + // Exported services and mesh config do not influence discovery chains. + return nil + case structs.ProxyDefaults: // Check anything that has a discovery chain entry. In the future we could // somehow omit the ones that have a default protocol configured. @@ -1414,52 +1418,6 @@ func configEntryWithOverridesTxn( return configEntryTxn(tx, ws, kind, name, entMeta) } -// getExportedServicesConfigEntriesTxn fetches exported-service config entries and -// filters their exported services to only those that match serviceName and entMeta. -// Because the resulting config entries may have had their exported services modified, -// they *should not* be used in subsequent writes. -func getExportedServiceConfigEntriesTxn( - tx ReadTxn, - ws memdb.WatchSet, - serviceName string, - entMeta *acl.EnterpriseMeta, -) (uint64, []*structs.ExportedServicesConfigEntry, error) { - var exportedServicesEntries []*structs.ExportedServicesConfigEntry - // slice of names to match config entries against - matchCandidates := getExportedServicesMatchServiceNames(serviceName, entMeta) - // matcher func generator for currying the matcher func over EnterpriseMeta values - // from the associated config entry - matchFunc := func(matchMeta *acl.EnterpriseMeta) func(structs.ExportedService) bool { - return func(exportedService structs.ExportedService) bool { - matchSvcName := structs.NewServiceName(exportedService.Name, matchMeta) - for _, candidate := range matchCandidates { - if candidate.Matches(matchSvcName) { - return true - } - } - return false - } - } - idx, entries, err := configEntriesByKindTxn(tx, ws, structs.ExportedServices, entMeta) - if err != nil { - return 0, nil, err - } - for _, entry := range entries { - esEntry, ok := entry.(*structs.ExportedServicesConfigEntry) - if !ok { - return 0, nil, fmt.Errorf("type %T is not a %s config entry", esEntry, structs.ExportedServices) - } - // get a copy of the config entry with Services filtered to match serviceName - newEntry := filterExportedServices(esEntry, matchFunc(entry.GetEnterpriseMeta())) - // the filter will return a new entry, so checking to see if its services is empty says that there - // were matches and that we should include it in the results - if len(newEntry.Services) > 0 { - exportedServicesEntries = append(exportedServicesEntries, newEntry) - } - } - return idx, exportedServicesEntries, nil -} - // protocolForService returns the service graph protocol associated to the // provided service, checking all relevant config entries. func protocolForService( @@ -1502,23 +1460,6 @@ func protocolForService( return maxIdx, chain.Protocol, nil } -// filterExportedServices returns the slice of ExportedService that matc ffor matching service names -// returning a copy of entry with only the services that match one of the -// services in candidates. -func filterExportedServices( - entry *structs.ExportedServicesConfigEntry, - testFunc func(structs.ExportedService) bool, -) *structs.ExportedServicesConfigEntry { - newEntry := *entry - newEntry.Services = []structs.ExportedService{} - for _, ceSvc := range entry.Services { - if testFunc(ceSvc) { - newEntry.Services = append(newEntry.Services, ceSvc) - } - } - return &newEntry -} - func newConfigEntryQuery(c structs.ConfigEntry) configentry.KindName { return configentry.NewKindName(c.GetKind(), c.GetName(), c.GetEnterpriseMeta()) } diff --git a/agent/consul/state/config_entry_oss_test.go b/agent/consul/state/config_entry_oss_test.go index 9f6a1ef44d..4d121ba32d 100644 --- a/agent/consul/state/config_entry_oss_test.go +++ b/agent/consul/state/config_entry_oss_test.go @@ -40,120 +40,124 @@ func testIndexerTableConfigEntries() map[string]indexerTestCase { } } -func TestStore_ExportedServices(t *testing.T) { +func TestStore_peersForService(t *testing.T) { + queryName := "foo" + type testCase struct { name string - write []structs.ConfigEntry - query string - expect []*structs.ExportedServicesConfigEntry + write structs.ConfigEntry + expect []string } cases := []testCase{ { name: "empty everything", - write: []structs.ConfigEntry{}, - query: "foo", - expect: []*structs.ExportedServicesConfigEntry{}, + expect: nil, }, { - name: "no matching exported services", - write: []structs.ConfigEntry{ - &structs.ProxyConfigEntry{Name: "foo"}, - &structs.ProxyConfigEntry{Name: "bar"}, - &structs.ExportedServicesConfigEntry{ - Name: "baz", - Services: []structs.ExportedService{ - {Name: "baz"}, + name: "service is not exported", + write: &structs.ExportedServicesConfigEntry{ + Name: "default", + Services: []structs.ExportedService{ + { + Name: "not-" + queryName, + Consumers: []structs.ServiceConsumer{ + { + PeerName: "zip", + }, + }, }, }, }, - query: "foo", - expect: []*structs.ExportedServicesConfigEntry{}, + expect: nil, }, { - name: "exact match service name", - write: []structs.ConfigEntry{ - &structs.ExportedServicesConfigEntry{ - Name: "foo", - Services: []structs.ExportedService{ - {Name: "foo"}, + name: "wildcard name matches", + write: &structs.ExportedServicesConfigEntry{ + Name: "default", + Services: []structs.ExportedService{ + { + Name: "not-" + queryName, + Consumers: []structs.ServiceConsumer{ + { + PeerName: "zip", + }, + }, }, - }, - &structs.ExportedServicesConfigEntry{ - Name: "bar", - Services: []structs.ExportedService{ - {Name: "bar"}, - }, - }, - }, - query: "bar", - expect: []*structs.ExportedServicesConfigEntry{ - { - Name: "bar", - Services: []structs.ExportedService{ - {Name: "bar"}, + { + Name: structs.WildcardSpecifier, + Consumers: []structs.ServiceConsumer{ + { + PeerName: "bar", + }, + { + PeerName: "baz", + }, + }, }, }, }, + expect: []string{"bar", "baz"}, }, { - name: "wildcard match on service name", - write: []structs.ConfigEntry{ - &structs.ExportedServicesConfigEntry{ - Name: "foo", - Services: []structs.ExportedService{ - {Name: "foo"}, + name: "exact name takes precedence over wildcard", + write: &structs.ExportedServicesConfigEntry{ + Name: "default", + Services: []structs.ExportedService{ + { + Name: queryName, + Consumers: []structs.ServiceConsumer{ + { + PeerName: "baz", + }, + }, }, - }, - &structs.ExportedServicesConfigEntry{ - Name: "wildcard", - Services: []structs.ExportedService{ - {Name: structs.WildcardSpecifier}, - }, - }, - }, - query: "foo", - expect: []*structs.ExportedServicesConfigEntry{ - { - Name: "foo", - Services: []structs.ExportedService{ - {Name: "foo"}, - }, - }, - { - Name: "wildcard", - Services: []structs.ExportedService{ - {Name: structs.WildcardSpecifier}, + { + Name: structs.WildcardSpecifier, + Consumers: []structs.ServiceConsumer{ + { + PeerName: "zip", + }, + }, }, }, }, + expect: []string{"baz"}, }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { s := testStateStore(t) + var lastIdx uint64 - // Write the entries. - for idx, entry := range tc.write { - require.NoError(t, s.EnsureConfigEntry(uint64(idx+1), entry)) + // Write the entry. + if tc.write != nil { + require.NoError(t, tc.write.Normalize()) + require.NoError(t, tc.write.Validate()) + + lastIdx++ + require.NoError(t, s.EnsureConfigEntry(lastIdx, tc.write)) } // Read the entries back. tx := s.db.ReadTxn() defer tx.Abort() - idx, entries, err := getExportedServiceConfigEntriesTxn(tx, nil, tc.query, acl.DefaultEnterpriseMeta()) + + idx, peers, err := peersForServiceTxn(tx, nil, queryName, acl.DefaultEnterpriseMeta()) require.NoError(t, err) - require.Equal(t, uint64(len(tc.write)), idx) + + // This is a little weird, but when there are no results, the index returned should be the max index for the + // config entries table so that the caller can watch for changes to it + if len(peers) == 0 { + require.Equal(t, maxIndexTxn(tx, tableConfigEntries), idx) + } else { + require.Equal(t, lastIdx, idx) + } // Verify the result. - require.Len(t, entries, len(tc.expect)) - for idx, got := range entries { - // ignore raft fields - got.ModifyIndex = 0 - got.CreateIndex = 0 - require.Equal(t, tc.expect[idx], got) - } + require.Len(t, peers, len(tc.expect)) + require.Equal(t, tc.expect, peers) }) } } diff --git a/agent/consul/state/peering.go b/agent/consul/state/peering.go index e566532b68..835e9642dc 100644 --- a/agent/consul/state/peering.go +++ b/agent/consul/state/peering.go @@ -439,35 +439,37 @@ func (s *Store) exportedServicesForPeerTxn(ws memdb.WatchSet, tx ReadTxn, peerin // PeeringsForService returns the list of peerings that are associated with the service name provided in the query. // This is used to configure connect proxies for a given service. The result is generated by querying for exported // service config entries and filtering for those that match the given service. +// // TODO(peering): this implementation does all of the work on read to materialize this list of peerings, we should explore // writing to a separate index that has service peerings prepared ahead of time should this become a performance bottleneck. func (s *Store) PeeringsForService(ws memdb.WatchSet, serviceName string, entMeta acl.EnterpriseMeta) (uint64, []*pbpeering.Peering, error) { tx := s.db.ReadTxn() defer tx.Abort() - // short-circuit if the service does not exist in the context of the query -- this prevents "leaking" services + // Short-circuit if the service does not exist in the context of the query -- this prevents "leaking" services // when there are wildcard rules in place. if svcIdx, svcExists, err := serviceExists(tx, ws, serviceName, &entMeta, ""); err != nil { return 0, nil, fmt.Errorf("failed to check if service exists: %w", err) + } else if !svcExists { - // if the service does not exist, return the max index for the services table so caller can watch for changes + // If the service does not exist, return the max index for the services table so caller can watch for changes. return svcIdx, nil, nil + } - // config entries must be defined in the default namespace, so we only need the partition here - meta := structs.DefaultEnterpriseMetaInPartition(entMeta.PartitionOrDefault()) - // return the idx of the config entry that was last modified so caller can watch for changes - idx, peeredServices, err := readPeeredServicesFromConfigEntriesTxn(tx, ws, serviceName, meta) + + // Return the idx of the config entry so the caller can watch for changes. + idx, peerNames, err := peersForServiceTxn(tx, ws, serviceName, &entMeta) if err != nil { - return 0, nil, fmt.Errorf("failed to read peered services for service name: %w", err) + return 0, nil, fmt.Errorf("failed to read peers for service name %q: %w", serviceName, err) } var peerings []*pbpeering.Peering - // lookup the peering for each matching peered service - for _, peeredService := range peeredServices { + // Lookup and return the peering corresponding to each name. + for _, name := range peerNames { readQuery := Query{ - Value: peeredService.PeerName, - EnterpriseMeta: peeredService.Name.EnterpriseMeta, + Value: name, + EnterpriseMeta: *structs.NodeEnterpriseMetaInPartition(entMeta.PartitionOrDefault()), } _, peering, err := peeringReadTxn(tx, ws, readQuery) if err != nil { @@ -478,7 +480,6 @@ func (s *Store) PeeringsForService(ws memdb.WatchSet, serviceName string, entMet } peerings = append(peerings, peering) } - // see note above about idx return idx, peerings, nil } @@ -597,50 +598,80 @@ func (r *Restore) PeeringTrustBundle(ptb *pbpeering.PeeringTrustBundle) error { return nil } -// readPeeredServicesFromConfigEntriesTxn queries exported-service config entries to return peers for serviceName -// in the form of a []structs.PeeredService. -func readPeeredServicesFromConfigEntriesTxn( +// peersForServiceTxn returns the names of all peers that a service is exported to. +func peersForServiceTxn( tx ReadTxn, ws memdb.WatchSet, serviceName string, entMeta *acl.EnterpriseMeta, -) (uint64, []structs.PeeredService, error) { - var results []structs.PeeredService +) (uint64, []string, error) { + // Exported service config entries are scoped to partitions so they are in the default namespace. + partitionMeta := structs.DefaultEnterpriseMetaInPartition(entMeta.PartitionOrDefault()) - // Get all exported-service config entries for that have exports for serviceName. This assumes the result - // has exported services filtered to only those matching serviceName so no futher filtering is needed. - idx, exportedServicesEntries, err := getExportedServiceConfigEntriesTxn(tx, ws, serviceName, entMeta) + idx, rawEntry, err := configEntryTxn(tx, ws, structs.ExportedServices, partitionMeta.PartitionOrDefault(), partitionMeta) if err != nil { return 0, nil, err } + if rawEntry == nil { + return idx, nil, err + } - // dedupe results by peer name - resultSet := make(map[string]struct{}) - // filter entries to only those that have a peer consumer defined - for _, entry := range exportedServicesEntries { - for _, service := range entry.Services { - // entries must have consumers - if service.Consumers == nil || len(service.Consumers) == 0 { - continue - } - for _, consumer := range service.Consumers { - // and consumers must have a peer - if consumer.PeerName == "" { - continue - } - // if we get here, we have a peer consumer, but we should dedupe peer names, so skip if it's already in the set - if _, ok := resultSet[consumer.PeerName]; ok { - continue - } + entry, ok := rawEntry.(*structs.ExportedServicesConfigEntry) + if !ok { + return 0, nil, fmt.Errorf("unexpected type %T for pbpeering.Peering index", rawEntry) + } - // if we got here, we can add to the result set - resultSet[consumer.PeerName] = struct{}{} - result := structs.PeeredService{ - Name: structs.NewServiceName(serviceName, entry.GetEnterpriseMeta()), - PeerName: consumer.PeerName, - } - results = append(results, result) - } + var ( + wildcardNamespaceIdx = -1 + wildcardServiceIdx = -1 + exactMatchIdx = -1 + ) + + // Ensure the metadata is defaulted since we make assertions against potentially empty values below. + // In OSS this is a no-op. + if entMeta == nil { + entMeta = acl.DefaultEnterpriseMeta() + } + entMeta.Normalize() + + // Services can be exported via wildcards or by their exact name: + // Namespace: *, Service: * + // Namespace: Exact, Service: * + // Namespace: Exact, Service: Exact + for i, service := range entry.Services { + switch { + case service.Namespace == structs.WildcardSpecifier: + wildcardNamespaceIdx = i + + case service.Name == structs.WildcardSpecifier && service.Namespace == entMeta.NamespaceOrEmpty(): + wildcardServiceIdx = i + + case service.Name == serviceName && service.Namespace == entMeta.NamespaceOrEmpty(): + exactMatchIdx = i + } + } + + var results []string + + // Prefer the exact match over the wildcard match. This matches how we handle intention precedence. + var targetIdx int + switch { + case exactMatchIdx >= 0: + targetIdx = exactMatchIdx + + case wildcardServiceIdx >= 0: + targetIdx = wildcardServiceIdx + + case wildcardNamespaceIdx >= 0: + targetIdx = wildcardNamespaceIdx + + default: + return idx, results, nil + } + + for _, c := range entry.Services[targetIdx].Consumers { + if c.PeerName != "" { + results = append(results, c.PeerName) } } return idx, results, nil diff --git a/agent/consul/state/peering_test.go b/agent/consul/state/peering_test.go index 53b80cb9c8..018cf81642 100644 --- a/agent/consul/state/peering_test.go +++ b/agent/consul/state/peering_test.go @@ -907,7 +907,7 @@ func TestStateStore_PeeringsForService(t *testing.T) { name string services []structs.ServiceName peerings []*pbpeering.Peering - entries []*structs.ExportedServicesConfigEntry + entry *structs.ExportedServicesConfigEntry query []string expect [][]*pbpeering.Peering expectIdx uint64 @@ -945,9 +945,10 @@ func TestStateStore_PeeringsForService(t *testing.T) { } // Write the config entries. - for _, entry := range tc.entries { + if tc.entry != nil { lastIdx++ - require.NoError(t, s.EnsureConfigEntry(lastIdx, entry)) + require.NoError(t, tc.entry.Normalize()) + require.NoError(t, s.EnsureConfigEntry(lastIdx, tc.entry)) } // Query for peers. @@ -976,7 +977,7 @@ func TestStateStore_PeeringsForService(t *testing.T) { {Name: "foo"}, }, peerings: []*pbpeering.Peering{}, - entries: []*structs.ExportedServicesConfigEntry{}, + entry: nil, query: []string{"foo"}, expect: [][]*pbpeering.Peering{{}}, }, @@ -986,7 +987,7 @@ func TestStateStore_PeeringsForService(t *testing.T) { {Name: "foo"}, }, peerings: []*pbpeering.Peering{}, - entries: []*structs.ExportedServicesConfigEntry{}, + entry: nil, query: []string{"bar"}, expect: [][]*pbpeering.Peering{{}}, expectIdx: uint64(2), // catalog services max index @@ -1001,24 +1002,22 @@ func TestStateStore_PeeringsForService(t *testing.T) { {Name: "peer1", State: pbpeering.PeeringState_INITIAL}, {Name: "peer2", State: pbpeering.PeeringState_INITIAL}, }, - entries: []*structs.ExportedServicesConfigEntry{ - { - Name: "ce1", - Services: []structs.ExportedService{ - { - Name: "foo", - Consumers: []structs.ServiceConsumer{ - { - PeerName: "peer1", - }, + entry: &structs.ExportedServicesConfigEntry{ + Name: "default", + Services: []structs.ExportedService{ + { + Name: "foo", + Consumers: []structs.ServiceConsumer{ + { + PeerName: "peer1", }, }, - { - Name: "bar", - Consumers: []structs.ServiceConsumer{ - { - PeerName: "peer2", - }, + }, + { + Name: "bar", + Consumers: []structs.ServiceConsumer{ + { + PeerName: "peer2", }, }, }, @@ -1046,27 +1045,25 @@ func TestStateStore_PeeringsForService(t *testing.T) { {Name: "peer2", State: pbpeering.PeeringState_INITIAL}, {Name: "peer3", State: pbpeering.PeeringState_INITIAL}, }, - entries: []*structs.ExportedServicesConfigEntry{ - { - Name: "ce1", - Services: []structs.ExportedService{ - { - Name: "*", - Consumers: []structs.ServiceConsumer{ - { - PeerName: "peer1", - }, - { - PeerName: "peer2", - }, + entry: &structs.ExportedServicesConfigEntry{ + Name: "default", + Services: []structs.ExportedService{ + { + Name: "*", + Consumers: []structs.ServiceConsumer{ + { + PeerName: "peer1", + }, + { + PeerName: "peer2", }, }, - { - Name: "bar", - Consumers: []structs.ServiceConsumer{ - { - PeerName: "peer3", - }, + }, + { + Name: "bar", + Consumers: []structs.ServiceConsumer{ + { + PeerName: "peer3", }, }, }, @@ -1079,8 +1076,6 @@ func TestStateStore_PeeringsForService(t *testing.T) { {Name: "peer2", State: pbpeering.PeeringState_INITIAL}, }, { - {Name: "peer1", State: pbpeering.PeeringState_INITIAL}, - {Name: "peer2", State: pbpeering.PeeringState_INITIAL}, {Name: "peer3", State: pbpeering.PeeringState_INITIAL}, }, }, diff --git a/agent/rpc/peering/service_test.go b/agent/rpc/peering/service_test.go index 0d30987da5..c1bb121ec0 100644 --- a/agent/rpc/peering/service_test.go +++ b/agent/rpc/peering/service_test.go @@ -359,24 +359,22 @@ func TestPeeringService_TrustBundleRead(t *testing.T) { } func TestPeeringService_TrustBundleListByService(t *testing.T) { - // test executes the following scenario: - // 0 - initial setup test server, state store, RPC client, verify empty results - // 1 - create a service, verify results still empty - // 2 - create a peering, verify results still empty - // 3 - create a config entry, verify results still empty - // 4 - create trust bundles, verify bundles are returned - // 5 - delete the config entry, verify results empty - // 6 - restore config entry, verify bundles are returned - // 7 - add peering, trust bundles, wildcard config entry, verify updated results are present - // 8 - delete first config entry, verify bundles are returned - // 9 - delete the service, verify results empty + // Test executes the following scenario: + // 0 - Initial setup test server, state store, RPC client, verify empty results + // 1 - Create a service, verify results still empty + // 2 - Create a peering, verify results still empty + // 3 - Create a config entry, verify results still empty + // 4 - Create trust bundles, verify bundles are returned + // 5 - Delete the config entry, verify results empty + // 6 - Restore config entry, verify bundles are returned + // 7 - Add a second peering that the test service is not exported to + // 8 - Export the service to the new peering + // 9 - Delete the service // Note: these steps are dependent on each other by design so that we can verify that // combinations of services, peerings, trust bundles, and config entries all affect results - // fixed for the test nodeName := "test-node" - // keep track of index across steps var lastIdx uint64 // Create test server @@ -445,6 +443,7 @@ func TestPeeringService_TrustBundleListByService(t *testing.T) { // Write any config entries for _, entry := range deps.entries { idx++ + require.NoError(t, entry.Normalize()) require.NoError(t, store.EnsureConfigEntry(idx, entry)) } @@ -460,6 +459,8 @@ func TestPeeringService_TrustBundleListByService(t *testing.T) { // TODO(peering): see note on newTestServer, once we have a better server mock, // we should add functionality here to verify errors from backend verify := func(t *testing.T, tc *testCase) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) t.Cleanup(cancel) @@ -478,7 +479,7 @@ func TestPeeringService_TrustBundleListByService(t *testing.T) { // Execute scenario steps // ---------------------- - // 0 - initial empty state + // 0 - Initial empty state. // ----------------------- verify(t, &testCase{ req: &pbpeering.TrustBundleListByServiceRequest{ @@ -489,7 +490,7 @@ func TestPeeringService_TrustBundleListByService(t *testing.T) { }, }) - // 1 - create a service, verify results still empty + // 1 - Create a service, verify results still empty. // ------------------------------------------------ lastIdx = setup(t, lastIdx, testDeps{services: []string{"foo"}}) verify(t, &testCase{ @@ -501,7 +502,7 @@ func TestPeeringService_TrustBundleListByService(t *testing.T) { }, }) - // 2 - create a peering, verify results still empty + // 2 - Create a peering, verify results still empty. // ------------------------------------------------ lastIdx = setup(t, lastIdx, testDeps{ peerings: []*pbpeering.Peering{ @@ -522,12 +523,12 @@ func TestPeeringService_TrustBundleListByService(t *testing.T) { }, }) - // 3 - create a config entry, verify results still empty + // 3 - Create a config entry, verify results still empty. // ----------------------------------------------------- lastIdx = setup(t, lastIdx, testDeps{ entries: []*structs.ExportedServicesConfigEntry{ { - Name: "export-foo", + Name: "default", Services: []structs.ExportedService{ { Name: "foo", @@ -550,7 +551,7 @@ func TestPeeringService_TrustBundleListByService(t *testing.T) { }, }) - // 4 - create trust bundles, verify bundles are returned + // 4 - Create trust bundles, verify bundles are returned. // ----------------------------------------------------- lastIdx = setup(t, lastIdx, testDeps{ bundles: []*pbpeering.PeeringTrustBundle{ @@ -576,10 +577,10 @@ func TestPeeringService_TrustBundleListByService(t *testing.T) { }, }) - // 5 - delete the config entry, verify results empty + // 5 - Delete the config entry, verify results empty. // ------------------------------------------------- lastIdx++ - require.NoError(t, store.DeleteConfigEntry(lastIdx, structs.ExportedServices, "export-foo", nil)) + require.NoError(t, store.DeleteConfigEntry(lastIdx, structs.ExportedServices, "default", nil)) verify(t, &testCase{ req: &pbpeering.TrustBundleListByServiceRequest{ ServiceName: "foo", @@ -589,12 +590,12 @@ func TestPeeringService_TrustBundleListByService(t *testing.T) { }, }) - // 6 - restore config entry, verify bundles are returned + // 6 - Restore config entry, verify bundles are returned. // ----------------------------------------------------- lastIdx = setup(t, lastIdx, testDeps{ entries: []*structs.ExportedServicesConfigEntry{ { - Name: "export-foo", + Name: "default", Services: []structs.ExportedService{ { Name: "foo", @@ -621,10 +622,9 @@ func TestPeeringService_TrustBundleListByService(t *testing.T) { }, }) - // 7 - add peering, trust bundles, wildcard config entry, verify updated results are present + // 7 - Add new peer and trust bundle. It should be ignored because foo is not exported to it. // ----------------------------------------------------------------------------------------- lastIdx = setup(t, lastIdx, testDeps{ - services: []string{"bar"}, peerings: []*pbpeering.Peering{ { Name: "peer2", @@ -633,20 +633,6 @@ func TestPeeringService_TrustBundleListByService(t *testing.T) { PeerServerAddresses: []string{"peer2-addr"}, }, }, - entries: []*structs.ExportedServicesConfigEntry{ - { - Name: "export-all", - Services: []structs.ExportedService{ - { - Name: structs.WildcardSpecifier, - Consumers: []structs.ServiceConsumer{ - {PeerName: "peer1"}, - {PeerName: "peer2"}, - }, - }, - }, - }, - }, bundles: []*pbpeering.PeeringTrustBundle{ { TrustDomain: "peer2.com", @@ -666,18 +652,28 @@ func TestPeeringService_TrustBundleListByService(t *testing.T) { PeerName: "peer1", RootPEMs: []string{"peer1-root-1"}, }, - { - TrustDomain: "peer2.com", - PeerName: "peer2", - RootPEMs: []string{"peer2-root-1"}, - }, }, }, }) - // 8 - delete first config entry, verify bundles are returned - lastIdx++ - require.NoError(t, store.DeleteConfigEntry(lastIdx, structs.ExportedServices, "export-foo", nil)) + // 8 - Replace config entry to export all services to both peers + // ----------------------------------------------------------------------------------------- + lastIdx = setup(t, lastIdx, testDeps{ + entries: []*structs.ExportedServicesConfigEntry{ + { + Name: "default", + Services: []structs.ExportedService{ + { + Name: structs.WildcardSpecifier, + Consumers: []structs.ServiceConsumer{ + {PeerName: "peer1"}, + {PeerName: "peer2"}, + }, + }, + }, + }, + }, + }) verify(t, &testCase{ req: &pbpeering.TrustBundleListByServiceRequest{ ServiceName: "foo", diff --git a/agent/structs/peering.go b/agent/structs/peering.go index bd0351bb36..1cf74bdaad 100644 --- a/agent/structs/peering.go +++ b/agent/structs/peering.go @@ -8,12 +8,6 @@ type PeeringToken struct { PeerID string } -// PeeredService is a service that has been configured with an exported-service config entry to be exported to a peer. -type PeeredService struct { - Name ServiceName - PeerName string -} - // NOTE: this is not serialized via msgpack so it can be changed without concern. type ExportedServiceList struct { // Services is a list of exported services that apply to both standard From 647c57a416a43eed0f24dd4ceea2939784fd3130 Mon Sep 17 00:00:00 2001 From: freddygv Date: Mon, 23 May 2022 17:57:42 -0600 Subject: [PATCH 2/3] Add agent cache-type for TrustBundleListByService There are a handful of changes in this commit: * When querying trust bundles for a service we need to be able to specify the namespace of the service. * The endpoint needs to track the index because the cache watches use it. * Extracted bulk of the endpoint's logic to a state store function so that index tracking could be tested more easily. * Removed check for service existence, deferring that sort of work to ACL authz * Added the cache type --- acl/enterprisemeta_oss.go | 4 + agent/agent.go | 2 + agent/cache-types/trust_bundle_test.go | 6 +- agent/cache-types/trust_bundles.go | 50 +++ agent/cache-types/trust_bundles_test.go | 152 +++++++++ agent/consul/state/catalog.go | 18 -- agent/consul/state/peering.go | 61 +++- agent/consul/state/peering_test.go | 227 +++++++++++++- agent/rpc/peering/service.go | 29 +- agent/rpc/peering/service_test.go | 399 +++++------------------- proto/pbpeering/peering.go | 29 ++ proto/pbpeering/peering.pb.go | 399 +++++++++++++----------- proto/pbpeering/peering.proto | 8 +- 13 files changed, 794 insertions(+), 590 deletions(-) create mode 100644 agent/cache-types/trust_bundles.go create mode 100644 agent/cache-types/trust_bundles_test.go diff --git a/acl/enterprisemeta_oss.go b/acl/enterprisemeta_oss.go index 44075a44f7..7623709a93 100644 --- a/acl/enterprisemeta_oss.go +++ b/acl/enterprisemeta_oss.go @@ -58,6 +58,10 @@ func (m *EnterpriseMeta) NamespaceOrDefault() string { return DefaultNamespaceName } +func EqualNamespaces(_, _ string) bool { + return true +} + func NamespaceOrDefault(_ string) string { return DefaultNamespaceName } diff --git a/agent/agent.go b/agent/agent.go index cb6ba1a940..ebaa339f13 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -4103,6 +4103,8 @@ func (a *Agent) registerCache() { a.cache.RegisterType(cachetype.FederationStateListMeshGatewaysName, &cachetype.FederationStateListMeshGateways{RPC: a}) + a.cache.RegisterType(cachetype.TrustBundleListName, &cachetype.TrustBundles{Client: a.rpcClientPeering}) + a.registerEntCache() } diff --git a/agent/cache-types/trust_bundle_test.go b/agent/cache-types/trust_bundle_test.go index 8e18e69fe7..fa3d016a29 100644 --- a/agent/cache-types/trust_bundle_test.go +++ b/agent/cache-types/trust_bundle_test.go @@ -11,7 +11,7 @@ import ( "github.com/stretchr/testify/require" ) -func TestTrustBundles(t *testing.T) { +func TestTrustBundle(t *testing.T) { client := NewMockTrustBundleReader(t) typ := &TrustBundle{Client: client} @@ -43,7 +43,7 @@ func TestTrustBundles(t *testing.T) { }, result) } -func TestTrustBundles_badReqType(t *testing.T) { +func TestTrustBundle_badReqType(t *testing.T) { client := pbpeering.NewPeeringServiceClient(nil) typ := &TrustBundle{Client: client} @@ -55,7 +55,7 @@ func TestTrustBundles_badReqType(t *testing.T) { } // This test asserts that we can continuously poll this cache type, given that it doesn't support blocking. -func TestTrustBundles_MultipleUpdates(t *testing.T) { +func TestTrustBundle_MultipleUpdates(t *testing.T) { c := cache.New(cache.Options{}) client := NewMockTrustBundleReader(t) diff --git a/agent/cache-types/trust_bundles.go b/agent/cache-types/trust_bundles.go new file mode 100644 index 0000000000..f4cd2d3c18 --- /dev/null +++ b/agent/cache-types/trust_bundles.go @@ -0,0 +1,50 @@ +package cachetype + +import ( + "context" + "fmt" + + "github.com/hashicorp/consul/agent/cache" + "github.com/hashicorp/consul/proto/pbpeering" + "google.golang.org/grpc" +) + +// Recommended name for registration. +const TrustBundleListName = "trust-bundles" + +// TrustBundles supports fetching discovering service instances via prepared +// queries. +type TrustBundles struct { + RegisterOptionsNoRefresh + Client TrustBundleLister +} + +type TrustBundleLister interface { + TrustBundleListByService( + ctx context.Context, in *pbpeering.TrustBundleListByServiceRequest, opts ...grpc.CallOption, + ) (*pbpeering.TrustBundleListByServiceResponse, error) +} + +func (t *TrustBundles) Fetch(_ cache.FetchOptions, req cache.Request) (cache.FetchResult, error) { + var result cache.FetchResult + + // The request should be a TrustBundleListByServiceRequest. + // We do not need to make a copy of this request type like in other cache types + // because the RequestInfo is synthetic. + reqReal, ok := req.(*pbpeering.TrustBundleListByServiceRequest) + if !ok { + return result, fmt.Errorf( + "Internal cache failure: request wrong type: %T", req) + } + + // Fetch + reply, err := t.Client.TrustBundleListByService(context.Background(), reqReal) + if err != nil { + return result, err + } + + result.Value = reply + result.Index = reply.Index + + return result, nil +} diff --git a/agent/cache-types/trust_bundles_test.go b/agent/cache-types/trust_bundles_test.go new file mode 100644 index 0000000000..451508b0d0 --- /dev/null +++ b/agent/cache-types/trust_bundles_test.go @@ -0,0 +1,152 @@ +package cachetype + +import ( + "context" + "testing" + "time" + + "github.com/hashicorp/consul/agent/cache" + "github.com/hashicorp/consul/proto/pbpeering" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" +) + +func TestTrustBundles(t *testing.T) { + client := NewMockTrustBundleLister(t) + typ := &TrustBundles{Client: client} + + resp := &pbpeering.TrustBundleListByServiceResponse{ + Index: 48, + Bundles: []*pbpeering.PeeringTrustBundle{ + { + PeerName: "peer1", + RootPEMs: []string{"peer1-roots"}, + }, + }, + } + + // Expect the proper call. + // This also returns the canned response above. + client.On("TrustBundleListByService", mock.Anything, mock.Anything). + Run(func(args mock.Arguments) { + req := args.Get(1).(*pbpeering.TrustBundleListByServiceRequest) + require.Equal(t, "foo", req.ServiceName) + }). + Return(resp, nil) + + // Fetch and assert against the result. + result, err := typ.Fetch(cache.FetchOptions{}, &pbpeering.TrustBundleListByServiceRequest{ + ServiceName: "foo", + }) + require.NoError(t, err) + require.Equal(t, cache.FetchResult{ + Value: resp, + Index: 48, + }, result) +} + +func TestTrustBundles_badReqType(t *testing.T) { + client := pbpeering.NewPeeringServiceClient(nil) + typ := &TrustBundles{Client: client} + + // Fetch + _, err := typ.Fetch(cache.FetchOptions{}, cache.TestRequest( + t, cache.RequestInfo{Key: "foo", MinIndex: 64})) + require.Error(t, err) + require.Contains(t, err.Error(), "wrong type") +} + +// This test asserts that we can continuously poll this cache type, given that it doesn't support blocking. +func TestTrustBundles_MultipleUpdates(t *testing.T) { + c := cache.New(cache.Options{}) + + client := NewMockTrustBundleLister(t) + + // On each mock client call to TrustBundleList by service we will increment the index by 1 + // to simulate new data arriving. + resp := &pbpeering.TrustBundleListByServiceResponse{ + Index: uint64(0), + } + + client.On("TrustBundleListByService", mock.Anything, mock.Anything). + Run(func(args mock.Arguments) { + req := args.Get(1).(*pbpeering.TrustBundleListByServiceRequest) + require.Equal(t, "foo", req.ServiceName) + + // Increment on each call. + resp.Index++ + }). + Return(resp, nil) + + c.RegisterType(TrustBundleListName, &TrustBundles{Client: client}) + + ch := make(chan cache.UpdateEvent) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + t.Cleanup(cancel) + + err := c.Notify(ctx, TrustBundleListName, &pbpeering.TrustBundleListByServiceRequest{ServiceName: "foo"}, "updates", ch) + require.NoError(t, err) + + i := uint64(1) + for { + select { + case <-ctx.Done(): + return + case update := <-ch: + // Expect to receive updates for increasing indexes serially. + resp := update.Result.(*pbpeering.TrustBundleListByServiceResponse) + require.Equal(t, i, resp.Index) + i++ + + if i > 3 { + return + } + } + } +} + +// MockTrustBundleLister is an autogenerated mock type for the TrustBundleLister type +type MockTrustBundleLister struct { + mock.Mock +} + +// TrustBundleListByService provides a mock function with given fields: ctx, in, opts +func (_m *MockTrustBundleLister) TrustBundleListByService(ctx context.Context, in *pbpeering.TrustBundleListByServiceRequest, opts ...grpc.CallOption) (*pbpeering.TrustBundleListByServiceResponse, error) { + _va := make([]interface{}, len(opts)) + for _i := range opts { + _va[_i] = opts[_i] + } + var _ca []interface{} + _ca = append(_ca, ctx, in) + _ca = append(_ca, _va...) + ret := _m.Called(_ca...) + + var r0 *pbpeering.TrustBundleListByServiceResponse + if rf, ok := ret.Get(0).(func(context.Context, *pbpeering.TrustBundleListByServiceRequest, ...grpc.CallOption) *pbpeering.TrustBundleListByServiceResponse); ok { + r0 = rf(ctx, in, opts...) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*pbpeering.TrustBundleListByServiceResponse) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, *pbpeering.TrustBundleListByServiceRequest, ...grpc.CallOption) error); ok { + r1 = rf(ctx, in, opts...) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// NewMockTrustBundleLister creates a new instance of MockTrustBundleLister. It also registers the testing.TB interface on the mock and a cleanup function to assert the mocks expectations. +func NewMockTrustBundleLister(t testing.TB) *MockTrustBundleLister { + mock := &MockTrustBundleLister{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/agent/consul/state/catalog.go b/agent/consul/state/catalog.go index ae16267e4b..ab4797fda8 100644 --- a/agent/consul/state/catalog.go +++ b/agent/consul/state/catalog.go @@ -1164,24 +1164,6 @@ func serviceListTxn(tx ReadTxn, ws memdb.WatchSet, entMeta *acl.EnterpriseMeta, return idx, results, nil } -func serviceExists(tx ReadTxn, ws memdb.WatchSet, name string, entMeta *acl.EnterpriseMeta, peerName string) (uint64, bool, error) { - idx := catalogServicesMaxIndex(tx, entMeta, peerName) - q := Query{ - Value: name, - EnterpriseMeta: *entMeta, - PeerName: peerName, - } - watchCh, existing, err := tx.FirstWatch(tableServices, indexService, q) - if err != nil { - return idx, false, fmt.Errorf("failed querying for service: %s", err) - } - ws.Add(watchCh) - if existing == nil { - return idx, false, nil - } - return idx, true, nil -} - // ServicesByNodeMeta returns all services, filtered by the given node metadata. func (s *Store) ServicesByNodeMeta(ws memdb.WatchSet, filters map[string]string, entMeta *acl.EnterpriseMeta, peerName string) (uint64, structs.Services, error) { tx := s.db.Txn(false) diff --git a/agent/consul/state/peering.go b/agent/consul/state/peering.go index 835e9642dc..d18d0ab808 100644 --- a/agent/consul/state/peering.go +++ b/agent/consul/state/peering.go @@ -2,6 +2,7 @@ package state import ( "fmt" + "strings" "github.com/golang/protobuf/proto" "github.com/hashicorp/go-memdb" @@ -446,19 +447,12 @@ func (s *Store) PeeringsForService(ws memdb.WatchSet, serviceName string, entMet tx := s.db.ReadTxn() defer tx.Abort() - // Short-circuit if the service does not exist in the context of the query -- this prevents "leaking" services - // when there are wildcard rules in place. - if svcIdx, svcExists, err := serviceExists(tx, ws, serviceName, &entMeta, ""); err != nil { - return 0, nil, fmt.Errorf("failed to check if service exists: %w", err) - - } else if !svcExists { - // If the service does not exist, return the max index for the services table so caller can watch for changes. - return svcIdx, nil, nil - - } + return peeringsForServiceTxn(tx, ws, serviceName, entMeta) +} +func peeringsForServiceTxn(tx ReadTxn, ws memdb.WatchSet, serviceName string, entMeta acl.EnterpriseMeta) (uint64, []*pbpeering.Peering, error) { // Return the idx of the config entry so the caller can watch for changes. - idx, peerNames, err := peersForServiceTxn(tx, ws, serviceName, &entMeta) + maxIdx, peerNames, err := peersForServiceTxn(tx, ws, serviceName, &entMeta) if err != nil { return 0, nil, fmt.Errorf("failed to read peers for service name %q: %w", serviceName, err) } @@ -471,16 +465,49 @@ func (s *Store) PeeringsForService(ws memdb.WatchSet, serviceName string, entMet Value: name, EnterpriseMeta: *structs.NodeEnterpriseMetaInPartition(entMeta.PartitionOrDefault()), } - _, peering, err := peeringReadTxn(tx, ws, readQuery) + idx, peering, err := peeringReadTxn(tx, ws, readQuery) if err != nil { return 0, nil, fmt.Errorf("failed to read peering: %w", err) } + if idx > maxIdx { + maxIdx = idx + } if peering == nil { continue } peerings = append(peerings, peering) } - return idx, peerings, nil + return maxIdx, peerings, nil +} + +// TrustBundleListByService returns the trust bundles for all peers that the given service is exported to. +func (s *Store) TrustBundleListByService(ws memdb.WatchSet, service string, entMeta acl.EnterpriseMeta) (uint64, []*pbpeering.PeeringTrustBundle, error) { + tx := s.db.ReadTxn() + defer tx.Abort() + + maxIdx, peers, err := peeringsForServiceTxn(tx, ws, service, entMeta) + if err != nil { + return 0, nil, fmt.Errorf("failed to get peers for service %s: %v", service, err) + } + + var resp []*pbpeering.PeeringTrustBundle + for _, peer := range peers { + pq := Query{ + Value: strings.ToLower(peer.Name), + EnterpriseMeta: *structs.NodeEnterpriseMetaInPartition(entMeta.PartitionOrDefault()), + } + idx, trustBundle, err := peeringTrustBundleReadTxn(tx, ws, pq) + if err != nil { + return 0, nil, fmt.Errorf("failed to read trust bundle for peer %s: %v", peer.Name, err) + } + if idx > maxIdx { + maxIdx = idx + } + if trustBundle != nil { + resp = append(resp, trustBundle) + } + } + return maxIdx, resp, nil } // PeeringTrustBundleRead returns the peering trust bundle for the peer name given as the query value. @@ -488,6 +515,10 @@ func (s *Store) PeeringTrustBundleRead(ws memdb.WatchSet, q Query) (uint64, *pbp tx := s.db.ReadTxn() defer tx.Abort() + return peeringTrustBundleReadTxn(tx, ws, q) +} + +func peeringTrustBundleReadTxn(tx ReadTxn, ws memdb.WatchSet, q Query) (uint64, *pbpeering.PeeringTrustBundle, error) { watchCh, ptbRaw, err := tx.FirstWatch(tablePeeringTrustBundles, indexID, q) if err != nil { return 0, nil, fmt.Errorf("failed peering trust bundle lookup: %w", err) @@ -643,10 +674,10 @@ func peersForServiceTxn( case service.Namespace == structs.WildcardSpecifier: wildcardNamespaceIdx = i - case service.Name == structs.WildcardSpecifier && service.Namespace == entMeta.NamespaceOrEmpty(): + case service.Name == structs.WildcardSpecifier && acl.EqualNamespaces(service.Namespace, entMeta.NamespaceOrDefault()): wildcardServiceIdx = i - case service.Name == serviceName && service.Namespace == entMeta.NamespaceOrEmpty(): + case service.Name == serviceName && acl.EqualNamespaces(service.Namespace, entMeta.NamespaceOrDefault()): exactMatchIdx = i } } diff --git a/agent/consul/state/peering_test.go b/agent/consul/state/peering_test.go index 018cf81642..8056f77ecf 100644 --- a/agent/consul/state/peering_test.go +++ b/agent/consul/state/peering_test.go @@ -981,17 +981,6 @@ func TestStateStore_PeeringsForService(t *testing.T) { query: []string{"foo"}, expect: [][]*pbpeering.Peering{{}}, }, - { - name: "service does not exist", - services: []structs.ServiceName{ - {Name: "foo"}, - }, - peerings: []*pbpeering.Peering{}, - entry: nil, - query: []string{"bar"}, - expect: [][]*pbpeering.Peering{{}}, - expectIdx: uint64(2), // catalog services max index - }, { name: "config entry with exact service name", services: []structs.ServiceName{ @@ -1089,3 +1078,219 @@ func TestStateStore_PeeringsForService(t *testing.T) { }) } } + +func TestStore_TrustBundleListByService(t *testing.T) { + store := testStateStore(t) + entMeta := *acl.DefaultEnterpriseMeta() + + var lastIdx uint64 + ws := memdb.NewWatchSet() + + testutil.RunStep(t, "no results on initial setup", func(t *testing.T) { + idx, resp, err := store.TrustBundleListByService(ws, "foo", entMeta) + require.NoError(t, err) + require.Equal(t, lastIdx, idx) + require.Len(t, resp, 0) + }) + + testutil.RunStep(t, "registering service does not yield trust bundles", func(t *testing.T) { + lastIdx++ + require.NoError(t, store.EnsureNode(lastIdx, &structs.Node{ + Node: "my-node", + Address: "127.0.0.1", + })) + + lastIdx++ + require.NoError(t, store.EnsureService(lastIdx, "my-node", &structs.NodeService{ + ID: "foo-1", + Service: "foo", + Port: 8000, + })) + + require.False(t, watchFired(ws)) + + idx, resp, err := store.TrustBundleListByService(ws, "foo", entMeta) + require.NoError(t, err) + require.Len(t, resp, 0) + require.Equal(t, lastIdx-2, idx) + }) + + testutil.RunStep(t, "creating peering does not yield trust bundles", func(t *testing.T) { + lastIdx++ + require.NoError(t, store.PeeringWrite(lastIdx, &pbpeering.Peering{ + Name: "peer1", + })) + + // The peering is only watched after the service is exported via config entry. + require.False(t, watchFired(ws)) + + idx, resp, err := store.TrustBundleListByService(ws, "foo", entMeta) + require.NoError(t, err) + require.Equal(t, uint64(0), idx) + require.Len(t, resp, 0) + }) + + testutil.RunStep(t, "exporting the service does not yield trust bundles", func(t *testing.T) { + lastIdx++ + require.NoError(t, store.EnsureConfigEntry(lastIdx, &structs.ExportedServicesConfigEntry{ + Name: "default", + Services: []structs.ExportedService{ + { + Name: "foo", + Consumers: []structs.ServiceConsumer{ + { + PeerName: "peer1", + }, + }, + }, + }, + })) + + // The config entry is watched. + require.True(t, watchFired(ws)) + ws = memdb.NewWatchSet() + + idx, resp, err := store.TrustBundleListByService(ws, "foo", entMeta) + require.NoError(t, err) + require.Equal(t, lastIdx, idx) + require.Len(t, resp, 0) + }) + + testutil.RunStep(t, "trust bundles are returned after they are created", func(t *testing.T) { + lastIdx++ + require.NoError(t, store.PeeringTrustBundleWrite(lastIdx, &pbpeering.PeeringTrustBundle{ + TrustDomain: "peer1.com", + PeerName: "peer1", + RootPEMs: []string{"peer-root-1"}, + })) + + require.True(t, watchFired(ws)) + ws = memdb.NewWatchSet() + + idx, resp, err := store.TrustBundleListByService(ws, "foo", entMeta) + require.NoError(t, err) + require.Equal(t, lastIdx, idx) + require.Len(t, resp, 1) + require.Equal(t, []string{"peer-root-1"}, resp[0].RootPEMs) + }) + + testutil.RunStep(t, "trust bundles are not returned after unexporting service", func(t *testing.T) { + lastIdx++ + require.NoError(t, store.DeleteConfigEntry(lastIdx, structs.ExportedServices, "default", &entMeta)) + + require.True(t, watchFired(ws)) + ws = memdb.NewWatchSet() + + idx, resp, err := store.TrustBundleListByService(ws, "foo", entMeta) + require.NoError(t, err) + require.Equal(t, lastIdx, idx) + require.Len(t, resp, 0) + }) + + testutil.RunStep(t, "trust bundles are returned after config entry is restored", func(t *testing.T) { + lastIdx++ + require.NoError(t, store.EnsureConfigEntry(lastIdx, &structs.ExportedServicesConfigEntry{ + Name: "default", + Services: []structs.ExportedService{ + { + Name: "foo", + Consumers: []structs.ServiceConsumer{ + { + PeerName: "peer1", + }, + }, + }, + }, + })) + + require.True(t, watchFired(ws)) + ws = memdb.NewWatchSet() + + idx, resp, err := store.TrustBundleListByService(ws, "foo", entMeta) + require.NoError(t, err) + require.Equal(t, lastIdx, idx) + require.Len(t, resp, 1) + require.Equal(t, []string{"peer-root-1"}, resp[0].RootPEMs) + }) + + testutil.RunStep(t, "bundles for other peers are ignored", func(t *testing.T) { + lastIdx++ + require.NoError(t, store.PeeringWrite(lastIdx, &pbpeering.Peering{ + Name: "peer2", + })) + + lastIdx++ + require.NoError(t, store.PeeringTrustBundleWrite(lastIdx, &pbpeering.PeeringTrustBundle{ + TrustDomain: "peer2.com", + PeerName: "peer2", + RootPEMs: []string{"peer-root-2"}, + })) + + // No relevant changes. + require.False(t, watchFired(ws)) + ws = memdb.NewWatchSet() + + idx, resp, err := store.TrustBundleListByService(ws, "foo", entMeta) + require.NoError(t, err) + require.Equal(t, lastIdx-2, idx) + require.Len(t, resp, 1) + require.Equal(t, []string{"peer-root-1"}, resp[0].RootPEMs) + }) + + testutil.RunStep(t, "second bundle is returned when service is exported to that peer", func(t *testing.T) { + lastIdx++ + require.NoError(t, store.EnsureConfigEntry(lastIdx, &structs.ExportedServicesConfigEntry{ + Name: "default", + Services: []structs.ExportedService{ + { + Name: "foo", + Consumers: []structs.ServiceConsumer{ + { + PeerName: "peer1", + }, + { + PeerName: "peer2", + }, + }, + }, + }, + })) + + require.True(t, watchFired(ws)) + ws = memdb.NewWatchSet() + + idx, resp, err := store.TrustBundleListByService(ws, "foo", entMeta) + require.NoError(t, err) + require.Equal(t, lastIdx, idx) + require.Len(t, resp, 2) + require.Equal(t, []string{"peer-root-1"}, resp[0].RootPEMs) + require.Equal(t, []string{"peer-root-2"}, resp[1].RootPEMs) + }) + + testutil.RunStep(t, "deleting the peering excludes its trust bundle", func(t *testing.T) { + lastIdx++ + require.NoError(t, store.PeeringDelete(lastIdx, Query{Value: "peer1"})) + + require.True(t, watchFired(ws)) + ws = memdb.NewWatchSet() + + idx, resp, err := store.TrustBundleListByService(ws, "foo", entMeta) + require.NoError(t, err) + require.Equal(t, lastIdx, idx) + require.Len(t, resp, 1) + require.Equal(t, []string{"peer-root-2"}, resp[0].RootPEMs) + }) + + testutil.RunStep(t, "deleting the service does not excludes its trust bundle", func(t *testing.T) { + lastIdx++ + require.NoError(t, store.DeleteService(lastIdx, "my-node", "foo-1", &entMeta, "")) + + require.False(t, watchFired(ws)) + + idx, resp, err := store.TrustBundleListByService(ws, "foo", entMeta) + require.NoError(t, err) + require.Equal(t, lastIdx-1, idx) + require.Len(t, resp, 1) + require.Equal(t, []string{"peer-root-2"}, resp[0].RootPEMs) + }) +} diff --git a/agent/rpc/peering/service.go b/agent/rpc/peering/service.go index 1ff2b0fa0e..8e1dcbee3c 100644 --- a/agent/rpc/peering/service.go +++ b/agent/rpc/peering/service.go @@ -130,9 +130,9 @@ type Store interface { PeeringList(ws memdb.WatchSet, entMeta acl.EnterpriseMeta) (uint64, []*pbpeering.Peering, error) PeeringTrustBundleRead(ws memdb.WatchSet, q state.Query) (uint64, *pbpeering.PeeringTrustBundle, error) ExportedServicesForPeer(ws memdb.WatchSet, peerID string) (uint64, *structs.ExportedServiceList, error) - PeeringsForService(ws memdb.WatchSet, serviceName string, entMeta acl.EnterpriseMeta) (uint64, []*pbpeering.Peering, error) ServiceDump(ws memdb.WatchSet, kind structs.ServiceKind, useKind bool, entMeta *acl.EnterpriseMeta, peerName string) (uint64, structs.CheckServiceNodes, error) CAConfig(ws memdb.WatchSet) (uint64, *structs.CAConfiguration, error) + TrustBundleListByService(ws memdb.WatchSet, service string, entMeta acl.EnterpriseMeta) (uint64, []*pbpeering.PeeringTrustBundle, error) AbandonCh() <-chan struct{} } @@ -450,33 +450,16 @@ func (s *Service) TrustBundleListByService(ctx context.Context, req *pbpeering.T } defer metrics.MeasureSince([]string{"peering", "trust_bundle_list_by_service"}, time.Now()) - // TODO(peering): ACL check request token + // TODO(peering): ACL check request token for service:write on the service name // TODO(peering): handle blocking queries - entMeta := *structs.NodeEnterpriseMetaInPartition(req.Partition) - // TODO(peering): we're throwing away the index here that would tell us how to execute a blocking query - _, peers, err := s.Backend.Store().PeeringsForService(nil, req.ServiceName, entMeta) + entMeta := acl.NewEnterpriseMetaWithPartition(req.Partition, req.Namespace) + idx, bundles, err := s.Backend.Store().TrustBundleListByService(nil, req.ServiceName, entMeta) if err != nil { - return nil, fmt.Errorf("failed to get peers for service %s: %v", req.ServiceName, err) + return nil, err } - - trustBundles := []*pbpeering.PeeringTrustBundle{} - for _, peer := range peers { - q := state.Query{ - Value: strings.ToLower(peer.Name), - EnterpriseMeta: *structs.NodeEnterpriseMetaInPartition(req.Partition), - } - _, trustBundle, err := s.Backend.Store().PeeringTrustBundleRead(nil, q) - if err != nil { - return nil, fmt.Errorf("failed to read trust bundle for peer %s: %v", peer.Name, err) - } - - if trustBundle != nil { - trustBundles = append(trustBundles, trustBundle) - } - } - return &pbpeering.TrustBundleListByServiceResponse{Bundles: trustBundles}, nil + return &pbpeering.TrustBundleListByServiceResponse{Index: idx, Bundles: bundles}, nil } type BidirectionalStream interface { diff --git a/agent/rpc/peering/service_test.go b/agent/rpc/peering/service_test.go index c1bb121ec0..fc67b65baa 100644 --- a/agent/rpc/peering/service_test.go +++ b/agent/rpc/peering/service_test.go @@ -358,353 +358,98 @@ func TestPeeringService_TrustBundleRead(t *testing.T) { prototest.AssertDeepEqual(t, bundle, resp.Bundle) } +// Setup: +// - Peerings "foo" and "bar" with trust bundles saved +// - "api" service exported to both "foo" and "bar" +// - "web" service exported to "baz" func TestPeeringService_TrustBundleListByService(t *testing.T) { - // Test executes the following scenario: - // 0 - Initial setup test server, state store, RPC client, verify empty results - // 1 - Create a service, verify results still empty - // 2 - Create a peering, verify results still empty - // 3 - Create a config entry, verify results still empty - // 4 - Create trust bundles, verify bundles are returned - // 5 - Delete the config entry, verify results empty - // 6 - Restore config entry, verify bundles are returned - // 7 - Add a second peering that the test service is not exported to - // 8 - Export the service to the new peering - // 9 - Delete the service - // Note: these steps are dependent on each other by design so that we can verify that - // combinations of services, peerings, trust bundles, and config entries all affect results + s := newTestServer(t, nil) + store := s.Server.FSM().State() - nodeName := "test-node" + var lastIdx uint64 = 10 - var lastIdx uint64 - - // Create test server - // TODO(peering): see note on newTestServer, refactor to not use this - srv := newTestServer(t, nil) - store := srv.Server.FSM().State() - client := pbpeering.NewPeeringServiceClient(srv.ClientConn(t)) - - // Create a node up-front so that we can assign services to it if needed - svcNode := &structs.Node{Node: nodeName, Address: "127.0.0.1"} lastIdx++ - require.NoError(t, store.EnsureNode(lastIdx, svcNode)) + require.NoError(t, s.Server.FSM().State().PeeringWrite(lastIdx, &pbpeering.Peering{ + Name: "foo", + State: pbpeering.PeeringState_INITIAL, + PeerServerName: "test", + PeerServerAddresses: []string{"addr1"}, + })) - type testDeps struct { - services []string - peerings []*pbpeering.Peering - entries []*structs.ExportedServicesConfigEntry - bundles []*pbpeering.PeeringTrustBundle - } + lastIdx++ + require.NoError(t, s.Server.FSM().State().PeeringWrite(lastIdx, &pbpeering.Peering{ + Name: "bar", + State: pbpeering.PeeringState_INITIAL, + PeerServerName: "test-bar", + PeerServerAddresses: []string{"addr2"}, + })) - setup := func(t *testing.T, idx uint64, deps testDeps) uint64 { - // Create any services (and node) - if len(deps.services) >= 0 { - svcNode := &structs.Node{Node: nodeName, Address: "127.0.0.1"} - idx++ - require.NoError(t, store.EnsureNode(idx, svcNode)) + lastIdx++ + require.NoError(t, store.PeeringTrustBundleWrite(lastIdx, &pbpeering.PeeringTrustBundle{ + TrustDomain: "foo.com", + PeerName: "foo", + RootPEMs: []string{"foo-root-1"}, + })) - // Create the test services - for _, svc := range deps.services { - idx++ - require.NoError(t, store.EnsureService(idx, svcNode.Node, &structs.NodeService{ - ID: svc, - Service: svc, - Port: int(8000 + idx), - })) - } - } + lastIdx++ + require.NoError(t, store.PeeringTrustBundleWrite(lastIdx, &pbpeering.PeeringTrustBundle{ + TrustDomain: "bar.com", + PeerName: "bar", + RootPEMs: []string{"bar-root-1"}, + })) - // Insert any peerings - for _, peering := range deps.peerings { - idx++ - require.NoError(t, store.PeeringWrite(idx, peering)) + lastIdx++ + require.NoError(t, store.EnsureNode(lastIdx, &structs.Node{ + Node: "my-node", Address: "127.0.0.1", + })) - // make sure it got created - q := state.Query{Value: peering.Name} - _, p, err := store.PeeringRead(nil, q) - require.NoError(t, err) - require.NotNil(t, p) - } + lastIdx++ + require.NoError(t, store.EnsureService(lastIdx, "my-node", &structs.NodeService{ + ID: "api", + Service: "api", + Port: 8000, + })) - // Insert any trust bundles - for _, bundle := range deps.bundles { - idx++ - require.NoError(t, store.PeeringTrustBundleWrite(idx, bundle)) - - q := state.Query{ - Value: bundle.PeerName, - EnterpriseMeta: *structs.NodeEnterpriseMetaInPartition(bundle.Partition), - } - gotIdx, ptb, err := store.PeeringTrustBundleRead(nil, q) - require.NoError(t, err) - require.NotNil(t, ptb) - require.Equal(t, gotIdx, idx) - } - - // Write any config entries - for _, entry := range deps.entries { - idx++ - require.NoError(t, entry.Normalize()) - require.NoError(t, store.EnsureConfigEntry(idx, entry)) - } - - return idx - } - - type testCase struct { - req *pbpeering.TrustBundleListByServiceRequest - expect *pbpeering.TrustBundleListByServiceResponse - expectErr string - } - - // TODO(peering): see note on newTestServer, once we have a better server mock, - // we should add functionality here to verify errors from backend - verify := func(t *testing.T, tc *testCase) { - t.Helper() - - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - t.Cleanup(cancel) - - resp, err := client.TrustBundleListByService(ctx, tc.req) - require.NoError(t, err) - // ignore raft fields - if resp.Bundles != nil { - for _, b := range resp.Bundles { - b.CreateIndex = 0 - b.ModifyIndex = 0 - } - } - prototest.AssertDeepEqual(t, tc.expect, resp) - } - - // Execute scenario steps - // ---------------------- - - // 0 - Initial empty state. - // ----------------------- - verify(t, &testCase{ - req: &pbpeering.TrustBundleListByServiceRequest{ - ServiceName: "foo", - }, - expect: &pbpeering.TrustBundleListByServiceResponse{ - Bundles: nil, - }, - }) - - // 1 - Create a service, verify results still empty. - // ------------------------------------------------ - lastIdx = setup(t, lastIdx, testDeps{services: []string{"foo"}}) - verify(t, &testCase{ - req: &pbpeering.TrustBundleListByServiceRequest{ - ServiceName: "foo", - }, - expect: &pbpeering.TrustBundleListByServiceResponse{ - Bundles: []*pbpeering.PeeringTrustBundle{}, - }, - }) - - // 2 - Create a peering, verify results still empty. - // ------------------------------------------------ - lastIdx = setup(t, lastIdx, testDeps{ - peerings: []*pbpeering.Peering{ + entry := structs.ExportedServicesConfigEntry{ + Name: "default", + Services: []structs.ExportedService{ { - Name: "peer1", - State: pbpeering.PeeringState_ACTIVE, - PeerServerName: "peer1-name", - PeerServerAddresses: []string{"peer1-addr"}, - }, - }, - }) - verify(t, &testCase{ - req: &pbpeering.TrustBundleListByServiceRequest{ - ServiceName: "foo", - }, - expect: &pbpeering.TrustBundleListByServiceResponse{ - Bundles: []*pbpeering.PeeringTrustBundle{}, - }, - }) - - // 3 - Create a config entry, verify results still empty. - // ----------------------------------------------------- - lastIdx = setup(t, lastIdx, testDeps{ - entries: []*structs.ExportedServicesConfigEntry{ - { - Name: "default", - Services: []structs.ExportedService{ + Name: "api", + Consumers: []structs.ServiceConsumer{ { - Name: "foo", - Consumers: []structs.ServiceConsumer{ - { - PeerName: "peer1", - }, - }, + PeerName: "foo", + }, + { + PeerName: "bar", + }, + }, + }, + { + Name: "web", + Consumers: []structs.ServiceConsumer{ + { + PeerName: "baz", }, }, }, }, - }) - verify(t, &testCase{ - req: &pbpeering.TrustBundleListByServiceRequest{ - ServiceName: "foo", - }, - expect: &pbpeering.TrustBundleListByServiceResponse{ - Bundles: []*pbpeering.PeeringTrustBundle{}, - }, - }) + } + require.NoError(t, entry.Normalize()) + require.NoError(t, entry.Validate()) - // 4 - Create trust bundles, verify bundles are returned. - // ----------------------------------------------------- - lastIdx = setup(t, lastIdx, testDeps{ - bundles: []*pbpeering.PeeringTrustBundle{ - { - TrustDomain: "peer1.com", - PeerName: "peer1", - RootPEMs: []string{"peer1-root-1"}, - }, - }, - }) - verify(t, &testCase{ - req: &pbpeering.TrustBundleListByServiceRequest{ - ServiceName: "foo", - }, - expect: &pbpeering.TrustBundleListByServiceResponse{ - Bundles: []*pbpeering.PeeringTrustBundle{ - { - TrustDomain: "peer1.com", - PeerName: "peer1", - RootPEMs: []string{"peer1-root-1"}, - }, - }, - }, - }) - - // 5 - Delete the config entry, verify results empty. - // ------------------------------------------------- lastIdx++ - require.NoError(t, store.DeleteConfigEntry(lastIdx, structs.ExportedServices, "default", nil)) - verify(t, &testCase{ - req: &pbpeering.TrustBundleListByServiceRequest{ - ServiceName: "foo", - }, - expect: &pbpeering.TrustBundleListByServiceResponse{ - Bundles: []*pbpeering.PeeringTrustBundle{}, - }, - }) + require.NoError(t, store.EnsureConfigEntry(lastIdx, &entry)) - // 6 - Restore config entry, verify bundles are returned. - // ----------------------------------------------------- - lastIdx = setup(t, lastIdx, testDeps{ - entries: []*structs.ExportedServicesConfigEntry{ - { - Name: "default", - Services: []structs.ExportedService{ - { - Name: "foo", - Consumers: []structs.ServiceConsumer{ - {PeerName: "peer1"}, - }, - }, - }, - }, - }, - }) - verify(t, &testCase{ - req: &pbpeering.TrustBundleListByServiceRequest{ - ServiceName: "foo", - }, - expect: &pbpeering.TrustBundleListByServiceResponse{ - Bundles: []*pbpeering.PeeringTrustBundle{ - { - TrustDomain: "peer1.com", - PeerName: "peer1", - RootPEMs: []string{"peer1-root-1"}, - }, - }, - }, - }) + client := pbpeering.NewPeeringServiceClient(s.ClientConn(t)) - // 7 - Add new peer and trust bundle. It should be ignored because foo is not exported to it. - // ----------------------------------------------------------------------------------------- - lastIdx = setup(t, lastIdx, testDeps{ - peerings: []*pbpeering.Peering{ - { - Name: "peer2", - State: pbpeering.PeeringState_ACTIVE, - PeerServerName: "peer2-name", - PeerServerAddresses: []string{"peer2-addr"}, - }, - }, - bundles: []*pbpeering.PeeringTrustBundle{ - { - TrustDomain: "peer2.com", - PeerName: "peer2", - RootPEMs: []string{"peer2-root-1"}, - }, - }, - }) - verify(t, &testCase{ - req: &pbpeering.TrustBundleListByServiceRequest{ - ServiceName: "foo", - }, - expect: &pbpeering.TrustBundleListByServiceResponse{ - Bundles: []*pbpeering.PeeringTrustBundle{ - { - TrustDomain: "peer1.com", - PeerName: "peer1", - RootPEMs: []string{"peer1-root-1"}, - }, - }, - }, - }) - - // 8 - Replace config entry to export all services to both peers - // ----------------------------------------------------------------------------------------- - lastIdx = setup(t, lastIdx, testDeps{ - entries: []*structs.ExportedServicesConfigEntry{ - { - Name: "default", - Services: []structs.ExportedService{ - { - Name: structs.WildcardSpecifier, - Consumers: []structs.ServiceConsumer{ - {PeerName: "peer1"}, - {PeerName: "peer2"}, - }, - }, - }, - }, - }, - }) - verify(t, &testCase{ - req: &pbpeering.TrustBundleListByServiceRequest{ - ServiceName: "foo", - }, - expect: &pbpeering.TrustBundleListByServiceResponse{ - Bundles: []*pbpeering.PeeringTrustBundle{ - { - TrustDomain: "peer1.com", - PeerName: "peer1", - RootPEMs: []string{"peer1-root-1"}, - }, - { - TrustDomain: "peer2.com", - PeerName: "peer2", - RootPEMs: []string{"peer2-root-1"}, - }, - }, - }, - }) - - // 9 - delete the service, verify results empty - lastIdx++ - require.NoError(t, store.DeleteService(lastIdx, nodeName, "foo", nil, "")) - verify(t, &testCase{ - req: &pbpeering.TrustBundleListByServiceRequest{ - ServiceName: "foo", - }, - expect: &pbpeering.TrustBundleListByServiceResponse{ - Bundles: []*pbpeering.PeeringTrustBundle{}, - }, - }) + req := pbpeering.TrustBundleListByServiceRequest{ + ServiceName: "api", + } + resp, err := client.TrustBundleListByService(context.Background(), &req) + require.NoError(t, err) + require.Len(t, resp.Bundles, 2) + require.Equal(t, []string{"foo-root-1"}, resp.Bundles[0].RootPEMs) + require.Equal(t, []string{"bar-root-1"}, resp.Bundles[1].RootPEMs) } func Test_StreamHandler_UpsertServices(t *testing.T) { diff --git a/proto/pbpeering/peering.go b/proto/pbpeering/peering.go index f19650f3c0..4c42650211 100644 --- a/proto/pbpeering/peering.go +++ b/proto/pbpeering/peering.go @@ -219,3 +219,32 @@ func NewInitiateRequestFromAPI(req *api.PeeringInitiateRequest) *InitiateRequest InitiateRequestFromAPI(req, t) return t } + +func (r *TrustBundleListByServiceRequest) CacheInfo() cache.RequestInfo { + info := cache.RequestInfo{ + // TODO(peering): Revisit whether this is the token to use once request types accept a token. + Token: r.Token(), + Datacenter: r.Datacenter, + MinIndex: 0, + Timeout: 0, + MustRevalidate: false, + + // TODO(peering): Cache.notifyPollingQuery polls at this interval. We need to revisit how that polling works. + // Using an exponential backoff when the result hasn't changed may be preferable. + MaxAge: 1 * time.Second, + } + + v, err := hashstructure.Hash([]interface{}{ + r.Partition, + r.Namespace, + r.ServiceName, + }, nil) + if err == nil { + // If there is an error, we don't set the key. A blank key forces + // no cache for this request so the request is forwarded directly + // to the server. + info.Key = strconv.FormatUint(v, 10) + } + + return info +} diff --git a/proto/pbpeering/peering.pb.go b/proto/pbpeering/peering.pb.go index c9d2320123..12e0a84fbb 100644 --- a/proto/pbpeering/peering.pb.go +++ b/proto/pbpeering/peering.pb.go @@ -813,10 +813,11 @@ type TrustBundleListByServiceRequest struct { unknownFields protoimpl.UnknownFields ServiceName string `protobuf:"bytes,1,opt,name=ServiceName,proto3" json:"ServiceName,omitempty"` - Partition string `protobuf:"bytes,2,opt,name=Partition,proto3" json:"Partition,omitempty"` + Namespace string `protobuf:"bytes,2,opt,name=Namespace,proto3" json:"Namespace,omitempty"` + Partition string `protobuf:"bytes,3,opt,name=Partition,proto3" json:"Partition,omitempty"` // these are common fields required for implementing structs.RPCInfo methods // that are used to forward requests - Datacenter string `protobuf:"bytes,3,opt,name=Datacenter,proto3" json:"Datacenter,omitempty"` + Datacenter string `protobuf:"bytes,4,opt,name=Datacenter,proto3" json:"Datacenter,omitempty"` } func (x *TrustBundleListByServiceRequest) Reset() { @@ -858,6 +859,13 @@ func (x *TrustBundleListByServiceRequest) GetServiceName() string { return "" } +func (x *TrustBundleListByServiceRequest) GetNamespace() string { + if x != nil { + return x.Namespace + } + return "" +} + func (x *TrustBundleListByServiceRequest) GetPartition() string { if x != nil { return x.Partition @@ -877,7 +885,8 @@ type TrustBundleListByServiceResponse struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Bundles []*PeeringTrustBundle `protobuf:"bytes,1,rep,name=Bundles,proto3" json:"Bundles,omitempty"` + Index uint64 `protobuf:"varint,1,opt,name=Index,proto3" json:"Index,omitempty"` + Bundles []*PeeringTrustBundle `protobuf:"bytes,2,rep,name=Bundles,proto3" json:"Bundles,omitempty"` } func (x *TrustBundleListByServiceResponse) Reset() { @@ -912,6 +921,13 @@ func (*TrustBundleListByServiceResponse) Descriptor() ([]byte, []int) { return file_proto_pbpeering_peering_proto_rawDescGZIP(), []int{11} } +func (x *TrustBundleListByServiceResponse) GetIndex() uint64 { + if x != nil { + return x.Index + } + return 0 +} + func (x *TrustBundleListByServiceResponse) GetBundles() []*PeeringTrustBundle { if x != nil { return x.Bundles @@ -2003,201 +2019,204 @@ var file_proto_pbpeering_peering_proto_rawDesc = []byte{ 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x22, 0x17, 0x0a, 0x15, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x81, 0x01, 0x0a, 0x1f, 0x54, 0x72, 0x75, 0x73, 0x74, 0x42, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x9f, 0x01, 0x0a, 0x1f, 0x54, 0x72, 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x42, 0x79, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x20, 0x0a, 0x0b, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, - 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x50, - 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, - 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1e, 0x0a, 0x0a, 0x44, 0x61, 0x74, - 0x61, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x44, - 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x22, 0x59, 0x0a, 0x20, 0x54, 0x72, 0x75, - 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x42, 0x79, 0x53, 0x65, - 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x35, 0x0a, - 0x07, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, - 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, - 0x54, 0x72, 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x07, 0x42, 0x75, 0x6e, - 0x64, 0x6c, 0x65, 0x73, 0x22, 0x6a, 0x0a, 0x16, 0x54, 0x72, 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, - 0x64, 0x6c, 0x65, 0x52, 0x65, 0x61, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, - 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, - 0x6d, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, - 0x12, 0x1e, 0x0a, 0x0a, 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, - 0x22, 0x64, 0x0a, 0x17, 0x54, 0x72, 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, - 0x65, 0x61, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x49, + 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x4e, + 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, + 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x50, 0x61, 0x72, + 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x50, 0x61, + 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1e, 0x0a, 0x0a, 0x44, 0x61, 0x74, 0x61, 0x63, + 0x65, 0x6e, 0x74, 0x65, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x44, 0x61, 0x74, + 0x61, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x22, 0x6f, 0x0a, 0x20, 0x54, 0x72, 0x75, 0x73, 0x74, + 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x42, 0x79, 0x53, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x49, 0x6e, 0x64, 0x65, - 0x78, 0x12, 0x33, 0x0a, 0x06, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x1b, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x65, 0x65, 0x72, - 0x69, 0x6e, 0x67, 0x54, 0x72, 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x06, - 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x22, 0x2d, 0x0a, 0x1b, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, - 0x67, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x65, 0x42, 0x79, 0x49, 0x44, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x02, 0x49, 0x44, 0x22, 0x1e, 0x0a, 0x1c, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, - 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x65, 0x42, 0x79, 0x49, 0x44, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x8d, 0x01, 0x0a, 0x1e, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, - 0x67, 0x54, 0x72, 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x57, 0x72, 0x69, 0x74, - 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x4b, 0x0a, 0x12, 0x50, 0x65, 0x65, 0x72, - 0x69, 0x6e, 0x67, 0x54, 0x72, 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x50, - 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x54, 0x72, 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, - 0x65, 0x52, 0x12, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x54, 0x72, 0x75, 0x73, 0x74, 0x42, - 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, - 0x74, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x44, 0x61, 0x74, 0x61, 0x63, - 0x65, 0x6e, 0x74, 0x65, 0x72, 0x22, 0x21, 0x0a, 0x1f, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, - 0x54, 0x72, 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x57, 0x72, 0x69, 0x74, 0x65, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x73, 0x0a, 0x1f, 0x50, 0x65, 0x65, 0x72, - 0x69, 0x6e, 0x67, 0x54, 0x72, 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x44, 0x65, - 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x4e, - 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, - 0x1c, 0x0a, 0x09, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x09, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1e, 0x0a, - 0x0a, 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0a, 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x22, 0x22, 0x0a, - 0x20, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x54, 0x72, 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, - 0x64, 0x6c, 0x65, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x22, 0xfc, 0x01, 0x0a, 0x14, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x54, 0x6f, - 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x50, 0x65, - 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x50, 0x65, - 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, + 0x78, 0x12, 0x35, 0x0a, 0x07, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x65, 0x65, + 0x72, 0x69, 0x6e, 0x67, 0x54, 0x72, 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, + 0x07, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x73, 0x22, 0x6a, 0x0a, 0x16, 0x54, 0x72, 0x75, 0x73, + 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x61, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1e, 0x0a, 0x0a, 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, - 0x6e, 0x74, 0x65, 0x72, 0x12, 0x14, 0x0a, 0x05, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x05, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x3b, 0x0a, 0x04, 0x4d, 0x65, - 0x74, 0x61, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, - 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, - 0x79, 0x52, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x1a, 0x37, 0x0a, 0x09, 0x4d, 0x65, 0x74, 0x61, 0x45, - 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, - 0x22, 0x3b, 0x0a, 0x15, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x54, 0x6f, 0x6b, 0x65, - 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x22, 0x0a, 0x0c, 0x50, 0x65, 0x65, - 0x72, 0x69, 0x6e, 0x67, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x0c, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x22, 0x96, 0x02, - 0x0a, 0x0f, 0x49, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x50, 0x65, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x08, 0x50, 0x65, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x22, 0x0a, - 0x0c, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, + 0x6e, 0x74, 0x65, 0x72, 0x22, 0x64, 0x0a, 0x17, 0x54, 0x72, 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, + 0x64, 0x6c, 0x65, 0x52, 0x65, 0x61, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x14, 0x0a, 0x05, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, + 0x49, 0x6e, 0x64, 0x65, 0x78, 0x12, 0x33, 0x0a, 0x06, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, + 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x54, 0x72, 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, + 0x6c, 0x65, 0x52, 0x06, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x22, 0x2d, 0x0a, 0x1b, 0x50, 0x65, + 0x65, 0x72, 0x69, 0x6e, 0x67, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x65, 0x42, 0x79, + 0x49, 0x44, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x44, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x49, 0x44, 0x22, 0x1e, 0x0a, 0x1c, 0x50, 0x65, 0x65, + 0x72, 0x69, 0x6e, 0x67, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x65, 0x42, 0x79, 0x49, + 0x44, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x8d, 0x01, 0x0a, 0x1e, 0x50, 0x65, + 0x65, 0x72, 0x69, 0x6e, 0x67, 0x54, 0x72, 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, + 0x57, 0x72, 0x69, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x4b, 0x0a, 0x12, + 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x54, 0x72, 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, + 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, + 0x6e, 0x67, 0x2e, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x54, 0x72, 0x75, 0x73, 0x74, 0x42, + 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x12, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x54, 0x72, + 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x44, 0x61, 0x74, + 0x61, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x44, + 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x22, 0x21, 0x0a, 0x1f, 0x50, 0x65, 0x65, + 0x72, 0x69, 0x6e, 0x67, 0x54, 0x72, 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x57, + 0x72, 0x69, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x73, 0x0a, 0x1f, + 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x54, 0x72, 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, + 0x6c, 0x65, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x12, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, + 0x61, 0x6d, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, + 0x6e, 0x12, 0x1e, 0x0a, 0x0a, 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, 0x74, 0x65, + 0x72, 0x22, 0x22, 0x0a, 0x20, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x54, 0x72, 0x75, 0x73, + 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0xfc, 0x01, 0x0a, 0x14, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, + 0x74, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, + 0x0a, 0x08, 0x50, 0x65, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x08, 0x50, 0x65, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x50, 0x61, + 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x50, + 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1e, 0x0a, 0x0a, 0x44, 0x61, 0x74, 0x61, + 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x44, 0x61, + 0x74, 0x61, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x12, 0x14, 0x0a, 0x05, 0x54, 0x6f, 0x6b, 0x65, + 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x3b, + 0x0a, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x70, + 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x54, + 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x4d, 0x65, 0x74, 0x61, + 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x1a, 0x37, 0x0a, 0x09, 0x4d, + 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x3a, 0x02, 0x38, 0x01, 0x22, 0x3b, 0x0a, 0x15, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, + 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x22, 0x0a, + 0x0c, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x54, 0x6f, 0x6b, 0x65, - 0x6e, 0x12, 0x1c, 0x0a, 0x09, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, - 0x1e, 0x0a, 0x0a, 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x0a, 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x12, - 0x14, 0x0a, 0x05, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, - 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x36, 0x0a, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x18, 0x06, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x49, 0x6e, - 0x69, 0x74, 0x69, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x4d, 0x65, - 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x1a, 0x37, 0x0a, - 0x09, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, - 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x12, 0x0a, 0x10, 0x49, 0x6e, 0x69, 0x74, 0x69, 0x61, - 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x94, 0x05, 0x0a, 0x12, 0x52, - 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, - 0x65, 0x12, 0x3f, 0x0a, 0x07, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x52, 0x65, 0x70, - 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x2e, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, 0x00, 0x52, 0x07, 0x72, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x12, 0x42, 0x0a, 0x08, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x52, - 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, - 0x65, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x00, 0x52, 0x08, 0x72, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x48, 0x0a, 0x0a, 0x74, 0x65, 0x72, 0x6d, 0x69, 0x6e, - 0x61, 0x74, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x70, 0x65, 0x65, - 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x2e, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, - 0x65, 0x64, 0x48, 0x00, 0x52, 0x0a, 0x74, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x65, 0x64, - 0x1a, 0x7f, 0x0a, 0x07, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x50, - 0x65, 0x65, 0x72, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x50, 0x65, 0x65, - 0x72, 0x49, 0x44, 0x12, 0x14, 0x0a, 0x05, 0x4e, 0x6f, 0x6e, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x05, 0x4e, 0x6f, 0x6e, 0x63, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x52, 0x65, 0x73, - 0x6f, 0x75, 0x72, 0x63, 0x65, 0x55, 0x52, 0x4c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, - 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x55, 0x52, 0x4c, 0x12, 0x24, 0x0a, 0x05, 0x45, - 0x72, 0x72, 0x6f, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x73, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x05, 0x45, 0x72, 0x72, 0x6f, - 0x72, 0x1a, 0x94, 0x02, 0x0a, 0x08, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, - 0x0a, 0x05, 0x4e, 0x6f, 0x6e, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x4e, - 0x6f, 0x6e, 0x63, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, - 0x55, 0x52, 0x4c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x52, 0x65, 0x73, 0x6f, 0x75, - 0x72, 0x63, 0x65, 0x55, 0x52, 0x4c, 0x12, 0x1e, 0x0a, 0x0a, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, - 0x63, 0x65, 0x49, 0x44, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x52, 0x65, 0x73, 0x6f, - 0x75, 0x72, 0x63, 0x65, 0x49, 0x44, 0x12, 0x30, 0x0a, 0x08, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, - 0x63, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, - 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x41, 0x6e, 0x79, 0x52, 0x08, - 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x4c, 0x0a, 0x09, 0x6f, 0x70, 0x65, 0x72, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2e, 0x2e, 0x70, 0x65, - 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x6f, 0x70, 0x65, - 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x30, 0x0a, 0x09, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x6e, 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x10, 0x00, - 0x12, 0x0a, 0x0a, 0x06, 0x55, 0x50, 0x53, 0x45, 0x52, 0x54, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, - 0x44, 0x45, 0x4c, 0x45, 0x54, 0x45, 0x10, 0x02, 0x1a, 0x0c, 0x0a, 0x0a, 0x54, 0x65, 0x72, 0x6d, - 0x69, 0x6e, 0x61, 0x74, 0x65, 0x64, 0x42, 0x09, 0x0a, 0x07, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, - 0x64, 0x2a, 0x53, 0x0a, 0x0c, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x53, 0x74, 0x61, 0x74, - 0x65, 0x12, 0x0d, 0x0a, 0x09, 0x55, 0x4e, 0x44, 0x45, 0x46, 0x49, 0x4e, 0x45, 0x44, 0x10, 0x00, - 0x12, 0x0b, 0x0a, 0x07, 0x49, 0x4e, 0x49, 0x54, 0x49, 0x41, 0x4c, 0x10, 0x01, 0x12, 0x0a, 0x0a, - 0x06, 0x41, 0x43, 0x54, 0x49, 0x56, 0x45, 0x10, 0x02, 0x12, 0x0b, 0x0a, 0x07, 0x46, 0x41, 0x49, - 0x4c, 0x49, 0x4e, 0x47, 0x10, 0x03, 0x12, 0x0e, 0x0a, 0x0a, 0x54, 0x45, 0x52, 0x4d, 0x49, 0x4e, - 0x41, 0x54, 0x45, 0x44, 0x10, 0x04, 0x32, 0xea, 0x05, 0x0a, 0x0e, 0x50, 0x65, 0x65, 0x72, 0x69, - 0x6e, 0x67, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x4e, 0x0a, 0x0d, 0x47, 0x65, 0x6e, - 0x65, 0x72, 0x61, 0x74, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1d, 0x2e, 0x70, 0x65, 0x65, - 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x54, 0x6f, 0x6b, - 0x65, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x70, 0x65, 0x65, 0x72, - 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x54, 0x6f, 0x6b, 0x65, - 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3f, 0x0a, 0x08, 0x49, 0x6e, 0x69, - 0x74, 0x69, 0x61, 0x74, 0x65, 0x12, 0x18, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, - 0x49, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x19, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x49, 0x6e, 0x69, 0x74, 0x69, 0x61, - 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x48, 0x0a, 0x0b, 0x50, 0x65, - 0x65, 0x72, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x61, 0x64, 0x12, 0x1b, 0x2e, 0x70, 0x65, 0x65, 0x72, - 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x61, 0x64, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, - 0x2e, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x61, 0x64, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x48, 0x0a, 0x0b, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x4c, - 0x69, 0x73, 0x74, 0x12, 0x1b, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x65, - 0x65, 0x72, 0x69, 0x6e, 0x67, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x1c, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x65, 0x65, 0x72, 0x69, - 0x6e, 0x67, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4e, - 0x0a, 0x0d, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x12, - 0x1d, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, - 0x67, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, - 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, - 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4b, - 0x0a, 0x0c, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x57, 0x72, 0x69, 0x74, 0x65, 0x12, 0x1c, - 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, - 0x57, 0x72, 0x69, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x70, - 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x57, 0x72, - 0x69, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6f, 0x0a, 0x18, 0x54, - 0x72, 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x42, 0x79, - 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x28, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, - 0x67, 0x2e, 0x54, 0x72, 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x4c, 0x69, 0x73, - 0x74, 0x42, 0x79, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x29, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x54, 0x72, 0x75, 0x73, - 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x42, 0x79, 0x53, 0x65, 0x72, - 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, - 0x54, 0x72, 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x61, 0x64, 0x12, - 0x1f, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x54, 0x72, 0x75, 0x73, 0x74, 0x42, - 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x61, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x20, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x54, 0x72, 0x75, 0x73, 0x74, - 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x61, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x4f, 0x0a, 0x0f, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x73, 0x6f, - 0x75, 0x72, 0x63, 0x65, 0x73, 0x12, 0x1b, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, - 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x73, 0x61, - 0x67, 0x65, 0x1a, 0x1b, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x52, 0x65, 0x70, - 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x28, - 0x01, 0x30, 0x01, 0x42, 0x84, 0x01, 0x0a, 0x0b, 0x63, 0x6f, 0x6d, 0x2e, 0x70, 0x65, 0x65, 0x72, - 0x69, 0x6e, 0x67, 0x42, 0x0c, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x50, 0x72, 0x6f, 0x74, - 0x6f, 0x50, 0x01, 0x5a, 0x2b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, - 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2f, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, - 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x70, 0x62, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, - 0xa2, 0x02, 0x03, 0x50, 0x58, 0x58, 0xaa, 0x02, 0x07, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, - 0xca, 0x02, 0x07, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0xe2, 0x02, 0x13, 0x50, 0x65, 0x65, - 0x72, 0x69, 0x6e, 0x67, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, - 0xea, 0x02, 0x07, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x33, + 0x6e, 0x22, 0x96, 0x02, 0x0a, 0x0f, 0x49, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x74, 0x65, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x50, 0x65, 0x65, 0x72, 0x4e, 0x61, 0x6d, + 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x50, 0x65, 0x65, 0x72, 0x4e, 0x61, 0x6d, + 0x65, 0x12, 0x22, 0x0a, 0x0c, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x54, 0x6f, 0x6b, 0x65, + 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, + 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1c, 0x0a, 0x09, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, + 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, + 0x69, 0x6f, 0x6e, 0x12, 0x1e, 0x0a, 0x0a, 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, 0x74, 0x65, + 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, + 0x74, 0x65, 0x72, 0x12, 0x14, 0x0a, 0x05, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x05, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x05, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x36, 0x0a, 0x04, 0x4d, 0x65, 0x74, + 0x61, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, + 0x67, 0x2e, 0x49, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x4d, 0x65, 0x74, + 0x61, 0x1a, 0x37, 0x0a, 0x09, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, + 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, + 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x12, 0x0a, 0x10, 0x49, 0x6e, + 0x69, 0x74, 0x69, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x94, + 0x05, 0x0a, 0x12, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, + 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x3f, 0x0a, 0x07, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, + 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x73, + 0x61, 0x67, 0x65, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, 0x00, 0x52, 0x07, 0x72, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x42, 0x0a, 0x08, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, + 0x6e, 0x67, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, + 0x73, 0x73, 0x61, 0x67, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x00, + 0x52, 0x08, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x48, 0x0a, 0x0a, 0x74, 0x65, + 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x26, + 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x2e, 0x54, 0x65, 0x72, 0x6d, + 0x69, 0x6e, 0x61, 0x74, 0x65, 0x64, 0x48, 0x00, 0x52, 0x0a, 0x74, 0x65, 0x72, 0x6d, 0x69, 0x6e, + 0x61, 0x74, 0x65, 0x64, 0x1a, 0x7f, 0x0a, 0x07, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x16, 0x0a, 0x06, 0x50, 0x65, 0x65, 0x72, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x06, 0x50, 0x65, 0x65, 0x72, 0x49, 0x44, 0x12, 0x14, 0x0a, 0x05, 0x4e, 0x6f, 0x6e, 0x63, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x4e, 0x6f, 0x6e, 0x63, 0x65, 0x12, 0x20, 0x0a, + 0x0b, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x55, 0x52, 0x4c, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x0b, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x55, 0x52, 0x4c, 0x12, + 0x24, 0x0a, 0x05, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, + 0x2e, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x05, + 0x45, 0x72, 0x72, 0x6f, 0x72, 0x1a, 0x94, 0x02, 0x0a, 0x08, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x4e, 0x6f, 0x6e, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x05, 0x4e, 0x6f, 0x6e, 0x63, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x52, 0x65, 0x73, 0x6f, + 0x75, 0x72, 0x63, 0x65, 0x55, 0x52, 0x4c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x52, + 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x55, 0x52, 0x4c, 0x12, 0x1e, 0x0a, 0x0a, 0x52, 0x65, + 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x44, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, + 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x44, 0x12, 0x30, 0x0a, 0x08, 0x52, 0x65, + 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x41, + 0x6e, 0x79, 0x52, 0x08, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x4c, 0x0a, 0x09, + 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0e, 0x32, + 0x2e, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x2e, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, + 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x30, 0x0a, 0x09, 0x4f, 0x70, + 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x6e, 0x6b, 0x6e, 0x6f, + 0x77, 0x6e, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x55, 0x50, 0x53, 0x45, 0x52, 0x54, 0x10, 0x01, + 0x12, 0x0a, 0x0a, 0x06, 0x44, 0x45, 0x4c, 0x45, 0x54, 0x45, 0x10, 0x02, 0x1a, 0x0c, 0x0a, 0x0a, + 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x65, 0x64, 0x42, 0x09, 0x0a, 0x07, 0x50, 0x61, + 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2a, 0x53, 0x0a, 0x0c, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, + 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x0d, 0x0a, 0x09, 0x55, 0x4e, 0x44, 0x45, 0x46, 0x49, 0x4e, + 0x45, 0x44, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x49, 0x4e, 0x49, 0x54, 0x49, 0x41, 0x4c, 0x10, + 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x41, 0x43, 0x54, 0x49, 0x56, 0x45, 0x10, 0x02, 0x12, 0x0b, 0x0a, + 0x07, 0x46, 0x41, 0x49, 0x4c, 0x49, 0x4e, 0x47, 0x10, 0x03, 0x12, 0x0e, 0x0a, 0x0a, 0x54, 0x45, + 0x52, 0x4d, 0x49, 0x4e, 0x41, 0x54, 0x45, 0x44, 0x10, 0x04, 0x32, 0xea, 0x05, 0x0a, 0x0e, 0x50, + 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x4e, 0x0a, + 0x0d, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1d, + 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, + 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, + 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, + 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3f, 0x0a, + 0x08, 0x49, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x74, 0x65, 0x12, 0x18, 0x2e, 0x70, 0x65, 0x65, 0x72, + 0x69, 0x6e, 0x67, 0x2e, 0x49, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x49, 0x6e, + 0x69, 0x74, 0x69, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x48, + 0x0a, 0x0b, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x61, 0x64, 0x12, 0x1b, 0x2e, + 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x52, + 0x65, 0x61, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x70, 0x65, 0x65, + 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x61, 0x64, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x48, 0x0a, 0x0b, 0x50, 0x65, 0x65, 0x72, + 0x69, 0x6e, 0x67, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x1b, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, + 0x67, 0x2e, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x50, + 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x4e, 0x0a, 0x0d, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x44, 0x65, 0x6c, + 0x65, 0x74, 0x65, 0x12, 0x1d, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x65, + 0x65, 0x72, 0x69, 0x6e, 0x67, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x65, 0x65, + 0x72, 0x69, 0x6e, 0x67, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x4b, 0x0a, 0x0c, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x57, 0x72, 0x69, + 0x74, 0x65, 0x12, 0x1c, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x65, 0x65, + 0x72, 0x69, 0x6e, 0x67, 0x57, 0x72, 0x69, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x1d, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x65, 0x65, 0x72, 0x69, + 0x6e, 0x67, 0x57, 0x72, 0x69, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x6f, 0x0a, 0x18, 0x54, 0x72, 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x4c, 0x69, + 0x73, 0x74, 0x42, 0x79, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x28, 0x2e, 0x70, 0x65, + 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x54, 0x72, 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, + 0x65, 0x4c, 0x69, 0x73, 0x74, 0x42, 0x79, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, + 0x54, 0x72, 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x42, + 0x79, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x54, 0x0a, 0x0f, 0x54, 0x72, 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, + 0x65, 0x61, 0x64, 0x12, 0x1f, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x54, 0x72, + 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x61, 0x64, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x54, + 0x72, 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x61, 0x64, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4f, 0x0a, 0x0f, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, + 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x12, 0x1b, 0x2e, 0x70, 0x65, 0x65, 0x72, + 0x69, 0x6e, 0x67, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4d, + 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x1b, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, + 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x73, + 0x61, 0x67, 0x65, 0x28, 0x01, 0x30, 0x01, 0x42, 0x84, 0x01, 0x0a, 0x0b, 0x63, 0x6f, 0x6d, 0x2e, + 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x42, 0x0c, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, + 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x2b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, + 0x63, 0x6f, 0x6d, 0x2f, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2f, 0x63, 0x6f, + 0x6e, 0x73, 0x75, 0x6c, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x70, 0x62, 0x70, 0x65, 0x65, + 0x72, 0x69, 0x6e, 0x67, 0xa2, 0x02, 0x03, 0x50, 0x58, 0x58, 0xaa, 0x02, 0x07, 0x50, 0x65, 0x65, + 0x72, 0x69, 0x6e, 0x67, 0xca, 0x02, 0x07, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0xe2, 0x02, + 0x13, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, + 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x07, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x62, 0x06, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/proto/pbpeering/peering.proto b/proto/pbpeering/peering.proto index 2103c3fe63..8ac8e353fe 100644 --- a/proto/pbpeering/peering.proto +++ b/proto/pbpeering/peering.proto @@ -181,15 +181,17 @@ message PeeringDeleteResponse {} // @consul-rpc-glue: Datacenter,ReadTODO message TrustBundleListByServiceRequest { string ServiceName = 1; - string Partition = 2; + string Namespace = 2; + string Partition = 3; // these are common fields required for implementing structs.RPCInfo methods // that are used to forward requests - string Datacenter = 3; + string Datacenter = 4; } message TrustBundleListByServiceResponse { - repeated PeeringTrustBundle Bundles = 1; + uint64 Index = 1; + repeated PeeringTrustBundle Bundles = 2; } // @consul-rpc-glue: Datacenter,ReadTODO From a09c7766456cc84ed07b122d6f3e6c04c60cd6bc Mon Sep 17 00:00:00 2001 From: Freddy Date: Wed, 1 Jun 2022 14:31:37 -0600 Subject: [PATCH 3/3] Update public listener with SPIFFE Validator Envoy's SPIFFE certificate validation extension allows for us to validate against different root certificates depending on the trust domain of the dialing proxy. If there are any trust bundles from peers in the config snapshot then we use the SPIFFE validator as the validation context, rather than the usual TrustedCA. The injected validation config includes the local root certificates as well. --- agent/agent.go | 1 + agent/proxycfg-glue/glue.go | 8 ++ agent/proxycfg/connect_proxy.go | 20 ++++ agent/proxycfg/data_sources.go | 10 ++ agent/proxycfg/snapshot.go | 8 +- agent/proxycfg/state.go | 1 + agent/proxycfg/state_test.go | 21 +++- agent/proxycfg/testing.go | 8 +- agent/xds/listeners.go | 103 +++++++++++++++++- agent/xds/listeners_test.go | 16 ++- ...nect-proxy-exported-to-peers.latest.golden | 92 ++++++++++++++++ proto/prototest/testing.go | 2 +- 12 files changed, 279 insertions(+), 11 deletions(-) create mode 100644 agent/xds/testdata/listeners/connect-proxy-exported-to-peers.latest.golden diff --git a/agent/agent.go b/agent/agent.go index ebaa339f13..8fee06587d 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -646,6 +646,7 @@ func (a *Agent) Start(ctx context.Context) error { ResolvedServiceConfig: proxycfgglue.CacheResolvedServiceConfig(a.cache), ServiceList: proxycfgglue.CacheServiceList(a.cache), TrustBundle: proxycfgglue.CacheTrustBundle(a.cache), + TrustBundleList: proxycfgglue.CacheTrustBundleList(a.cache), } a.fillEnterpriseProxyDataSources(&proxyDataSources) a.proxyConfig, err = proxycfg.NewManager(proxycfg.ManagerConfig{ diff --git a/agent/proxycfg-glue/glue.go b/agent/proxycfg-glue/glue.go index b18c6487f4..5538aab5be 100644 --- a/agent/proxycfg-glue/glue.go +++ b/agent/proxycfg-glue/glue.go @@ -101,10 +101,18 @@ func CacheServiceList(c *cache.Cache) proxycfg.ServiceList { return &cacheProxyDataSource[*structs.DCSpecificRequest]{c, cachetype.CatalogServiceListName} } +// CacheTrustBundle satisfies the proxycfg.TrustBundle interface by sourcing +// data from the agent cache. func CacheTrustBundle(c *cache.Cache) proxycfg.TrustBundle { return &cacheProxyDataSource[*pbpeering.TrustBundleReadRequest]{c, cachetype.TrustBundleReadName} } +// CacheTrustBundleList satisfies the proxycfg.TrustBundleList interface by sourcing +// data from the agent cache. +func CacheTrustBundleList(c *cache.Cache) proxycfg.TrustBundleList { + return &cacheProxyDataSource[*pbpeering.TrustBundleListByServiceRequest]{c, cachetype.TrustBundleListName} +} + // cacheProxyDataSource implements a generic wrapper around the agent cache to // provide data to the proxycfg.Manager. type cacheProxyDataSource[ReqType cache.Request] struct { diff --git a/agent/proxycfg/connect_proxy.go b/agent/proxycfg/connect_proxy.go index 923b15b73b..d2931cf27e 100644 --- a/agent/proxycfg/connect_proxy.go +++ b/agent/proxycfg/connect_proxy.go @@ -44,6 +44,16 @@ func (s *handlerConnectProxy) initialize(ctx context.Context) (ConfigSnapshot, e return snap, err } + err = s.dataSources.TrustBundleList.Notify(ctx, &pbpeering.TrustBundleListByServiceRequest{ + // TODO(peering): Pass ACL token + ServiceName: s.proxyCfg.DestinationServiceName, + Namespace: s.proxyID.NamespaceOrDefault(), + Partition: s.proxyID.PartitionOrDefault(), + }, peeringTrustBundlesWatchID, s.ch) + if err != nil { + return snap, err + } + // Watch the leaf cert err = s.dataSources.LeafCertificate.Notify(ctx, &cachetype.ConnectCALeafRequest{ Datacenter: s.source.Datacenter, @@ -259,6 +269,16 @@ func (s *handlerConnectProxy) handleUpdate(ctx context.Context, u UpdateEvent, s snap.ConnectProxy.PeerTrustBundles[peer] = resp.Bundle } + case u.CorrelationID == peeringTrustBundlesWatchID: + resp, ok := u.Result.(*pbpeering.TrustBundleListByServiceResponse) + if !ok { + return fmt.Errorf("invalid type for response: %T", u.Result) + } + if len(resp.Bundles) > 0 { + snap.ConnectProxy.PeeringTrustBundles = resp.Bundles + } + snap.ConnectProxy.PeeringTrustBundlesSet = true + case u.CorrelationID == intentionsWatchID: resp, ok := u.Result.(*structs.IndexedIntentionMatches) if !ok { diff --git a/agent/proxycfg/data_sources.go b/agent/proxycfg/data_sources.go index 1f48c15d89..c454c2052e 100644 --- a/agent/proxycfg/data_sources.go +++ b/agent/proxycfg/data_sources.go @@ -82,6 +82,10 @@ type DataSources struct { // TrustBundle provides updates about the trust bundle for a single peer. TrustBundle TrustBundle + // TrustBundleList provides updates about the list of trust bundles for + // peered clusters that the given proxy is exported to. + TrustBundleList TrustBundleList + DataSourcesEnterprise } @@ -185,3 +189,9 @@ type ServiceList interface { type TrustBundle interface { Notify(ctx context.Context, req *pbpeering.TrustBundleReadRequest, correlationID string, ch chan<- UpdateEvent) error } + +// TrustBundleList is the interface used to consume updates about trust bundles +// for peered clusters that the given proxy is exported to. +type TrustBundleList interface { + Notify(ctx context.Context, req *pbpeering.TrustBundleListByServiceRequest, correlationID string, ch chan<- UpdateEvent) error +} diff --git a/agent/proxycfg/snapshot.go b/agent/proxycfg/snapshot.go index cd8afd2cef..45a250b486 100644 --- a/agent/proxycfg/snapshot.go +++ b/agent/proxycfg/snapshot.go @@ -6,12 +6,12 @@ import ( "sort" "strings" - "github.com/hashicorp/consul/lib" - "github.com/hashicorp/consul/proto/pbpeering" "github.com/mitchellh/copystructure" "github.com/hashicorp/consul/acl" "github.com/hashicorp/consul/agent/structs" + "github.com/hashicorp/consul/lib" + "github.com/hashicorp/consul/proto/pbpeering" ) // TODO(ingress): Can we think of a better for this bag of data? @@ -122,6 +122,9 @@ func gatewayKeyFromString(s string) GatewayKey { type configSnapshotConnectProxy struct { ConfigSnapshotUpstreams + PeeringTrustBundlesSet bool + PeeringTrustBundles []*pbpeering.PeeringTrustBundle + WatchedServiceChecks map[structs.ServiceID][]structs.CheckType // TODO: missing garbage collection PreparedQueryEndpoints map[UpstreamID]structs.CheckServiceNodes // DEPRECATED:see:WatchedUpstreamEndpoints @@ -152,6 +155,7 @@ func (c *configSnapshotConnectProxy) isEmpty() bool { len(c.UpstreamConfig) == 0 && len(c.PassthroughUpstreams) == 0 && len(c.IntentionUpstreams) == 0 && + !c.PeeringTrustBundlesSet && !c.MeshConfigSet } diff --git a/agent/proxycfg/state.go b/agent/proxycfg/state.go index de0eec2368..bce6510d47 100644 --- a/agent/proxycfg/state.go +++ b/agent/proxycfg/state.go @@ -19,6 +19,7 @@ import ( const ( coalesceTimeout = 200 * time.Millisecond rootsWatchID = "roots" + peeringTrustBundlesWatchID = "peering-trust-bundles" leafWatchID = "leaf" peerTrustBundleIDPrefix = "peer-trust-bundle:" intentionsWatchID = "intentions" diff --git a/agent/proxycfg/state_test.go b/agent/proxycfg/state_test.go index 7ca93da661..20446b2dae 100644 --- a/agent/proxycfg/state_test.go +++ b/agent/proxycfg/state_test.go @@ -7,7 +7,6 @@ import ( "testing" "time" - "github.com/hashicorp/consul/proto/pbpeering" "github.com/hashicorp/go-hclog" "github.com/stretchr/testify/require" @@ -15,6 +14,8 @@ import ( cachetype "github.com/hashicorp/consul/agent/cache-types" "github.com/hashicorp/consul/agent/consul/discoverychain" "github.com/hashicorp/consul/agent/structs" + "github.com/hashicorp/consul/proto/pbpeering" + "github.com/hashicorp/consul/proto/prototest" "github.com/hashicorp/consul/sdk/testutil" ) @@ -134,6 +135,7 @@ func recordWatches(sc *stateConfig) *watchRecorder { ResolvedServiceConfig: typedWatchRecorder[*structs.ServiceConfigRequest]{wr}, ServiceList: typedWatchRecorder[*structs.DCSpecificRequest]{wr}, TrustBundle: typedWatchRecorder[*pbpeering.TrustBundleReadRequest]{wr}, + TrustBundleList: typedWatchRecorder[*pbpeering.TrustBundleListByServiceRequest]{wr}, } recordWatchesEnterprise(sc, wr) @@ -217,6 +219,14 @@ func genVerifyLeafWatch(expectedService string, expectedDatacenter string) verif return genVerifyLeafWatchWithDNSSANs(expectedService, expectedDatacenter, nil) } +func genVerifyTrustBundleListWatch(service string) verifyWatchRequest { + return func(t testing.TB, request any) { + reqReal, ok := request.(*pbpeering.TrustBundleListByServiceRequest) + require.True(t, ok) + require.Equal(t, service, reqReal.ServiceName) + } +} + func genVerifyResolverWatch(expectedService, expectedDatacenter, expectedKind string) verifyWatchRequest { return func(t testing.TB, request any) { reqReal, ok := request.(*structs.ConfigEntryQuery) @@ -2492,6 +2502,7 @@ func TestState_WatchesAndUpdates(t *testing.T) { }), rootsWatchID: genVerifyDCSpecificWatch("dc1"), leafWatchID: genVerifyLeafWatch("web", "dc1"), + peeringTrustBundlesWatchID: genVerifyTrustBundleListWatch("web"), peerTrustBundleIDPrefix + "peer-a": genVerifyTrustBundleReadWatch("peer-a"), // No Peering watch }, @@ -2514,12 +2525,18 @@ func TestState_WatchesAndUpdates(t *testing.T) { require.Len(t, snap.ConnectProxy.WatchedServiceChecks, 0, "%+v", snap.ConnectProxy.WatchedServiceChecks) require.Len(t, snap.ConnectProxy.PreparedQueryEndpoints, 0, "%+v", snap.ConnectProxy.PreparedQueryEndpoints) + require.Len(t, snap.ConnectProxy.PeeringTrustBundles, 0, "%+v", snap.ConnectProxy.PeeringTrustBundles) + require.False(t, snap.ConnectProxy.PeeringTrustBundlesSet) }, }, { // This time add the events events: []UpdateEvent{ rootWatchEvent(), + { + CorrelationID: peeringTrustBundlesWatchID, + Result: peerTrustBundles, + }, { CorrelationID: leafWatchID, Result: issuedCert, @@ -2551,8 +2568,10 @@ func TestState_WatchesAndUpdates(t *testing.T) { verifySnapshot: func(t testing.TB, snap *ConfigSnapshot) { require.True(t, snap.Valid()) require.True(t, snap.MeshGateway.isEmpty()) + require.Equal(t, indexedRoots, snap.Roots) require.Equal(t, issuedCert, snap.ConnectProxy.Leaf) + prototest.AssertDeepEqual(t, peerTrustBundles.Bundles, snap.ConnectProxy.PeeringTrustBundles) require.Len(t, snap.ConnectProxy.DiscoveryChain, 2, "%+v", snap.ConnectProxy.DiscoveryChain) require.Len(t, snap.ConnectProxy.WatchedUpstreams, 2, "%+v", snap.ConnectProxy.WatchedUpstreams) diff --git a/agent/proxycfg/testing.go b/agent/proxycfg/testing.go index 8f12ec8f39..77eb84eb08 100644 --- a/agent/proxycfg/testing.go +++ b/agent/proxycfg/testing.go @@ -24,8 +24,6 @@ import ( ) func TestPeerTrustBundles(t testing.T) *pbpeering.TrustBundleListByServiceResponse { - t.Helper() - return &pbpeering.TrustBundleListByServiceResponse{ Bundles: []*pbpeering.PeeringTrustBundle{ { @@ -722,6 +720,7 @@ func testConfigSnapshotFixture( ResolvedServiceConfig: &noopDataSource[*structs.ServiceConfigRequest]{}, ServiceList: &noopDataSource[*structs.DCSpecificRequest]{}, TrustBundle: &noopDataSource[*pbpeering.TrustBundleReadRequest]{}, + TrustBundleList: &noopDataSource[*pbpeering.TrustBundleListByServiceRequest]{}, }, dnsConfig: DNSConfig{ // TODO: make configurable Domain: "consul", @@ -922,6 +921,7 @@ func NewTestDataSources() *TestDataSources { ResolvedServiceConfig: NewTestDataSource[*structs.ServiceConfigRequest, *structs.ServiceConfigResponse](), ServiceList: NewTestDataSource[*structs.DCSpecificRequest, *structs.IndexedServiceList](), TrustBundle: NewTestDataSource[*pbpeering.TrustBundleReadRequest, *pbpeering.TrustBundleReadResponse](), + TrustBundleList: NewTestDataSource[*pbpeering.TrustBundleListByServiceRequest, *pbpeering.TrustBundleListByServiceResponse](), } srcs.buildEnterpriseSources() return srcs @@ -945,6 +945,9 @@ type TestDataSources struct { ResolvedServiceConfig *TestDataSource[*structs.ServiceConfigRequest, *structs.ServiceConfigResponse] ServiceList *TestDataSource[*structs.DCSpecificRequest, *structs.IndexedServiceList] TrustBundle *TestDataSource[*pbpeering.TrustBundleReadRequest, *pbpeering.TrustBundleReadResponse] + TrustBundleList *TestDataSource[*pbpeering.TrustBundleListByServiceRequest, *pbpeering.TrustBundleListByServiceResponse] + + TestDataSourcesEnterprise } func (t *TestDataSources) ToDataSources() DataSources { @@ -965,6 +968,7 @@ func (t *TestDataSources) ToDataSources() DataSources { ResolvedServiceConfig: t.ResolvedServiceConfig, ServiceList: t.ServiceList, TrustBundle: t.TrustBundle, + TrustBundleList: t.TrustBundleList, } t.fillEnterpriseDataSources(&ds) return ds diff --git a/agent/xds/listeners.go b/agent/xds/listeners.go index 01df376410..62d70a7f60 100644 --- a/agent/xds/listeners.go +++ b/agent/xds/listeners.go @@ -11,9 +11,6 @@ import ( "strings" "time" - "github.com/hashicorp/consul/acl" - "github.com/hashicorp/consul/lib" - "github.com/hashicorp/consul/types" "google.golang.org/protobuf/types/known/durationpb" "google.golang.org/protobuf/types/known/wrapperspb" @@ -39,10 +36,14 @@ import ( "github.com/golang/protobuf/ptypes/any" "github.com/golang/protobuf/ptypes/wrappers" + "github.com/hashicorp/consul/acl" "github.com/hashicorp/consul/agent/connect" "github.com/hashicorp/consul/agent/proxycfg" "github.com/hashicorp/consul/agent/structs" + "github.com/hashicorp/consul/lib" + "github.com/hashicorp/consul/proto/pbpeering" "github.com/hashicorp/consul/sdk/iptables" + "github.com/hashicorp/consul/types" ) const virtualIPTag = "virtual" @@ -777,6 +778,100 @@ func (s *ResourceGenerator) injectConnectTLSOnFilterChains(cfgSnap *proxycfg.Con return nil } +// +// NOTE: This method MUST only be used for connect proxy public listeners, +// since TLS validation will be done against root certs for all peers +// that might dial this proxy. +func (s *ResourceGenerator) injectConnectTLSForPublicListener(cfgSnap *proxycfg.ConfigSnapshot, listener *envoy_listener_v3.Listener) error { + if cfgSnap.Kind != structs.ServiceKindConnectProxy { + return fmt.Errorf("cannot inject peering trust bundles for kind %q", cfgSnap.Kind) + } + + // Create TLS validation context for mTLS with leaf certificate and root certs. + tlsContext := makeCommonTLSContext( + cfgSnap.Leaf(), + cfgSnap.RootPEMs(), + makeTLSParametersFromProxyTLSConfig(cfgSnap.MeshConfigTLSIncoming()), + ) + + // Inject peering trust bundles if this service is exported to peered clusters. + if len(cfgSnap.ConnectProxy.PeeringTrustBundles) > 0 { + spiffeConfig, err := makeSpiffeValidatorConfig(cfgSnap.Roots.TrustDomain, cfgSnap.RootPEMs(), cfgSnap.ConnectProxy.PeeringTrustBundles) + if err != nil { + return err + } + + typ, ok := tlsContext.ValidationContextType.(*envoy_tls_v3.CommonTlsContext_ValidationContext) + if !ok { + return fmt.Errorf("unexpected type for TLS context validation: %T", tlsContext.ValidationContextType) + } + + // makeCommonTLSFromLead injects the local trust domain's CA root certs as the TrustedCA. + // We nil it out here since the local roots are included in the SPIFFE validator config. + typ.ValidationContext.TrustedCa = nil + typ.ValidationContext.CustomValidatorConfig = &envoy_core_v3.TypedExtensionConfig{ + // The typed config name is hard-coded because it is not available as a wellknown var in the control plane lib. + Name: "envoy.tls.cert_validator.spiffe", + TypedConfig: spiffeConfig, + } + } + + transportSocket, err := makeDownstreamTLSTransportSocket(&envoy_tls_v3.DownstreamTlsContext{ + CommonTlsContext: tlsContext, + RequireClientCertificate: &wrappers.BoolValue{Value: true}, + }) + if err != nil { + return err + } + + for idx := range listener.FilterChains { + listener.FilterChains[idx].TransportSocket = transportSocket + } + return nil +} + +// SPIFFECertValidatorConfig is used to validate certificates from trust domains other than our own. +// With cluster peering we expect peered clusters to have independent certificate authorities. +// This means that we cannot use a single set of root CA certificates to validate client certificates for mTLS, +// but rather we need to validate against different roots depending on the trust domain of the certificate presented. +func makeSpiffeValidatorConfig(trustDomain, roots string, peerBundles []*pbpeering.PeeringTrustBundle) (*any.Any, error) { + // Store the trust bundle for the local trust domain. + bundles := map[string]string{trustDomain: roots} + + // Store the trust bundle for each trust domain of the peers this proxy is exported to. + // This allows us to validate traffic from other trust domains. + for _, b := range peerBundles { + var pems string + for _, pem := range b.RootPEMs { + pems += lib.EnsureTrailingNewline(pem) + } + bundles[b.TrustDomain] = pems + } + + cfg := &envoy_tls_v3.SPIFFECertValidatorConfig{ + TrustDomains: make([]*envoy_tls_v3.SPIFFECertValidatorConfig_TrustDomain, 0, len(bundles)), + } + + for domain, bundle := range bundles { + cfg.TrustDomains = append(cfg.TrustDomains, &envoy_tls_v3.SPIFFECertValidatorConfig_TrustDomain{ + Name: domain, + TrustBundle: &envoy_core_v3.DataSource{ + Specifier: &envoy_core_v3.DataSource_InlineString{ + InlineString: bundle, + }, + }, + }) + } + + // Sort the trust domains so that the output is stable. + // This benefits tests but also prevents Envoy from mistakenly thinking the listener + // changed and needs to be drained only because this ordering is different. + sort.Slice(cfg.TrustDomains, func(i int, j int) bool { + return cfg.TrustDomains[i].Name < cfg.TrustDomains[j].Name + }) + return ptypes.MarshalAny(cfg) +} + func (s *ResourceGenerator) makeInboundListener(cfgSnap *proxycfg.ConfigSnapshot, name string) (proto.Message, error) { var l *envoy_listener_v3.Listener var err error @@ -899,7 +994,7 @@ func (s *ResourceGenerator) finalizePublicListenerFromConfig(l *envoy_listener_v } // Always apply TLS certificates - if err := s.injectConnectTLSOnFilterChains(cfgSnap, l); err != nil { + if err := s.injectConnectTLSForPublicListener(cfgSnap, l); err != nil { return nil } diff --git a/agent/xds/listeners_test.go b/agent/xds/listeners_test.go index 8fb1cbddc9..b43b5c0b27 100644 --- a/agent/xds/listeners_test.go +++ b/agent/xds/listeners_test.go @@ -9,7 +9,6 @@ import ( "time" envoy_listener_v3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" - testinf "github.com/mitchellh/go-testing-interface" "github.com/stretchr/testify/require" @@ -42,6 +41,21 @@ func TestListenersFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshot(t, nil, nil) }, }, + { + name: "connect-proxy-exported-to-peers", + create: func(t testinf.T) *proxycfg.ConfigSnapshot { + return proxycfg.TestConfigSnapshot(t, func(ns *structs.NodeService) { + // This test is only concerned about the SPIFFE cert validator config in the public listener + // so we empty out the upstreams to avoid generating unnecessary upstream listeners. + ns.Proxy.Upstreams = structs.Upstreams{} + }, []proxycfg.UpdateEvent{ + { + CorrelationID: "peering-trust-bundles", + Result: proxycfg.TestPeerTrustBundles(t), + }, + }) + }, + }, { name: "connect-proxy-with-tls-outgoing-min-version-auto", create: func(t testinf.T) *proxycfg.ConfigSnapshot { diff --git a/agent/xds/testdata/listeners/connect-proxy-exported-to-peers.latest.golden b/agent/xds/testdata/listeners/connect-proxy-exported-to-peers.latest.golden new file mode 100644 index 0000000000..c61cb12d71 --- /dev/null +++ b/agent/xds/testdata/listeners/connect-proxy-exported-to-peers.latest.golden @@ -0,0 +1,92 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", + "name": "public_listener:0.0.0.0:9999", + "address": { + "socketAddress": { + "address": "0.0.0.0", + "portValue": 9999 + } + }, + "filterChains": [ + { + "filters": [ + { + "name": "envoy.filters.network.rbac", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.rbac.v3.RBAC", + "rules": { + + }, + "statPrefix": "connect_authz" + } + }, + { + "name": "envoy.filters.network.tcp_proxy", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", + "statPrefix": "public_listener", + "cluster": "local_app" + } + } + ], + "transportSocket": { + "name": "tls", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext", + "commonTlsContext": { + "tlsParams": { + + }, + "tlsCertificates": [ + { + "certificateChain": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n" + }, + "privateKey": { + "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n" + } + } + ], + "validationContext": { + "customValidatorConfig": { + "name": "envoy.tls.cert_validator.spiffe", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.SPIFFECertValidatorConfig", + "trustDomains": [ + { + "name": "11111111-2222-3333-4444-555555555555.consul", + "trustBundle": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n" + } + }, + { + "name": "1c053652-8512-4373-90cf-5a7f6263a994.consul", + "trustBundle": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICczCCAdwCCQC3BLnEmLCrSjANBgkqhkiG9w0BAQsFADB+MQswCQYDVQQGEwJV\nUzELMAkGA1UECAwCQVoxEjAQBgNVBAcMCUZsYWdzdGFmZjEMMAoGA1UECgwDRm9v\nMRAwDgYDVQQLDAdleGFtcGxlMQ8wDQYDVQQDDAZwZWVyLWExHTAbBgkqhkiG9w0B\nCQEWDmZvb0BwZWVyLWEuY29tMB4XDTIyMDUyNjAxMDQ0NFoXDTIzMDUyNjAxMDQ0\nNFowfjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkFaMRIwEAYDVQQHDAlGbGFnc3Rh\nZmYxDDAKBgNVBAoMA0ZvbzEQMA4GA1UECwwHZXhhbXBsZTEPMA0GA1UEAwwGcGVl\nci1hMR0wGwYJKoZIhvcNAQkBFg5mb29AcGVlci1hLmNvbTCBnzANBgkqhkiG9w0B\nAQEFAAOBjQAwgYkCgYEA2zFYGTbXDAntT5pLTpZ2+VTiqx4J63VRJH1kdu11f0FV\nc2jl1pqCuYDbQXknDU0Pv1Q5y0+nSAihD2KqGS571r+vHQiPtKYPYRqPEe9FzAhR\n2KhWH6v/tk5DG1HqOjV9/zWRKB12gdFNZZqnw/e7NjLNq3wZ2UAwxXip5uJ8uwMC\nAwEAATANBgkqhkiG9w0BAQsFAAOBgQC/CJ9Syf4aL91wZizKTejwouRYoWv4gRAk\nyto45ZcNMHfJ0G2z+XAMl9ZbQsLgXmzAx4IM6y5Jckq8pKC4PEijCjlKTktLHlEy\n0ggmFxtNB1tid2NC8dOzcQ3l45+gDjDqdILhAvLDjlAIebdkqVqb2CfFNW/I2CQH\nZAuKN1aoKA==\n-----END CERTIFICATE-----\n" + } + }, + { + "name": "d89ac423-e95a-475d-94f2-1c557c57bf31.consul", + "trustBundle": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICcTCCAdoCCQDyGxC08cD0BDANBgkqhkiG9w0BAQsFADB9MQswCQYDVQQGEwJV\nUzELMAkGA1UECAwCQ0ExETAPBgNVBAcMCENhcmxzYmFkMQwwCgYDVQQKDANGb28x\nEDAOBgNVBAsMB2V4YW1wbGUxDzANBgNVBAMMBnBlZXItYjEdMBsGCSqGSIb3DQEJ\nARYOZm9vQHBlZXItYi5jb20wHhcNMjIwNTI2MDExNjE2WhcNMjMwNTI2MDExNjE2\nWjB9MQswCQYDVQQGEwJVUzELMAkGA1UECAwCQ0ExETAPBgNVBAcMCENhcmxzYmFk\nMQwwCgYDVQQKDANGb28xEDAOBgNVBAsMB2V4YW1wbGUxDzANBgNVBAMMBnBlZXIt\nYjEdMBsGCSqGSIb3DQEJARYOZm9vQHBlZXItYi5jb20wgZ8wDQYJKoZIhvcNAQEB\nBQADgY0AMIGJAoGBAL4i5erdZ5vKk3mzW9Qt6Wvw/WN/IpMDlL0a28wz9oDCtMLN\ncD/XQB9yT5jUwb2s4mD1lCDZtee8MHeD8zygICozufWVB+u2KvMaoA50T9GMQD0E\nz/0nz/Z703I4q13VHeTpltmEpYcfxw/7nJ3leKA34+Nj3zteJ70iqvD/TNBBAgMB\nAAEwDQYJKoZIhvcNAQELBQADgYEAbL04gicH+EIznDNhZJEb1guMBtBBJ8kujPyU\nao8xhlUuorDTLwhLpkKsOhD8619oSS8KynjEBichidQRkwxIaze0a2mrGT+tGBMf\npVz6UeCkqpde6bSJ/ozEe/2seQzKqYvRT1oUjLwYvY7OIh2DzYibOAxh6fewYAmU\n5j5qNLc=\n-----END CERTIFICATE-----\n" + } + } + ] + } + } + } + }, + "requireClientCertificate": true + } + } + } + ], + "trafficDirection": "INBOUND" + } + ], + "typeUrl": "type.googleapis.com/envoy.config.listener.v3.Listener", + "nonce": "00000001" +} \ No newline at end of file diff --git a/proto/prototest/testing.go b/proto/prototest/testing.go index b17f359b37..c196d77b3a 100644 --- a/proto/prototest/testing.go +++ b/proto/prototest/testing.go @@ -7,7 +7,7 @@ import ( "google.golang.org/protobuf/testing/protocmp" ) -func AssertDeepEqual(t *testing.T, x, y interface{}, opts ...cmp.Option) { +func AssertDeepEqual(t testing.TB, x, y interface{}, opts ...cmp.Option) { t.Helper() opts = append(opts, protocmp.Transform())