diff --git a/acl/enterprisemeta_oss.go b/acl/enterprisemeta_oss.go index 44075a44f7..7623709a93 100644 --- a/acl/enterprisemeta_oss.go +++ b/acl/enterprisemeta_oss.go @@ -58,6 +58,10 @@ func (m *EnterpriseMeta) NamespaceOrDefault() string { return DefaultNamespaceName } +func EqualNamespaces(_, _ string) bool { + return true +} + func NamespaceOrDefault(_ string) string { return DefaultNamespaceName } diff --git a/agent/agent.go b/agent/agent.go index cb6ba1a940..8fee06587d 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -646,6 +646,7 @@ func (a *Agent) Start(ctx context.Context) error { ResolvedServiceConfig: proxycfgglue.CacheResolvedServiceConfig(a.cache), ServiceList: proxycfgglue.CacheServiceList(a.cache), TrustBundle: proxycfgglue.CacheTrustBundle(a.cache), + TrustBundleList: proxycfgglue.CacheTrustBundleList(a.cache), } a.fillEnterpriseProxyDataSources(&proxyDataSources) a.proxyConfig, err = proxycfg.NewManager(proxycfg.ManagerConfig{ @@ -4103,6 +4104,8 @@ func (a *Agent) registerCache() { a.cache.RegisterType(cachetype.FederationStateListMeshGatewaysName, &cachetype.FederationStateListMeshGateways{RPC: a}) + a.cache.RegisterType(cachetype.TrustBundleListName, &cachetype.TrustBundles{Client: a.rpcClientPeering}) + a.registerEntCache() } diff --git a/agent/cache-types/trust_bundle_test.go b/agent/cache-types/trust_bundle_test.go index 8e18e69fe7..fa3d016a29 100644 --- a/agent/cache-types/trust_bundle_test.go +++ b/agent/cache-types/trust_bundle_test.go @@ -11,7 +11,7 @@ import ( "github.com/stretchr/testify/require" ) -func TestTrustBundles(t *testing.T) { +func TestTrustBundle(t *testing.T) { client := NewMockTrustBundleReader(t) typ := &TrustBundle{Client: client} @@ -43,7 +43,7 @@ func TestTrustBundles(t *testing.T) { }, result) } -func TestTrustBundles_badReqType(t *testing.T) { +func TestTrustBundle_badReqType(t *testing.T) { client := pbpeering.NewPeeringServiceClient(nil) typ := &TrustBundle{Client: client} @@ -55,7 +55,7 @@ func TestTrustBundles_badReqType(t *testing.T) { } // This test asserts that we can continuously poll this cache type, given that it doesn't support blocking. -func TestTrustBundles_MultipleUpdates(t *testing.T) { +func TestTrustBundle_MultipleUpdates(t *testing.T) { c := cache.New(cache.Options{}) client := NewMockTrustBundleReader(t) diff --git a/agent/cache-types/trust_bundles.go b/agent/cache-types/trust_bundles.go new file mode 100644 index 0000000000..f4cd2d3c18 --- /dev/null +++ b/agent/cache-types/trust_bundles.go @@ -0,0 +1,50 @@ +package cachetype + +import ( + "context" + "fmt" + + "github.com/hashicorp/consul/agent/cache" + "github.com/hashicorp/consul/proto/pbpeering" + "google.golang.org/grpc" +) + +// Recommended name for registration. +const TrustBundleListName = "trust-bundles" + +// TrustBundles supports fetching discovering service instances via prepared +// queries. +type TrustBundles struct { + RegisterOptionsNoRefresh + Client TrustBundleLister +} + +type TrustBundleLister interface { + TrustBundleListByService( + ctx context.Context, in *pbpeering.TrustBundleListByServiceRequest, opts ...grpc.CallOption, + ) (*pbpeering.TrustBundleListByServiceResponse, error) +} + +func (t *TrustBundles) Fetch(_ cache.FetchOptions, req cache.Request) (cache.FetchResult, error) { + var result cache.FetchResult + + // The request should be a TrustBundleListByServiceRequest. + // We do not need to make a copy of this request type like in other cache types + // because the RequestInfo is synthetic. + reqReal, ok := req.(*pbpeering.TrustBundleListByServiceRequest) + if !ok { + return result, fmt.Errorf( + "Internal cache failure: request wrong type: %T", req) + } + + // Fetch + reply, err := t.Client.TrustBundleListByService(context.Background(), reqReal) + if err != nil { + return result, err + } + + result.Value = reply + result.Index = reply.Index + + return result, nil +} diff --git a/agent/cache-types/trust_bundles_test.go b/agent/cache-types/trust_bundles_test.go new file mode 100644 index 0000000000..451508b0d0 --- /dev/null +++ b/agent/cache-types/trust_bundles_test.go @@ -0,0 +1,152 @@ +package cachetype + +import ( + "context" + "testing" + "time" + + "github.com/hashicorp/consul/agent/cache" + "github.com/hashicorp/consul/proto/pbpeering" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" +) + +func TestTrustBundles(t *testing.T) { + client := NewMockTrustBundleLister(t) + typ := &TrustBundles{Client: client} + + resp := &pbpeering.TrustBundleListByServiceResponse{ + Index: 48, + Bundles: []*pbpeering.PeeringTrustBundle{ + { + PeerName: "peer1", + RootPEMs: []string{"peer1-roots"}, + }, + }, + } + + // Expect the proper call. + // This also returns the canned response above. + client.On("TrustBundleListByService", mock.Anything, mock.Anything). + Run(func(args mock.Arguments) { + req := args.Get(1).(*pbpeering.TrustBundleListByServiceRequest) + require.Equal(t, "foo", req.ServiceName) + }). + Return(resp, nil) + + // Fetch and assert against the result. + result, err := typ.Fetch(cache.FetchOptions{}, &pbpeering.TrustBundleListByServiceRequest{ + ServiceName: "foo", + }) + require.NoError(t, err) + require.Equal(t, cache.FetchResult{ + Value: resp, + Index: 48, + }, result) +} + +func TestTrustBundles_badReqType(t *testing.T) { + client := pbpeering.NewPeeringServiceClient(nil) + typ := &TrustBundles{Client: client} + + // Fetch + _, err := typ.Fetch(cache.FetchOptions{}, cache.TestRequest( + t, cache.RequestInfo{Key: "foo", MinIndex: 64})) + require.Error(t, err) + require.Contains(t, err.Error(), "wrong type") +} + +// This test asserts that we can continuously poll this cache type, given that it doesn't support blocking. +func TestTrustBundles_MultipleUpdates(t *testing.T) { + c := cache.New(cache.Options{}) + + client := NewMockTrustBundleLister(t) + + // On each mock client call to TrustBundleList by service we will increment the index by 1 + // to simulate new data arriving. + resp := &pbpeering.TrustBundleListByServiceResponse{ + Index: uint64(0), + } + + client.On("TrustBundleListByService", mock.Anything, mock.Anything). + Run(func(args mock.Arguments) { + req := args.Get(1).(*pbpeering.TrustBundleListByServiceRequest) + require.Equal(t, "foo", req.ServiceName) + + // Increment on each call. + resp.Index++ + }). + Return(resp, nil) + + c.RegisterType(TrustBundleListName, &TrustBundles{Client: client}) + + ch := make(chan cache.UpdateEvent) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + t.Cleanup(cancel) + + err := c.Notify(ctx, TrustBundleListName, &pbpeering.TrustBundleListByServiceRequest{ServiceName: "foo"}, "updates", ch) + require.NoError(t, err) + + i := uint64(1) + for { + select { + case <-ctx.Done(): + return + case update := <-ch: + // Expect to receive updates for increasing indexes serially. + resp := update.Result.(*pbpeering.TrustBundleListByServiceResponse) + require.Equal(t, i, resp.Index) + i++ + + if i > 3 { + return + } + } + } +} + +// MockTrustBundleLister is an autogenerated mock type for the TrustBundleLister type +type MockTrustBundleLister struct { + mock.Mock +} + +// TrustBundleListByService provides a mock function with given fields: ctx, in, opts +func (_m *MockTrustBundleLister) TrustBundleListByService(ctx context.Context, in *pbpeering.TrustBundleListByServiceRequest, opts ...grpc.CallOption) (*pbpeering.TrustBundleListByServiceResponse, error) { + _va := make([]interface{}, len(opts)) + for _i := range opts { + _va[_i] = opts[_i] + } + var _ca []interface{} + _ca = append(_ca, ctx, in) + _ca = append(_ca, _va...) + ret := _m.Called(_ca...) + + var r0 *pbpeering.TrustBundleListByServiceResponse + if rf, ok := ret.Get(0).(func(context.Context, *pbpeering.TrustBundleListByServiceRequest, ...grpc.CallOption) *pbpeering.TrustBundleListByServiceResponse); ok { + r0 = rf(ctx, in, opts...) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*pbpeering.TrustBundleListByServiceResponse) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, *pbpeering.TrustBundleListByServiceRequest, ...grpc.CallOption) error); ok { + r1 = rf(ctx, in, opts...) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// NewMockTrustBundleLister creates a new instance of MockTrustBundleLister. It also registers the testing.TB interface on the mock and a cleanup function to assert the mocks expectations. +func NewMockTrustBundleLister(t testing.TB) *MockTrustBundleLister { + mock := &MockTrustBundleLister{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/agent/consul/state/catalog.go b/agent/consul/state/catalog.go index ae16267e4b..ab4797fda8 100644 --- a/agent/consul/state/catalog.go +++ b/agent/consul/state/catalog.go @@ -1164,24 +1164,6 @@ func serviceListTxn(tx ReadTxn, ws memdb.WatchSet, entMeta *acl.EnterpriseMeta, return idx, results, nil } -func serviceExists(tx ReadTxn, ws memdb.WatchSet, name string, entMeta *acl.EnterpriseMeta, peerName string) (uint64, bool, error) { - idx := catalogServicesMaxIndex(tx, entMeta, peerName) - q := Query{ - Value: name, - EnterpriseMeta: *entMeta, - PeerName: peerName, - } - watchCh, existing, err := tx.FirstWatch(tableServices, indexService, q) - if err != nil { - return idx, false, fmt.Errorf("failed querying for service: %s", err) - } - ws.Add(watchCh) - if existing == nil { - return idx, false, nil - } - return idx, true, nil -} - // ServicesByNodeMeta returns all services, filtered by the given node metadata. func (s *Store) ServicesByNodeMeta(ws memdb.WatchSet, filters map[string]string, entMeta *acl.EnterpriseMeta, peerName string) (uint64, structs.Services, error) { tx := s.db.Txn(false) diff --git a/agent/consul/state/config_entry.go b/agent/consul/state/config_entry.go index 195cc3b913..e2cd8600f1 100644 --- a/agent/consul/state/config_entry.go +++ b/agent/consul/state/config_entry.go @@ -603,6 +603,10 @@ func validateProposedConfigEntryInServiceGraph( wildcardEntMeta := kindName.WithWildcardNamespace() switch kindName.Kind { + case structs.ExportedServices, structs.MeshConfig: + // Exported services and mesh config do not influence discovery chains. + return nil + case structs.ProxyDefaults: // Check anything that has a discovery chain entry. In the future we could // somehow omit the ones that have a default protocol configured. @@ -1414,52 +1418,6 @@ func configEntryWithOverridesTxn( return configEntryTxn(tx, ws, kind, name, entMeta) } -// getExportedServicesConfigEntriesTxn fetches exported-service config entries and -// filters their exported services to only those that match serviceName and entMeta. -// Because the resulting config entries may have had their exported services modified, -// they *should not* be used in subsequent writes. -func getExportedServiceConfigEntriesTxn( - tx ReadTxn, - ws memdb.WatchSet, - serviceName string, - entMeta *acl.EnterpriseMeta, -) (uint64, []*structs.ExportedServicesConfigEntry, error) { - var exportedServicesEntries []*structs.ExportedServicesConfigEntry - // slice of names to match config entries against - matchCandidates := getExportedServicesMatchServiceNames(serviceName, entMeta) - // matcher func generator for currying the matcher func over EnterpriseMeta values - // from the associated config entry - matchFunc := func(matchMeta *acl.EnterpriseMeta) func(structs.ExportedService) bool { - return func(exportedService structs.ExportedService) bool { - matchSvcName := structs.NewServiceName(exportedService.Name, matchMeta) - for _, candidate := range matchCandidates { - if candidate.Matches(matchSvcName) { - return true - } - } - return false - } - } - idx, entries, err := configEntriesByKindTxn(tx, ws, structs.ExportedServices, entMeta) - if err != nil { - return 0, nil, err - } - for _, entry := range entries { - esEntry, ok := entry.(*structs.ExportedServicesConfigEntry) - if !ok { - return 0, nil, fmt.Errorf("type %T is not a %s config entry", esEntry, structs.ExportedServices) - } - // get a copy of the config entry with Services filtered to match serviceName - newEntry := filterExportedServices(esEntry, matchFunc(entry.GetEnterpriseMeta())) - // the filter will return a new entry, so checking to see if its services is empty says that there - // were matches and that we should include it in the results - if len(newEntry.Services) > 0 { - exportedServicesEntries = append(exportedServicesEntries, newEntry) - } - } - return idx, exportedServicesEntries, nil -} - // protocolForService returns the service graph protocol associated to the // provided service, checking all relevant config entries. func protocolForService( @@ -1502,23 +1460,6 @@ func protocolForService( return maxIdx, chain.Protocol, nil } -// filterExportedServices returns the slice of ExportedService that matc ffor matching service names -// returning a copy of entry with only the services that match one of the -// services in candidates. -func filterExportedServices( - entry *structs.ExportedServicesConfigEntry, - testFunc func(structs.ExportedService) bool, -) *structs.ExportedServicesConfigEntry { - newEntry := *entry - newEntry.Services = []structs.ExportedService{} - for _, ceSvc := range entry.Services { - if testFunc(ceSvc) { - newEntry.Services = append(newEntry.Services, ceSvc) - } - } - return &newEntry -} - func newConfigEntryQuery(c structs.ConfigEntry) configentry.KindName { return configentry.NewKindName(c.GetKind(), c.GetName(), c.GetEnterpriseMeta()) } diff --git a/agent/consul/state/config_entry_oss_test.go b/agent/consul/state/config_entry_oss_test.go index 9f6a1ef44d..4d121ba32d 100644 --- a/agent/consul/state/config_entry_oss_test.go +++ b/agent/consul/state/config_entry_oss_test.go @@ -40,120 +40,124 @@ func testIndexerTableConfigEntries() map[string]indexerTestCase { } } -func TestStore_ExportedServices(t *testing.T) { +func TestStore_peersForService(t *testing.T) { + queryName := "foo" + type testCase struct { name string - write []structs.ConfigEntry - query string - expect []*structs.ExportedServicesConfigEntry + write structs.ConfigEntry + expect []string } cases := []testCase{ { name: "empty everything", - write: []structs.ConfigEntry{}, - query: "foo", - expect: []*structs.ExportedServicesConfigEntry{}, + expect: nil, }, { - name: "no matching exported services", - write: []structs.ConfigEntry{ - &structs.ProxyConfigEntry{Name: "foo"}, - &structs.ProxyConfigEntry{Name: "bar"}, - &structs.ExportedServicesConfigEntry{ - Name: "baz", - Services: []structs.ExportedService{ - {Name: "baz"}, + name: "service is not exported", + write: &structs.ExportedServicesConfigEntry{ + Name: "default", + Services: []structs.ExportedService{ + { + Name: "not-" + queryName, + Consumers: []structs.ServiceConsumer{ + { + PeerName: "zip", + }, + }, }, }, }, - query: "foo", - expect: []*structs.ExportedServicesConfigEntry{}, + expect: nil, }, { - name: "exact match service name", - write: []structs.ConfigEntry{ - &structs.ExportedServicesConfigEntry{ - Name: "foo", - Services: []structs.ExportedService{ - {Name: "foo"}, + name: "wildcard name matches", + write: &structs.ExportedServicesConfigEntry{ + Name: "default", + Services: []structs.ExportedService{ + { + Name: "not-" + queryName, + Consumers: []structs.ServiceConsumer{ + { + PeerName: "zip", + }, + }, }, - }, - &structs.ExportedServicesConfigEntry{ - Name: "bar", - Services: []structs.ExportedService{ - {Name: "bar"}, - }, - }, - }, - query: "bar", - expect: []*structs.ExportedServicesConfigEntry{ - { - Name: "bar", - Services: []structs.ExportedService{ - {Name: "bar"}, + { + Name: structs.WildcardSpecifier, + Consumers: []structs.ServiceConsumer{ + { + PeerName: "bar", + }, + { + PeerName: "baz", + }, + }, }, }, }, + expect: []string{"bar", "baz"}, }, { - name: "wildcard match on service name", - write: []structs.ConfigEntry{ - &structs.ExportedServicesConfigEntry{ - Name: "foo", - Services: []structs.ExportedService{ - {Name: "foo"}, + name: "exact name takes precedence over wildcard", + write: &structs.ExportedServicesConfigEntry{ + Name: "default", + Services: []structs.ExportedService{ + { + Name: queryName, + Consumers: []structs.ServiceConsumer{ + { + PeerName: "baz", + }, + }, }, - }, - &structs.ExportedServicesConfigEntry{ - Name: "wildcard", - Services: []structs.ExportedService{ - {Name: structs.WildcardSpecifier}, - }, - }, - }, - query: "foo", - expect: []*structs.ExportedServicesConfigEntry{ - { - Name: "foo", - Services: []structs.ExportedService{ - {Name: "foo"}, - }, - }, - { - Name: "wildcard", - Services: []structs.ExportedService{ - {Name: structs.WildcardSpecifier}, + { + Name: structs.WildcardSpecifier, + Consumers: []structs.ServiceConsumer{ + { + PeerName: "zip", + }, + }, }, }, }, + expect: []string{"baz"}, }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { s := testStateStore(t) + var lastIdx uint64 - // Write the entries. - for idx, entry := range tc.write { - require.NoError(t, s.EnsureConfigEntry(uint64(idx+1), entry)) + // Write the entry. + if tc.write != nil { + require.NoError(t, tc.write.Normalize()) + require.NoError(t, tc.write.Validate()) + + lastIdx++ + require.NoError(t, s.EnsureConfigEntry(lastIdx, tc.write)) } // Read the entries back. tx := s.db.ReadTxn() defer tx.Abort() - idx, entries, err := getExportedServiceConfigEntriesTxn(tx, nil, tc.query, acl.DefaultEnterpriseMeta()) + + idx, peers, err := peersForServiceTxn(tx, nil, queryName, acl.DefaultEnterpriseMeta()) require.NoError(t, err) - require.Equal(t, uint64(len(tc.write)), idx) + + // This is a little weird, but when there are no results, the index returned should be the max index for the + // config entries table so that the caller can watch for changes to it + if len(peers) == 0 { + require.Equal(t, maxIndexTxn(tx, tableConfigEntries), idx) + } else { + require.Equal(t, lastIdx, idx) + } // Verify the result. - require.Len(t, entries, len(tc.expect)) - for idx, got := range entries { - // ignore raft fields - got.ModifyIndex = 0 - got.CreateIndex = 0 - require.Equal(t, tc.expect[idx], got) - } + require.Len(t, peers, len(tc.expect)) + require.Equal(t, tc.expect, peers) }) } } diff --git a/agent/consul/state/peering.go b/agent/consul/state/peering.go index e566532b68..d18d0ab808 100644 --- a/agent/consul/state/peering.go +++ b/agent/consul/state/peering.go @@ -2,6 +2,7 @@ package state import ( "fmt" + "strings" "github.com/golang/protobuf/proto" "github.com/hashicorp/go-memdb" @@ -439,47 +440,74 @@ func (s *Store) exportedServicesForPeerTxn(ws memdb.WatchSet, tx ReadTxn, peerin // PeeringsForService returns the list of peerings that are associated with the service name provided in the query. // This is used to configure connect proxies for a given service. The result is generated by querying for exported // service config entries and filtering for those that match the given service. +// // TODO(peering): this implementation does all of the work on read to materialize this list of peerings, we should explore // writing to a separate index that has service peerings prepared ahead of time should this become a performance bottleneck. func (s *Store) PeeringsForService(ws memdb.WatchSet, serviceName string, entMeta acl.EnterpriseMeta) (uint64, []*pbpeering.Peering, error) { tx := s.db.ReadTxn() defer tx.Abort() - // short-circuit if the service does not exist in the context of the query -- this prevents "leaking" services - // when there are wildcard rules in place. - if svcIdx, svcExists, err := serviceExists(tx, ws, serviceName, &entMeta, ""); err != nil { - return 0, nil, fmt.Errorf("failed to check if service exists: %w", err) - } else if !svcExists { - // if the service does not exist, return the max index for the services table so caller can watch for changes - return svcIdx, nil, nil - } - // config entries must be defined in the default namespace, so we only need the partition here - meta := structs.DefaultEnterpriseMetaInPartition(entMeta.PartitionOrDefault()) - // return the idx of the config entry that was last modified so caller can watch for changes - idx, peeredServices, err := readPeeredServicesFromConfigEntriesTxn(tx, ws, serviceName, meta) + return peeringsForServiceTxn(tx, ws, serviceName, entMeta) +} + +func peeringsForServiceTxn(tx ReadTxn, ws memdb.WatchSet, serviceName string, entMeta acl.EnterpriseMeta) (uint64, []*pbpeering.Peering, error) { + // Return the idx of the config entry so the caller can watch for changes. + maxIdx, peerNames, err := peersForServiceTxn(tx, ws, serviceName, &entMeta) if err != nil { - return 0, nil, fmt.Errorf("failed to read peered services for service name: %w", err) + return 0, nil, fmt.Errorf("failed to read peers for service name %q: %w", serviceName, err) } var peerings []*pbpeering.Peering - // lookup the peering for each matching peered service - for _, peeredService := range peeredServices { + // Lookup and return the peering corresponding to each name. + for _, name := range peerNames { readQuery := Query{ - Value: peeredService.PeerName, - EnterpriseMeta: peeredService.Name.EnterpriseMeta, + Value: name, + EnterpriseMeta: *structs.NodeEnterpriseMetaInPartition(entMeta.PartitionOrDefault()), } - _, peering, err := peeringReadTxn(tx, ws, readQuery) + idx, peering, err := peeringReadTxn(tx, ws, readQuery) if err != nil { return 0, nil, fmt.Errorf("failed to read peering: %w", err) } + if idx > maxIdx { + maxIdx = idx + } if peering == nil { continue } peerings = append(peerings, peering) } - // see note above about idx - return idx, peerings, nil + return maxIdx, peerings, nil +} + +// TrustBundleListByService returns the trust bundles for all peers that the given service is exported to. +func (s *Store) TrustBundleListByService(ws memdb.WatchSet, service string, entMeta acl.EnterpriseMeta) (uint64, []*pbpeering.PeeringTrustBundle, error) { + tx := s.db.ReadTxn() + defer tx.Abort() + + maxIdx, peers, err := peeringsForServiceTxn(tx, ws, service, entMeta) + if err != nil { + return 0, nil, fmt.Errorf("failed to get peers for service %s: %v", service, err) + } + + var resp []*pbpeering.PeeringTrustBundle + for _, peer := range peers { + pq := Query{ + Value: strings.ToLower(peer.Name), + EnterpriseMeta: *structs.NodeEnterpriseMetaInPartition(entMeta.PartitionOrDefault()), + } + idx, trustBundle, err := peeringTrustBundleReadTxn(tx, ws, pq) + if err != nil { + return 0, nil, fmt.Errorf("failed to read trust bundle for peer %s: %v", peer.Name, err) + } + if idx > maxIdx { + maxIdx = idx + } + if trustBundle != nil { + resp = append(resp, trustBundle) + } + } + return maxIdx, resp, nil } // PeeringTrustBundleRead returns the peering trust bundle for the peer name given as the query value. @@ -487,6 +515,10 @@ func (s *Store) PeeringTrustBundleRead(ws memdb.WatchSet, q Query) (uint64, *pbp tx := s.db.ReadTxn() defer tx.Abort() + return peeringTrustBundleReadTxn(tx, ws, q) +} + +func peeringTrustBundleReadTxn(tx ReadTxn, ws memdb.WatchSet, q Query) (uint64, *pbpeering.PeeringTrustBundle, error) { watchCh, ptbRaw, err := tx.FirstWatch(tablePeeringTrustBundles, indexID, q) if err != nil { return 0, nil, fmt.Errorf("failed peering trust bundle lookup: %w", err) @@ -597,50 +629,80 @@ func (r *Restore) PeeringTrustBundle(ptb *pbpeering.PeeringTrustBundle) error { return nil } -// readPeeredServicesFromConfigEntriesTxn queries exported-service config entries to return peers for serviceName -// in the form of a []structs.PeeredService. -func readPeeredServicesFromConfigEntriesTxn( +// peersForServiceTxn returns the names of all peers that a service is exported to. +func peersForServiceTxn( tx ReadTxn, ws memdb.WatchSet, serviceName string, entMeta *acl.EnterpriseMeta, -) (uint64, []structs.PeeredService, error) { - var results []structs.PeeredService +) (uint64, []string, error) { + // Exported service config entries are scoped to partitions so they are in the default namespace. + partitionMeta := structs.DefaultEnterpriseMetaInPartition(entMeta.PartitionOrDefault()) - // Get all exported-service config entries for that have exports for serviceName. This assumes the result - // has exported services filtered to only those matching serviceName so no futher filtering is needed. - idx, exportedServicesEntries, err := getExportedServiceConfigEntriesTxn(tx, ws, serviceName, entMeta) + idx, rawEntry, err := configEntryTxn(tx, ws, structs.ExportedServices, partitionMeta.PartitionOrDefault(), partitionMeta) if err != nil { return 0, nil, err } + if rawEntry == nil { + return idx, nil, err + } - // dedupe results by peer name - resultSet := make(map[string]struct{}) - // filter entries to only those that have a peer consumer defined - for _, entry := range exportedServicesEntries { - for _, service := range entry.Services { - // entries must have consumers - if service.Consumers == nil || len(service.Consumers) == 0 { - continue - } - for _, consumer := range service.Consumers { - // and consumers must have a peer - if consumer.PeerName == "" { - continue - } - // if we get here, we have a peer consumer, but we should dedupe peer names, so skip if it's already in the set - if _, ok := resultSet[consumer.PeerName]; ok { - continue - } + entry, ok := rawEntry.(*structs.ExportedServicesConfigEntry) + if !ok { + return 0, nil, fmt.Errorf("unexpected type %T for pbpeering.Peering index", rawEntry) + } - // if we got here, we can add to the result set - resultSet[consumer.PeerName] = struct{}{} - result := structs.PeeredService{ - Name: structs.NewServiceName(serviceName, entry.GetEnterpriseMeta()), - PeerName: consumer.PeerName, - } - results = append(results, result) - } + var ( + wildcardNamespaceIdx = -1 + wildcardServiceIdx = -1 + exactMatchIdx = -1 + ) + + // Ensure the metadata is defaulted since we make assertions against potentially empty values below. + // In OSS this is a no-op. + if entMeta == nil { + entMeta = acl.DefaultEnterpriseMeta() + } + entMeta.Normalize() + + // Services can be exported via wildcards or by their exact name: + // Namespace: *, Service: * + // Namespace: Exact, Service: * + // Namespace: Exact, Service: Exact + for i, service := range entry.Services { + switch { + case service.Namespace == structs.WildcardSpecifier: + wildcardNamespaceIdx = i + + case service.Name == structs.WildcardSpecifier && acl.EqualNamespaces(service.Namespace, entMeta.NamespaceOrDefault()): + wildcardServiceIdx = i + + case service.Name == serviceName && acl.EqualNamespaces(service.Namespace, entMeta.NamespaceOrDefault()): + exactMatchIdx = i + } + } + + var results []string + + // Prefer the exact match over the wildcard match. This matches how we handle intention precedence. + var targetIdx int + switch { + case exactMatchIdx >= 0: + targetIdx = exactMatchIdx + + case wildcardServiceIdx >= 0: + targetIdx = wildcardServiceIdx + + case wildcardNamespaceIdx >= 0: + targetIdx = wildcardNamespaceIdx + + default: + return idx, results, nil + } + + for _, c := range entry.Services[targetIdx].Consumers { + if c.PeerName != "" { + results = append(results, c.PeerName) } } return idx, results, nil diff --git a/agent/consul/state/peering_test.go b/agent/consul/state/peering_test.go index 53b80cb9c8..8056f77ecf 100644 --- a/agent/consul/state/peering_test.go +++ b/agent/consul/state/peering_test.go @@ -907,7 +907,7 @@ func TestStateStore_PeeringsForService(t *testing.T) { name string services []structs.ServiceName peerings []*pbpeering.Peering - entries []*structs.ExportedServicesConfigEntry + entry *structs.ExportedServicesConfigEntry query []string expect [][]*pbpeering.Peering expectIdx uint64 @@ -945,9 +945,10 @@ func TestStateStore_PeeringsForService(t *testing.T) { } // Write the config entries. - for _, entry := range tc.entries { + if tc.entry != nil { lastIdx++ - require.NoError(t, s.EnsureConfigEntry(lastIdx, entry)) + require.NoError(t, tc.entry.Normalize()) + require.NoError(t, s.EnsureConfigEntry(lastIdx, tc.entry)) } // Query for peers. @@ -976,21 +977,10 @@ func TestStateStore_PeeringsForService(t *testing.T) { {Name: "foo"}, }, peerings: []*pbpeering.Peering{}, - entries: []*structs.ExportedServicesConfigEntry{}, + entry: nil, query: []string{"foo"}, expect: [][]*pbpeering.Peering{{}}, }, - { - name: "service does not exist", - services: []structs.ServiceName{ - {Name: "foo"}, - }, - peerings: []*pbpeering.Peering{}, - entries: []*structs.ExportedServicesConfigEntry{}, - query: []string{"bar"}, - expect: [][]*pbpeering.Peering{{}}, - expectIdx: uint64(2), // catalog services max index - }, { name: "config entry with exact service name", services: []structs.ServiceName{ @@ -1001,24 +991,22 @@ func TestStateStore_PeeringsForService(t *testing.T) { {Name: "peer1", State: pbpeering.PeeringState_INITIAL}, {Name: "peer2", State: pbpeering.PeeringState_INITIAL}, }, - entries: []*structs.ExportedServicesConfigEntry{ - { - Name: "ce1", - Services: []structs.ExportedService{ - { - Name: "foo", - Consumers: []structs.ServiceConsumer{ - { - PeerName: "peer1", - }, + entry: &structs.ExportedServicesConfigEntry{ + Name: "default", + Services: []structs.ExportedService{ + { + Name: "foo", + Consumers: []structs.ServiceConsumer{ + { + PeerName: "peer1", }, }, - { - Name: "bar", - Consumers: []structs.ServiceConsumer{ - { - PeerName: "peer2", - }, + }, + { + Name: "bar", + Consumers: []structs.ServiceConsumer{ + { + PeerName: "peer2", }, }, }, @@ -1046,27 +1034,25 @@ func TestStateStore_PeeringsForService(t *testing.T) { {Name: "peer2", State: pbpeering.PeeringState_INITIAL}, {Name: "peer3", State: pbpeering.PeeringState_INITIAL}, }, - entries: []*structs.ExportedServicesConfigEntry{ - { - Name: "ce1", - Services: []structs.ExportedService{ - { - Name: "*", - Consumers: []structs.ServiceConsumer{ - { - PeerName: "peer1", - }, - { - PeerName: "peer2", - }, + entry: &structs.ExportedServicesConfigEntry{ + Name: "default", + Services: []structs.ExportedService{ + { + Name: "*", + Consumers: []structs.ServiceConsumer{ + { + PeerName: "peer1", + }, + { + PeerName: "peer2", }, }, - { - Name: "bar", - Consumers: []structs.ServiceConsumer{ - { - PeerName: "peer3", - }, + }, + { + Name: "bar", + Consumers: []structs.ServiceConsumer{ + { + PeerName: "peer3", }, }, }, @@ -1079,8 +1065,6 @@ func TestStateStore_PeeringsForService(t *testing.T) { {Name: "peer2", State: pbpeering.PeeringState_INITIAL}, }, { - {Name: "peer1", State: pbpeering.PeeringState_INITIAL}, - {Name: "peer2", State: pbpeering.PeeringState_INITIAL}, {Name: "peer3", State: pbpeering.PeeringState_INITIAL}, }, }, @@ -1094,3 +1078,219 @@ func TestStateStore_PeeringsForService(t *testing.T) { }) } } + +func TestStore_TrustBundleListByService(t *testing.T) { + store := testStateStore(t) + entMeta := *acl.DefaultEnterpriseMeta() + + var lastIdx uint64 + ws := memdb.NewWatchSet() + + testutil.RunStep(t, "no results on initial setup", func(t *testing.T) { + idx, resp, err := store.TrustBundleListByService(ws, "foo", entMeta) + require.NoError(t, err) + require.Equal(t, lastIdx, idx) + require.Len(t, resp, 0) + }) + + testutil.RunStep(t, "registering service does not yield trust bundles", func(t *testing.T) { + lastIdx++ + require.NoError(t, store.EnsureNode(lastIdx, &structs.Node{ + Node: "my-node", + Address: "127.0.0.1", + })) + + lastIdx++ + require.NoError(t, store.EnsureService(lastIdx, "my-node", &structs.NodeService{ + ID: "foo-1", + Service: "foo", + Port: 8000, + })) + + require.False(t, watchFired(ws)) + + idx, resp, err := store.TrustBundleListByService(ws, "foo", entMeta) + require.NoError(t, err) + require.Len(t, resp, 0) + require.Equal(t, lastIdx-2, idx) + }) + + testutil.RunStep(t, "creating peering does not yield trust bundles", func(t *testing.T) { + lastIdx++ + require.NoError(t, store.PeeringWrite(lastIdx, &pbpeering.Peering{ + Name: "peer1", + })) + + // The peering is only watched after the service is exported via config entry. + require.False(t, watchFired(ws)) + + idx, resp, err := store.TrustBundleListByService(ws, "foo", entMeta) + require.NoError(t, err) + require.Equal(t, uint64(0), idx) + require.Len(t, resp, 0) + }) + + testutil.RunStep(t, "exporting the service does not yield trust bundles", func(t *testing.T) { + lastIdx++ + require.NoError(t, store.EnsureConfigEntry(lastIdx, &structs.ExportedServicesConfigEntry{ + Name: "default", + Services: []structs.ExportedService{ + { + Name: "foo", + Consumers: []structs.ServiceConsumer{ + { + PeerName: "peer1", + }, + }, + }, + }, + })) + + // The config entry is watched. + require.True(t, watchFired(ws)) + ws = memdb.NewWatchSet() + + idx, resp, err := store.TrustBundleListByService(ws, "foo", entMeta) + require.NoError(t, err) + require.Equal(t, lastIdx, idx) + require.Len(t, resp, 0) + }) + + testutil.RunStep(t, "trust bundles are returned after they are created", func(t *testing.T) { + lastIdx++ + require.NoError(t, store.PeeringTrustBundleWrite(lastIdx, &pbpeering.PeeringTrustBundle{ + TrustDomain: "peer1.com", + PeerName: "peer1", + RootPEMs: []string{"peer-root-1"}, + })) + + require.True(t, watchFired(ws)) + ws = memdb.NewWatchSet() + + idx, resp, err := store.TrustBundleListByService(ws, "foo", entMeta) + require.NoError(t, err) + require.Equal(t, lastIdx, idx) + require.Len(t, resp, 1) + require.Equal(t, []string{"peer-root-1"}, resp[0].RootPEMs) + }) + + testutil.RunStep(t, "trust bundles are not returned after unexporting service", func(t *testing.T) { + lastIdx++ + require.NoError(t, store.DeleteConfigEntry(lastIdx, structs.ExportedServices, "default", &entMeta)) + + require.True(t, watchFired(ws)) + ws = memdb.NewWatchSet() + + idx, resp, err := store.TrustBundleListByService(ws, "foo", entMeta) + require.NoError(t, err) + require.Equal(t, lastIdx, idx) + require.Len(t, resp, 0) + }) + + testutil.RunStep(t, "trust bundles are returned after config entry is restored", func(t *testing.T) { + lastIdx++ + require.NoError(t, store.EnsureConfigEntry(lastIdx, &structs.ExportedServicesConfigEntry{ + Name: "default", + Services: []structs.ExportedService{ + { + Name: "foo", + Consumers: []structs.ServiceConsumer{ + { + PeerName: "peer1", + }, + }, + }, + }, + })) + + require.True(t, watchFired(ws)) + ws = memdb.NewWatchSet() + + idx, resp, err := store.TrustBundleListByService(ws, "foo", entMeta) + require.NoError(t, err) + require.Equal(t, lastIdx, idx) + require.Len(t, resp, 1) + require.Equal(t, []string{"peer-root-1"}, resp[0].RootPEMs) + }) + + testutil.RunStep(t, "bundles for other peers are ignored", func(t *testing.T) { + lastIdx++ + require.NoError(t, store.PeeringWrite(lastIdx, &pbpeering.Peering{ + Name: "peer2", + })) + + lastIdx++ + require.NoError(t, store.PeeringTrustBundleWrite(lastIdx, &pbpeering.PeeringTrustBundle{ + TrustDomain: "peer2.com", + PeerName: "peer2", + RootPEMs: []string{"peer-root-2"}, + })) + + // No relevant changes. + require.False(t, watchFired(ws)) + ws = memdb.NewWatchSet() + + idx, resp, err := store.TrustBundleListByService(ws, "foo", entMeta) + require.NoError(t, err) + require.Equal(t, lastIdx-2, idx) + require.Len(t, resp, 1) + require.Equal(t, []string{"peer-root-1"}, resp[0].RootPEMs) + }) + + testutil.RunStep(t, "second bundle is returned when service is exported to that peer", func(t *testing.T) { + lastIdx++ + require.NoError(t, store.EnsureConfigEntry(lastIdx, &structs.ExportedServicesConfigEntry{ + Name: "default", + Services: []structs.ExportedService{ + { + Name: "foo", + Consumers: []structs.ServiceConsumer{ + { + PeerName: "peer1", + }, + { + PeerName: "peer2", + }, + }, + }, + }, + })) + + require.True(t, watchFired(ws)) + ws = memdb.NewWatchSet() + + idx, resp, err := store.TrustBundleListByService(ws, "foo", entMeta) + require.NoError(t, err) + require.Equal(t, lastIdx, idx) + require.Len(t, resp, 2) + require.Equal(t, []string{"peer-root-1"}, resp[0].RootPEMs) + require.Equal(t, []string{"peer-root-2"}, resp[1].RootPEMs) + }) + + testutil.RunStep(t, "deleting the peering excludes its trust bundle", func(t *testing.T) { + lastIdx++ + require.NoError(t, store.PeeringDelete(lastIdx, Query{Value: "peer1"})) + + require.True(t, watchFired(ws)) + ws = memdb.NewWatchSet() + + idx, resp, err := store.TrustBundleListByService(ws, "foo", entMeta) + require.NoError(t, err) + require.Equal(t, lastIdx, idx) + require.Len(t, resp, 1) + require.Equal(t, []string{"peer-root-2"}, resp[0].RootPEMs) + }) + + testutil.RunStep(t, "deleting the service does not excludes its trust bundle", func(t *testing.T) { + lastIdx++ + require.NoError(t, store.DeleteService(lastIdx, "my-node", "foo-1", &entMeta, "")) + + require.False(t, watchFired(ws)) + + idx, resp, err := store.TrustBundleListByService(ws, "foo", entMeta) + require.NoError(t, err) + require.Equal(t, lastIdx-1, idx) + require.Len(t, resp, 1) + require.Equal(t, []string{"peer-root-2"}, resp[0].RootPEMs) + }) +} diff --git a/agent/proxycfg-glue/glue.go b/agent/proxycfg-glue/glue.go index b18c6487f4..5538aab5be 100644 --- a/agent/proxycfg-glue/glue.go +++ b/agent/proxycfg-glue/glue.go @@ -101,10 +101,18 @@ func CacheServiceList(c *cache.Cache) proxycfg.ServiceList { return &cacheProxyDataSource[*structs.DCSpecificRequest]{c, cachetype.CatalogServiceListName} } +// CacheTrustBundle satisfies the proxycfg.TrustBundle interface by sourcing +// data from the agent cache. func CacheTrustBundle(c *cache.Cache) proxycfg.TrustBundle { return &cacheProxyDataSource[*pbpeering.TrustBundleReadRequest]{c, cachetype.TrustBundleReadName} } +// CacheTrustBundleList satisfies the proxycfg.TrustBundleList interface by sourcing +// data from the agent cache. +func CacheTrustBundleList(c *cache.Cache) proxycfg.TrustBundleList { + return &cacheProxyDataSource[*pbpeering.TrustBundleListByServiceRequest]{c, cachetype.TrustBundleListName} +} + // cacheProxyDataSource implements a generic wrapper around the agent cache to // provide data to the proxycfg.Manager. type cacheProxyDataSource[ReqType cache.Request] struct { diff --git a/agent/proxycfg/connect_proxy.go b/agent/proxycfg/connect_proxy.go index 923b15b73b..d2931cf27e 100644 --- a/agent/proxycfg/connect_proxy.go +++ b/agent/proxycfg/connect_proxy.go @@ -44,6 +44,16 @@ func (s *handlerConnectProxy) initialize(ctx context.Context) (ConfigSnapshot, e return snap, err } + err = s.dataSources.TrustBundleList.Notify(ctx, &pbpeering.TrustBundleListByServiceRequest{ + // TODO(peering): Pass ACL token + ServiceName: s.proxyCfg.DestinationServiceName, + Namespace: s.proxyID.NamespaceOrDefault(), + Partition: s.proxyID.PartitionOrDefault(), + }, peeringTrustBundlesWatchID, s.ch) + if err != nil { + return snap, err + } + // Watch the leaf cert err = s.dataSources.LeafCertificate.Notify(ctx, &cachetype.ConnectCALeafRequest{ Datacenter: s.source.Datacenter, @@ -259,6 +269,16 @@ func (s *handlerConnectProxy) handleUpdate(ctx context.Context, u UpdateEvent, s snap.ConnectProxy.PeerTrustBundles[peer] = resp.Bundle } + case u.CorrelationID == peeringTrustBundlesWatchID: + resp, ok := u.Result.(*pbpeering.TrustBundleListByServiceResponse) + if !ok { + return fmt.Errorf("invalid type for response: %T", u.Result) + } + if len(resp.Bundles) > 0 { + snap.ConnectProxy.PeeringTrustBundles = resp.Bundles + } + snap.ConnectProxy.PeeringTrustBundlesSet = true + case u.CorrelationID == intentionsWatchID: resp, ok := u.Result.(*structs.IndexedIntentionMatches) if !ok { diff --git a/agent/proxycfg/data_sources.go b/agent/proxycfg/data_sources.go index 1f48c15d89..c454c2052e 100644 --- a/agent/proxycfg/data_sources.go +++ b/agent/proxycfg/data_sources.go @@ -82,6 +82,10 @@ type DataSources struct { // TrustBundle provides updates about the trust bundle for a single peer. TrustBundle TrustBundle + // TrustBundleList provides updates about the list of trust bundles for + // peered clusters that the given proxy is exported to. + TrustBundleList TrustBundleList + DataSourcesEnterprise } @@ -185,3 +189,9 @@ type ServiceList interface { type TrustBundle interface { Notify(ctx context.Context, req *pbpeering.TrustBundleReadRequest, correlationID string, ch chan<- UpdateEvent) error } + +// TrustBundleList is the interface used to consume updates about trust bundles +// for peered clusters that the given proxy is exported to. +type TrustBundleList interface { + Notify(ctx context.Context, req *pbpeering.TrustBundleListByServiceRequest, correlationID string, ch chan<- UpdateEvent) error +} diff --git a/agent/proxycfg/snapshot.go b/agent/proxycfg/snapshot.go index cd8afd2cef..45a250b486 100644 --- a/agent/proxycfg/snapshot.go +++ b/agent/proxycfg/snapshot.go @@ -6,12 +6,12 @@ import ( "sort" "strings" - "github.com/hashicorp/consul/lib" - "github.com/hashicorp/consul/proto/pbpeering" "github.com/mitchellh/copystructure" "github.com/hashicorp/consul/acl" "github.com/hashicorp/consul/agent/structs" + "github.com/hashicorp/consul/lib" + "github.com/hashicorp/consul/proto/pbpeering" ) // TODO(ingress): Can we think of a better for this bag of data? @@ -122,6 +122,9 @@ func gatewayKeyFromString(s string) GatewayKey { type configSnapshotConnectProxy struct { ConfigSnapshotUpstreams + PeeringTrustBundlesSet bool + PeeringTrustBundles []*pbpeering.PeeringTrustBundle + WatchedServiceChecks map[structs.ServiceID][]structs.CheckType // TODO: missing garbage collection PreparedQueryEndpoints map[UpstreamID]structs.CheckServiceNodes // DEPRECATED:see:WatchedUpstreamEndpoints @@ -152,6 +155,7 @@ func (c *configSnapshotConnectProxy) isEmpty() bool { len(c.UpstreamConfig) == 0 && len(c.PassthroughUpstreams) == 0 && len(c.IntentionUpstreams) == 0 && + !c.PeeringTrustBundlesSet && !c.MeshConfigSet } diff --git a/agent/proxycfg/state.go b/agent/proxycfg/state.go index de0eec2368..bce6510d47 100644 --- a/agent/proxycfg/state.go +++ b/agent/proxycfg/state.go @@ -19,6 +19,7 @@ import ( const ( coalesceTimeout = 200 * time.Millisecond rootsWatchID = "roots" + peeringTrustBundlesWatchID = "peering-trust-bundles" leafWatchID = "leaf" peerTrustBundleIDPrefix = "peer-trust-bundle:" intentionsWatchID = "intentions" diff --git a/agent/proxycfg/state_test.go b/agent/proxycfg/state_test.go index 7ca93da661..20446b2dae 100644 --- a/agent/proxycfg/state_test.go +++ b/agent/proxycfg/state_test.go @@ -7,7 +7,6 @@ import ( "testing" "time" - "github.com/hashicorp/consul/proto/pbpeering" "github.com/hashicorp/go-hclog" "github.com/stretchr/testify/require" @@ -15,6 +14,8 @@ import ( cachetype "github.com/hashicorp/consul/agent/cache-types" "github.com/hashicorp/consul/agent/consul/discoverychain" "github.com/hashicorp/consul/agent/structs" + "github.com/hashicorp/consul/proto/pbpeering" + "github.com/hashicorp/consul/proto/prototest" "github.com/hashicorp/consul/sdk/testutil" ) @@ -134,6 +135,7 @@ func recordWatches(sc *stateConfig) *watchRecorder { ResolvedServiceConfig: typedWatchRecorder[*structs.ServiceConfigRequest]{wr}, ServiceList: typedWatchRecorder[*structs.DCSpecificRequest]{wr}, TrustBundle: typedWatchRecorder[*pbpeering.TrustBundleReadRequest]{wr}, + TrustBundleList: typedWatchRecorder[*pbpeering.TrustBundleListByServiceRequest]{wr}, } recordWatchesEnterprise(sc, wr) @@ -217,6 +219,14 @@ func genVerifyLeafWatch(expectedService string, expectedDatacenter string) verif return genVerifyLeafWatchWithDNSSANs(expectedService, expectedDatacenter, nil) } +func genVerifyTrustBundleListWatch(service string) verifyWatchRequest { + return func(t testing.TB, request any) { + reqReal, ok := request.(*pbpeering.TrustBundleListByServiceRequest) + require.True(t, ok) + require.Equal(t, service, reqReal.ServiceName) + } +} + func genVerifyResolverWatch(expectedService, expectedDatacenter, expectedKind string) verifyWatchRequest { return func(t testing.TB, request any) { reqReal, ok := request.(*structs.ConfigEntryQuery) @@ -2492,6 +2502,7 @@ func TestState_WatchesAndUpdates(t *testing.T) { }), rootsWatchID: genVerifyDCSpecificWatch("dc1"), leafWatchID: genVerifyLeafWatch("web", "dc1"), + peeringTrustBundlesWatchID: genVerifyTrustBundleListWatch("web"), peerTrustBundleIDPrefix + "peer-a": genVerifyTrustBundleReadWatch("peer-a"), // No Peering watch }, @@ -2514,12 +2525,18 @@ func TestState_WatchesAndUpdates(t *testing.T) { require.Len(t, snap.ConnectProxy.WatchedServiceChecks, 0, "%+v", snap.ConnectProxy.WatchedServiceChecks) require.Len(t, snap.ConnectProxy.PreparedQueryEndpoints, 0, "%+v", snap.ConnectProxy.PreparedQueryEndpoints) + require.Len(t, snap.ConnectProxy.PeeringTrustBundles, 0, "%+v", snap.ConnectProxy.PeeringTrustBundles) + require.False(t, snap.ConnectProxy.PeeringTrustBundlesSet) }, }, { // This time add the events events: []UpdateEvent{ rootWatchEvent(), + { + CorrelationID: peeringTrustBundlesWatchID, + Result: peerTrustBundles, + }, { CorrelationID: leafWatchID, Result: issuedCert, @@ -2551,8 +2568,10 @@ func TestState_WatchesAndUpdates(t *testing.T) { verifySnapshot: func(t testing.TB, snap *ConfigSnapshot) { require.True(t, snap.Valid()) require.True(t, snap.MeshGateway.isEmpty()) + require.Equal(t, indexedRoots, snap.Roots) require.Equal(t, issuedCert, snap.ConnectProxy.Leaf) + prototest.AssertDeepEqual(t, peerTrustBundles.Bundles, snap.ConnectProxy.PeeringTrustBundles) require.Len(t, snap.ConnectProxy.DiscoveryChain, 2, "%+v", snap.ConnectProxy.DiscoveryChain) require.Len(t, snap.ConnectProxy.WatchedUpstreams, 2, "%+v", snap.ConnectProxy.WatchedUpstreams) diff --git a/agent/proxycfg/testing.go b/agent/proxycfg/testing.go index 8f12ec8f39..77eb84eb08 100644 --- a/agent/proxycfg/testing.go +++ b/agent/proxycfg/testing.go @@ -24,8 +24,6 @@ import ( ) func TestPeerTrustBundles(t testing.T) *pbpeering.TrustBundleListByServiceResponse { - t.Helper() - return &pbpeering.TrustBundleListByServiceResponse{ Bundles: []*pbpeering.PeeringTrustBundle{ { @@ -722,6 +720,7 @@ func testConfigSnapshotFixture( ResolvedServiceConfig: &noopDataSource[*structs.ServiceConfigRequest]{}, ServiceList: &noopDataSource[*structs.DCSpecificRequest]{}, TrustBundle: &noopDataSource[*pbpeering.TrustBundleReadRequest]{}, + TrustBundleList: &noopDataSource[*pbpeering.TrustBundleListByServiceRequest]{}, }, dnsConfig: DNSConfig{ // TODO: make configurable Domain: "consul", @@ -922,6 +921,7 @@ func NewTestDataSources() *TestDataSources { ResolvedServiceConfig: NewTestDataSource[*structs.ServiceConfigRequest, *structs.ServiceConfigResponse](), ServiceList: NewTestDataSource[*structs.DCSpecificRequest, *structs.IndexedServiceList](), TrustBundle: NewTestDataSource[*pbpeering.TrustBundleReadRequest, *pbpeering.TrustBundleReadResponse](), + TrustBundleList: NewTestDataSource[*pbpeering.TrustBundleListByServiceRequest, *pbpeering.TrustBundleListByServiceResponse](), } srcs.buildEnterpriseSources() return srcs @@ -945,6 +945,9 @@ type TestDataSources struct { ResolvedServiceConfig *TestDataSource[*structs.ServiceConfigRequest, *structs.ServiceConfigResponse] ServiceList *TestDataSource[*structs.DCSpecificRequest, *structs.IndexedServiceList] TrustBundle *TestDataSource[*pbpeering.TrustBundleReadRequest, *pbpeering.TrustBundleReadResponse] + TrustBundleList *TestDataSource[*pbpeering.TrustBundleListByServiceRequest, *pbpeering.TrustBundleListByServiceResponse] + + TestDataSourcesEnterprise } func (t *TestDataSources) ToDataSources() DataSources { @@ -965,6 +968,7 @@ func (t *TestDataSources) ToDataSources() DataSources { ResolvedServiceConfig: t.ResolvedServiceConfig, ServiceList: t.ServiceList, TrustBundle: t.TrustBundle, + TrustBundleList: t.TrustBundleList, } t.fillEnterpriseDataSources(&ds) return ds diff --git a/agent/rpc/peering/service.go b/agent/rpc/peering/service.go index 1ff2b0fa0e..8e1dcbee3c 100644 --- a/agent/rpc/peering/service.go +++ b/agent/rpc/peering/service.go @@ -130,9 +130,9 @@ type Store interface { PeeringList(ws memdb.WatchSet, entMeta acl.EnterpriseMeta) (uint64, []*pbpeering.Peering, error) PeeringTrustBundleRead(ws memdb.WatchSet, q state.Query) (uint64, *pbpeering.PeeringTrustBundle, error) ExportedServicesForPeer(ws memdb.WatchSet, peerID string) (uint64, *structs.ExportedServiceList, error) - PeeringsForService(ws memdb.WatchSet, serviceName string, entMeta acl.EnterpriseMeta) (uint64, []*pbpeering.Peering, error) ServiceDump(ws memdb.WatchSet, kind structs.ServiceKind, useKind bool, entMeta *acl.EnterpriseMeta, peerName string) (uint64, structs.CheckServiceNodes, error) CAConfig(ws memdb.WatchSet) (uint64, *structs.CAConfiguration, error) + TrustBundleListByService(ws memdb.WatchSet, service string, entMeta acl.EnterpriseMeta) (uint64, []*pbpeering.PeeringTrustBundle, error) AbandonCh() <-chan struct{} } @@ -450,33 +450,16 @@ func (s *Service) TrustBundleListByService(ctx context.Context, req *pbpeering.T } defer metrics.MeasureSince([]string{"peering", "trust_bundle_list_by_service"}, time.Now()) - // TODO(peering): ACL check request token + // TODO(peering): ACL check request token for service:write on the service name // TODO(peering): handle blocking queries - entMeta := *structs.NodeEnterpriseMetaInPartition(req.Partition) - // TODO(peering): we're throwing away the index here that would tell us how to execute a blocking query - _, peers, err := s.Backend.Store().PeeringsForService(nil, req.ServiceName, entMeta) + entMeta := acl.NewEnterpriseMetaWithPartition(req.Partition, req.Namespace) + idx, bundles, err := s.Backend.Store().TrustBundleListByService(nil, req.ServiceName, entMeta) if err != nil { - return nil, fmt.Errorf("failed to get peers for service %s: %v", req.ServiceName, err) + return nil, err } - - trustBundles := []*pbpeering.PeeringTrustBundle{} - for _, peer := range peers { - q := state.Query{ - Value: strings.ToLower(peer.Name), - EnterpriseMeta: *structs.NodeEnterpriseMetaInPartition(req.Partition), - } - _, trustBundle, err := s.Backend.Store().PeeringTrustBundleRead(nil, q) - if err != nil { - return nil, fmt.Errorf("failed to read trust bundle for peer %s: %v", peer.Name, err) - } - - if trustBundle != nil { - trustBundles = append(trustBundles, trustBundle) - } - } - return &pbpeering.TrustBundleListByServiceResponse{Bundles: trustBundles}, nil + return &pbpeering.TrustBundleListByServiceResponse{Index: idx, Bundles: bundles}, nil } type BidirectionalStream interface { diff --git a/agent/rpc/peering/service_test.go b/agent/rpc/peering/service_test.go index 0d30987da5..fc67b65baa 100644 --- a/agent/rpc/peering/service_test.go +++ b/agent/rpc/peering/service_test.go @@ -358,357 +358,98 @@ func TestPeeringService_TrustBundleRead(t *testing.T) { prototest.AssertDeepEqual(t, bundle, resp.Bundle) } +// Setup: +// - Peerings "foo" and "bar" with trust bundles saved +// - "api" service exported to both "foo" and "bar" +// - "web" service exported to "baz" func TestPeeringService_TrustBundleListByService(t *testing.T) { - // test executes the following scenario: - // 0 - initial setup test server, state store, RPC client, verify empty results - // 1 - create a service, verify results still empty - // 2 - create a peering, verify results still empty - // 3 - create a config entry, verify results still empty - // 4 - create trust bundles, verify bundles are returned - // 5 - delete the config entry, verify results empty - // 6 - restore config entry, verify bundles are returned - // 7 - add peering, trust bundles, wildcard config entry, verify updated results are present - // 8 - delete first config entry, verify bundles are returned - // 9 - delete the service, verify results empty - // Note: these steps are dependent on each other by design so that we can verify that - // combinations of services, peerings, trust bundles, and config entries all affect results + s := newTestServer(t, nil) + store := s.Server.FSM().State() - // fixed for the test - nodeName := "test-node" + var lastIdx uint64 = 10 - // keep track of index across steps - var lastIdx uint64 - - // Create test server - // TODO(peering): see note on newTestServer, refactor to not use this - srv := newTestServer(t, nil) - store := srv.Server.FSM().State() - client := pbpeering.NewPeeringServiceClient(srv.ClientConn(t)) - - // Create a node up-front so that we can assign services to it if needed - svcNode := &structs.Node{Node: nodeName, Address: "127.0.0.1"} lastIdx++ - require.NoError(t, store.EnsureNode(lastIdx, svcNode)) + require.NoError(t, s.Server.FSM().State().PeeringWrite(lastIdx, &pbpeering.Peering{ + Name: "foo", + State: pbpeering.PeeringState_INITIAL, + PeerServerName: "test", + PeerServerAddresses: []string{"addr1"}, + })) - type testDeps struct { - services []string - peerings []*pbpeering.Peering - entries []*structs.ExportedServicesConfigEntry - bundles []*pbpeering.PeeringTrustBundle - } + lastIdx++ + require.NoError(t, s.Server.FSM().State().PeeringWrite(lastIdx, &pbpeering.Peering{ + Name: "bar", + State: pbpeering.PeeringState_INITIAL, + PeerServerName: "test-bar", + PeerServerAddresses: []string{"addr2"}, + })) - setup := func(t *testing.T, idx uint64, deps testDeps) uint64 { - // Create any services (and node) - if len(deps.services) >= 0 { - svcNode := &structs.Node{Node: nodeName, Address: "127.0.0.1"} - idx++ - require.NoError(t, store.EnsureNode(idx, svcNode)) + lastIdx++ + require.NoError(t, store.PeeringTrustBundleWrite(lastIdx, &pbpeering.PeeringTrustBundle{ + TrustDomain: "foo.com", + PeerName: "foo", + RootPEMs: []string{"foo-root-1"}, + })) - // Create the test services - for _, svc := range deps.services { - idx++ - require.NoError(t, store.EnsureService(idx, svcNode.Node, &structs.NodeService{ - ID: svc, - Service: svc, - Port: int(8000 + idx), - })) - } - } + lastIdx++ + require.NoError(t, store.PeeringTrustBundleWrite(lastIdx, &pbpeering.PeeringTrustBundle{ + TrustDomain: "bar.com", + PeerName: "bar", + RootPEMs: []string{"bar-root-1"}, + })) - // Insert any peerings - for _, peering := range deps.peerings { - idx++ - require.NoError(t, store.PeeringWrite(idx, peering)) + lastIdx++ + require.NoError(t, store.EnsureNode(lastIdx, &structs.Node{ + Node: "my-node", Address: "127.0.0.1", + })) - // make sure it got created - q := state.Query{Value: peering.Name} - _, p, err := store.PeeringRead(nil, q) - require.NoError(t, err) - require.NotNil(t, p) - } + lastIdx++ + require.NoError(t, store.EnsureService(lastIdx, "my-node", &structs.NodeService{ + ID: "api", + Service: "api", + Port: 8000, + })) - // Insert any trust bundles - for _, bundle := range deps.bundles { - idx++ - require.NoError(t, store.PeeringTrustBundleWrite(idx, bundle)) - - q := state.Query{ - Value: bundle.PeerName, - EnterpriseMeta: *structs.NodeEnterpriseMetaInPartition(bundle.Partition), - } - gotIdx, ptb, err := store.PeeringTrustBundleRead(nil, q) - require.NoError(t, err) - require.NotNil(t, ptb) - require.Equal(t, gotIdx, idx) - } - - // Write any config entries - for _, entry := range deps.entries { - idx++ - require.NoError(t, store.EnsureConfigEntry(idx, entry)) - } - - return idx - } - - type testCase struct { - req *pbpeering.TrustBundleListByServiceRequest - expect *pbpeering.TrustBundleListByServiceResponse - expectErr string - } - - // TODO(peering): see note on newTestServer, once we have a better server mock, - // we should add functionality here to verify errors from backend - verify := func(t *testing.T, tc *testCase) { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - t.Cleanup(cancel) - - resp, err := client.TrustBundleListByService(ctx, tc.req) - require.NoError(t, err) - // ignore raft fields - if resp.Bundles != nil { - for _, b := range resp.Bundles { - b.CreateIndex = 0 - b.ModifyIndex = 0 - } - } - prototest.AssertDeepEqual(t, tc.expect, resp) - } - - // Execute scenario steps - // ---------------------- - - // 0 - initial empty state - // ----------------------- - verify(t, &testCase{ - req: &pbpeering.TrustBundleListByServiceRequest{ - ServiceName: "foo", - }, - expect: &pbpeering.TrustBundleListByServiceResponse{ - Bundles: nil, - }, - }) - - // 1 - create a service, verify results still empty - // ------------------------------------------------ - lastIdx = setup(t, lastIdx, testDeps{services: []string{"foo"}}) - verify(t, &testCase{ - req: &pbpeering.TrustBundleListByServiceRequest{ - ServiceName: "foo", - }, - expect: &pbpeering.TrustBundleListByServiceResponse{ - Bundles: []*pbpeering.PeeringTrustBundle{}, - }, - }) - - // 2 - create a peering, verify results still empty - // ------------------------------------------------ - lastIdx = setup(t, lastIdx, testDeps{ - peerings: []*pbpeering.Peering{ + entry := structs.ExportedServicesConfigEntry{ + Name: "default", + Services: []structs.ExportedService{ { - Name: "peer1", - State: pbpeering.PeeringState_ACTIVE, - PeerServerName: "peer1-name", - PeerServerAddresses: []string{"peer1-addr"}, - }, - }, - }) - verify(t, &testCase{ - req: &pbpeering.TrustBundleListByServiceRequest{ - ServiceName: "foo", - }, - expect: &pbpeering.TrustBundleListByServiceResponse{ - Bundles: []*pbpeering.PeeringTrustBundle{}, - }, - }) - - // 3 - create a config entry, verify results still empty - // ----------------------------------------------------- - lastIdx = setup(t, lastIdx, testDeps{ - entries: []*structs.ExportedServicesConfigEntry{ - { - Name: "export-foo", - Services: []structs.ExportedService{ + Name: "api", + Consumers: []structs.ServiceConsumer{ { - Name: "foo", - Consumers: []structs.ServiceConsumer{ - { - PeerName: "peer1", - }, - }, + PeerName: "foo", + }, + { + PeerName: "bar", + }, + }, + }, + { + Name: "web", + Consumers: []structs.ServiceConsumer{ + { + PeerName: "baz", }, }, }, }, - }) - verify(t, &testCase{ - req: &pbpeering.TrustBundleListByServiceRequest{ - ServiceName: "foo", - }, - expect: &pbpeering.TrustBundleListByServiceResponse{ - Bundles: []*pbpeering.PeeringTrustBundle{}, - }, - }) + } + require.NoError(t, entry.Normalize()) + require.NoError(t, entry.Validate()) - // 4 - create trust bundles, verify bundles are returned - // ----------------------------------------------------- - lastIdx = setup(t, lastIdx, testDeps{ - bundles: []*pbpeering.PeeringTrustBundle{ - { - TrustDomain: "peer1.com", - PeerName: "peer1", - RootPEMs: []string{"peer1-root-1"}, - }, - }, - }) - verify(t, &testCase{ - req: &pbpeering.TrustBundleListByServiceRequest{ - ServiceName: "foo", - }, - expect: &pbpeering.TrustBundleListByServiceResponse{ - Bundles: []*pbpeering.PeeringTrustBundle{ - { - TrustDomain: "peer1.com", - PeerName: "peer1", - RootPEMs: []string{"peer1-root-1"}, - }, - }, - }, - }) - - // 5 - delete the config entry, verify results empty - // ------------------------------------------------- lastIdx++ - require.NoError(t, store.DeleteConfigEntry(lastIdx, structs.ExportedServices, "export-foo", nil)) - verify(t, &testCase{ - req: &pbpeering.TrustBundleListByServiceRequest{ - ServiceName: "foo", - }, - expect: &pbpeering.TrustBundleListByServiceResponse{ - Bundles: []*pbpeering.PeeringTrustBundle{}, - }, - }) + require.NoError(t, store.EnsureConfigEntry(lastIdx, &entry)) - // 6 - restore config entry, verify bundles are returned - // ----------------------------------------------------- - lastIdx = setup(t, lastIdx, testDeps{ - entries: []*structs.ExportedServicesConfigEntry{ - { - Name: "export-foo", - Services: []structs.ExportedService{ - { - Name: "foo", - Consumers: []structs.ServiceConsumer{ - {PeerName: "peer1"}, - }, - }, - }, - }, - }, - }) - verify(t, &testCase{ - req: &pbpeering.TrustBundleListByServiceRequest{ - ServiceName: "foo", - }, - expect: &pbpeering.TrustBundleListByServiceResponse{ - Bundles: []*pbpeering.PeeringTrustBundle{ - { - TrustDomain: "peer1.com", - PeerName: "peer1", - RootPEMs: []string{"peer1-root-1"}, - }, - }, - }, - }) + client := pbpeering.NewPeeringServiceClient(s.ClientConn(t)) - // 7 - add peering, trust bundles, wildcard config entry, verify updated results are present - // ----------------------------------------------------------------------------------------- - lastIdx = setup(t, lastIdx, testDeps{ - services: []string{"bar"}, - peerings: []*pbpeering.Peering{ - { - Name: "peer2", - State: pbpeering.PeeringState_ACTIVE, - PeerServerName: "peer2-name", - PeerServerAddresses: []string{"peer2-addr"}, - }, - }, - entries: []*structs.ExportedServicesConfigEntry{ - { - Name: "export-all", - Services: []structs.ExportedService{ - { - Name: structs.WildcardSpecifier, - Consumers: []structs.ServiceConsumer{ - {PeerName: "peer1"}, - {PeerName: "peer2"}, - }, - }, - }, - }, - }, - bundles: []*pbpeering.PeeringTrustBundle{ - { - TrustDomain: "peer2.com", - PeerName: "peer2", - RootPEMs: []string{"peer2-root-1"}, - }, - }, - }) - verify(t, &testCase{ - req: &pbpeering.TrustBundleListByServiceRequest{ - ServiceName: "foo", - }, - expect: &pbpeering.TrustBundleListByServiceResponse{ - Bundles: []*pbpeering.PeeringTrustBundle{ - { - TrustDomain: "peer1.com", - PeerName: "peer1", - RootPEMs: []string{"peer1-root-1"}, - }, - { - TrustDomain: "peer2.com", - PeerName: "peer2", - RootPEMs: []string{"peer2-root-1"}, - }, - }, - }, - }) - - // 8 - delete first config entry, verify bundles are returned - lastIdx++ - require.NoError(t, store.DeleteConfigEntry(lastIdx, structs.ExportedServices, "export-foo", nil)) - verify(t, &testCase{ - req: &pbpeering.TrustBundleListByServiceRequest{ - ServiceName: "foo", - }, - expect: &pbpeering.TrustBundleListByServiceResponse{ - Bundles: []*pbpeering.PeeringTrustBundle{ - { - TrustDomain: "peer1.com", - PeerName: "peer1", - RootPEMs: []string{"peer1-root-1"}, - }, - { - TrustDomain: "peer2.com", - PeerName: "peer2", - RootPEMs: []string{"peer2-root-1"}, - }, - }, - }, - }) - - // 9 - delete the service, verify results empty - lastIdx++ - require.NoError(t, store.DeleteService(lastIdx, nodeName, "foo", nil, "")) - verify(t, &testCase{ - req: &pbpeering.TrustBundleListByServiceRequest{ - ServiceName: "foo", - }, - expect: &pbpeering.TrustBundleListByServiceResponse{ - Bundles: []*pbpeering.PeeringTrustBundle{}, - }, - }) + req := pbpeering.TrustBundleListByServiceRequest{ + ServiceName: "api", + } + resp, err := client.TrustBundleListByService(context.Background(), &req) + require.NoError(t, err) + require.Len(t, resp.Bundles, 2) + require.Equal(t, []string{"foo-root-1"}, resp.Bundles[0].RootPEMs) + require.Equal(t, []string{"bar-root-1"}, resp.Bundles[1].RootPEMs) } func Test_StreamHandler_UpsertServices(t *testing.T) { diff --git a/agent/structs/peering.go b/agent/structs/peering.go index bd0351bb36..1cf74bdaad 100644 --- a/agent/structs/peering.go +++ b/agent/structs/peering.go @@ -8,12 +8,6 @@ type PeeringToken struct { PeerID string } -// PeeredService is a service that has been configured with an exported-service config entry to be exported to a peer. -type PeeredService struct { - Name ServiceName - PeerName string -} - // NOTE: this is not serialized via msgpack so it can be changed without concern. type ExportedServiceList struct { // Services is a list of exported services that apply to both standard diff --git a/agent/xds/listeners.go b/agent/xds/listeners.go index 01df376410..62d70a7f60 100644 --- a/agent/xds/listeners.go +++ b/agent/xds/listeners.go @@ -11,9 +11,6 @@ import ( "strings" "time" - "github.com/hashicorp/consul/acl" - "github.com/hashicorp/consul/lib" - "github.com/hashicorp/consul/types" "google.golang.org/protobuf/types/known/durationpb" "google.golang.org/protobuf/types/known/wrapperspb" @@ -39,10 +36,14 @@ import ( "github.com/golang/protobuf/ptypes/any" "github.com/golang/protobuf/ptypes/wrappers" + "github.com/hashicorp/consul/acl" "github.com/hashicorp/consul/agent/connect" "github.com/hashicorp/consul/agent/proxycfg" "github.com/hashicorp/consul/agent/structs" + "github.com/hashicorp/consul/lib" + "github.com/hashicorp/consul/proto/pbpeering" "github.com/hashicorp/consul/sdk/iptables" + "github.com/hashicorp/consul/types" ) const virtualIPTag = "virtual" @@ -777,6 +778,100 @@ func (s *ResourceGenerator) injectConnectTLSOnFilterChains(cfgSnap *proxycfg.Con return nil } +// +// NOTE: This method MUST only be used for connect proxy public listeners, +// since TLS validation will be done against root certs for all peers +// that might dial this proxy. +func (s *ResourceGenerator) injectConnectTLSForPublicListener(cfgSnap *proxycfg.ConfigSnapshot, listener *envoy_listener_v3.Listener) error { + if cfgSnap.Kind != structs.ServiceKindConnectProxy { + return fmt.Errorf("cannot inject peering trust bundles for kind %q", cfgSnap.Kind) + } + + // Create TLS validation context for mTLS with leaf certificate and root certs. + tlsContext := makeCommonTLSContext( + cfgSnap.Leaf(), + cfgSnap.RootPEMs(), + makeTLSParametersFromProxyTLSConfig(cfgSnap.MeshConfigTLSIncoming()), + ) + + // Inject peering trust bundles if this service is exported to peered clusters. + if len(cfgSnap.ConnectProxy.PeeringTrustBundles) > 0 { + spiffeConfig, err := makeSpiffeValidatorConfig(cfgSnap.Roots.TrustDomain, cfgSnap.RootPEMs(), cfgSnap.ConnectProxy.PeeringTrustBundles) + if err != nil { + return err + } + + typ, ok := tlsContext.ValidationContextType.(*envoy_tls_v3.CommonTlsContext_ValidationContext) + if !ok { + return fmt.Errorf("unexpected type for TLS context validation: %T", tlsContext.ValidationContextType) + } + + // makeCommonTLSFromLead injects the local trust domain's CA root certs as the TrustedCA. + // We nil it out here since the local roots are included in the SPIFFE validator config. + typ.ValidationContext.TrustedCa = nil + typ.ValidationContext.CustomValidatorConfig = &envoy_core_v3.TypedExtensionConfig{ + // The typed config name is hard-coded because it is not available as a wellknown var in the control plane lib. + Name: "envoy.tls.cert_validator.spiffe", + TypedConfig: spiffeConfig, + } + } + + transportSocket, err := makeDownstreamTLSTransportSocket(&envoy_tls_v3.DownstreamTlsContext{ + CommonTlsContext: tlsContext, + RequireClientCertificate: &wrappers.BoolValue{Value: true}, + }) + if err != nil { + return err + } + + for idx := range listener.FilterChains { + listener.FilterChains[idx].TransportSocket = transportSocket + } + return nil +} + +// SPIFFECertValidatorConfig is used to validate certificates from trust domains other than our own. +// With cluster peering we expect peered clusters to have independent certificate authorities. +// This means that we cannot use a single set of root CA certificates to validate client certificates for mTLS, +// but rather we need to validate against different roots depending on the trust domain of the certificate presented. +func makeSpiffeValidatorConfig(trustDomain, roots string, peerBundles []*pbpeering.PeeringTrustBundle) (*any.Any, error) { + // Store the trust bundle for the local trust domain. + bundles := map[string]string{trustDomain: roots} + + // Store the trust bundle for each trust domain of the peers this proxy is exported to. + // This allows us to validate traffic from other trust domains. + for _, b := range peerBundles { + var pems string + for _, pem := range b.RootPEMs { + pems += lib.EnsureTrailingNewline(pem) + } + bundles[b.TrustDomain] = pems + } + + cfg := &envoy_tls_v3.SPIFFECertValidatorConfig{ + TrustDomains: make([]*envoy_tls_v3.SPIFFECertValidatorConfig_TrustDomain, 0, len(bundles)), + } + + for domain, bundle := range bundles { + cfg.TrustDomains = append(cfg.TrustDomains, &envoy_tls_v3.SPIFFECertValidatorConfig_TrustDomain{ + Name: domain, + TrustBundle: &envoy_core_v3.DataSource{ + Specifier: &envoy_core_v3.DataSource_InlineString{ + InlineString: bundle, + }, + }, + }) + } + + // Sort the trust domains so that the output is stable. + // This benefits tests but also prevents Envoy from mistakenly thinking the listener + // changed and needs to be drained only because this ordering is different. + sort.Slice(cfg.TrustDomains, func(i int, j int) bool { + return cfg.TrustDomains[i].Name < cfg.TrustDomains[j].Name + }) + return ptypes.MarshalAny(cfg) +} + func (s *ResourceGenerator) makeInboundListener(cfgSnap *proxycfg.ConfigSnapshot, name string) (proto.Message, error) { var l *envoy_listener_v3.Listener var err error @@ -899,7 +994,7 @@ func (s *ResourceGenerator) finalizePublicListenerFromConfig(l *envoy_listener_v } // Always apply TLS certificates - if err := s.injectConnectTLSOnFilterChains(cfgSnap, l); err != nil { + if err := s.injectConnectTLSForPublicListener(cfgSnap, l); err != nil { return nil } diff --git a/agent/xds/listeners_test.go b/agent/xds/listeners_test.go index 8fb1cbddc9..b43b5c0b27 100644 --- a/agent/xds/listeners_test.go +++ b/agent/xds/listeners_test.go @@ -9,7 +9,6 @@ import ( "time" envoy_listener_v3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" - testinf "github.com/mitchellh/go-testing-interface" "github.com/stretchr/testify/require" @@ -42,6 +41,21 @@ func TestListenersFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshot(t, nil, nil) }, }, + { + name: "connect-proxy-exported-to-peers", + create: func(t testinf.T) *proxycfg.ConfigSnapshot { + return proxycfg.TestConfigSnapshot(t, func(ns *structs.NodeService) { + // This test is only concerned about the SPIFFE cert validator config in the public listener + // so we empty out the upstreams to avoid generating unnecessary upstream listeners. + ns.Proxy.Upstreams = structs.Upstreams{} + }, []proxycfg.UpdateEvent{ + { + CorrelationID: "peering-trust-bundles", + Result: proxycfg.TestPeerTrustBundles(t), + }, + }) + }, + }, { name: "connect-proxy-with-tls-outgoing-min-version-auto", create: func(t testinf.T) *proxycfg.ConfigSnapshot { diff --git a/agent/xds/testdata/listeners/connect-proxy-exported-to-peers.latest.golden b/agent/xds/testdata/listeners/connect-proxy-exported-to-peers.latest.golden new file mode 100644 index 0000000000..c61cb12d71 --- /dev/null +++ b/agent/xds/testdata/listeners/connect-proxy-exported-to-peers.latest.golden @@ -0,0 +1,92 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", + "name": "public_listener:0.0.0.0:9999", + "address": { + "socketAddress": { + "address": "0.0.0.0", + "portValue": 9999 + } + }, + "filterChains": [ + { + "filters": [ + { + "name": "envoy.filters.network.rbac", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.rbac.v3.RBAC", + "rules": { + + }, + "statPrefix": "connect_authz" + } + }, + { + "name": "envoy.filters.network.tcp_proxy", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", + "statPrefix": "public_listener", + "cluster": "local_app" + } + } + ], + "transportSocket": { + "name": "tls", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext", + "commonTlsContext": { + "tlsParams": { + + }, + "tlsCertificates": [ + { + "certificateChain": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n" + }, + "privateKey": { + "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n" + } + } + ], + "validationContext": { + "customValidatorConfig": { + "name": "envoy.tls.cert_validator.spiffe", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.SPIFFECertValidatorConfig", + "trustDomains": [ + { + "name": "11111111-2222-3333-4444-555555555555.consul", + "trustBundle": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n" + } + }, + { + "name": "1c053652-8512-4373-90cf-5a7f6263a994.consul", + "trustBundle": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICczCCAdwCCQC3BLnEmLCrSjANBgkqhkiG9w0BAQsFADB+MQswCQYDVQQGEwJV\nUzELMAkGA1UECAwCQVoxEjAQBgNVBAcMCUZsYWdzdGFmZjEMMAoGA1UECgwDRm9v\nMRAwDgYDVQQLDAdleGFtcGxlMQ8wDQYDVQQDDAZwZWVyLWExHTAbBgkqhkiG9w0B\nCQEWDmZvb0BwZWVyLWEuY29tMB4XDTIyMDUyNjAxMDQ0NFoXDTIzMDUyNjAxMDQ0\nNFowfjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkFaMRIwEAYDVQQHDAlGbGFnc3Rh\nZmYxDDAKBgNVBAoMA0ZvbzEQMA4GA1UECwwHZXhhbXBsZTEPMA0GA1UEAwwGcGVl\nci1hMR0wGwYJKoZIhvcNAQkBFg5mb29AcGVlci1hLmNvbTCBnzANBgkqhkiG9w0B\nAQEFAAOBjQAwgYkCgYEA2zFYGTbXDAntT5pLTpZ2+VTiqx4J63VRJH1kdu11f0FV\nc2jl1pqCuYDbQXknDU0Pv1Q5y0+nSAihD2KqGS571r+vHQiPtKYPYRqPEe9FzAhR\n2KhWH6v/tk5DG1HqOjV9/zWRKB12gdFNZZqnw/e7NjLNq3wZ2UAwxXip5uJ8uwMC\nAwEAATANBgkqhkiG9w0BAQsFAAOBgQC/CJ9Syf4aL91wZizKTejwouRYoWv4gRAk\nyto45ZcNMHfJ0G2z+XAMl9ZbQsLgXmzAx4IM6y5Jckq8pKC4PEijCjlKTktLHlEy\n0ggmFxtNB1tid2NC8dOzcQ3l45+gDjDqdILhAvLDjlAIebdkqVqb2CfFNW/I2CQH\nZAuKN1aoKA==\n-----END CERTIFICATE-----\n" + } + }, + { + "name": "d89ac423-e95a-475d-94f2-1c557c57bf31.consul", + "trustBundle": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICcTCCAdoCCQDyGxC08cD0BDANBgkqhkiG9w0BAQsFADB9MQswCQYDVQQGEwJV\nUzELMAkGA1UECAwCQ0ExETAPBgNVBAcMCENhcmxzYmFkMQwwCgYDVQQKDANGb28x\nEDAOBgNVBAsMB2V4YW1wbGUxDzANBgNVBAMMBnBlZXItYjEdMBsGCSqGSIb3DQEJ\nARYOZm9vQHBlZXItYi5jb20wHhcNMjIwNTI2MDExNjE2WhcNMjMwNTI2MDExNjE2\nWjB9MQswCQYDVQQGEwJVUzELMAkGA1UECAwCQ0ExETAPBgNVBAcMCENhcmxzYmFk\nMQwwCgYDVQQKDANGb28xEDAOBgNVBAsMB2V4YW1wbGUxDzANBgNVBAMMBnBlZXIt\nYjEdMBsGCSqGSIb3DQEJARYOZm9vQHBlZXItYi5jb20wgZ8wDQYJKoZIhvcNAQEB\nBQADgY0AMIGJAoGBAL4i5erdZ5vKk3mzW9Qt6Wvw/WN/IpMDlL0a28wz9oDCtMLN\ncD/XQB9yT5jUwb2s4mD1lCDZtee8MHeD8zygICozufWVB+u2KvMaoA50T9GMQD0E\nz/0nz/Z703I4q13VHeTpltmEpYcfxw/7nJ3leKA34+Nj3zteJ70iqvD/TNBBAgMB\nAAEwDQYJKoZIhvcNAQELBQADgYEAbL04gicH+EIznDNhZJEb1guMBtBBJ8kujPyU\nao8xhlUuorDTLwhLpkKsOhD8619oSS8KynjEBichidQRkwxIaze0a2mrGT+tGBMf\npVz6UeCkqpde6bSJ/ozEe/2seQzKqYvRT1oUjLwYvY7OIh2DzYibOAxh6fewYAmU\n5j5qNLc=\n-----END CERTIFICATE-----\n" + } + } + ] + } + } + } + }, + "requireClientCertificate": true + } + } + } + ], + "trafficDirection": "INBOUND" + } + ], + "typeUrl": "type.googleapis.com/envoy.config.listener.v3.Listener", + "nonce": "00000001" +} \ No newline at end of file diff --git a/proto/pbpeering/peering.go b/proto/pbpeering/peering.go index f19650f3c0..4c42650211 100644 --- a/proto/pbpeering/peering.go +++ b/proto/pbpeering/peering.go @@ -219,3 +219,32 @@ func NewInitiateRequestFromAPI(req *api.PeeringInitiateRequest) *InitiateRequest InitiateRequestFromAPI(req, t) return t } + +func (r *TrustBundleListByServiceRequest) CacheInfo() cache.RequestInfo { + info := cache.RequestInfo{ + // TODO(peering): Revisit whether this is the token to use once request types accept a token. + Token: r.Token(), + Datacenter: r.Datacenter, + MinIndex: 0, + Timeout: 0, + MustRevalidate: false, + + // TODO(peering): Cache.notifyPollingQuery polls at this interval. We need to revisit how that polling works. + // Using an exponential backoff when the result hasn't changed may be preferable. + MaxAge: 1 * time.Second, + } + + v, err := hashstructure.Hash([]interface{}{ + r.Partition, + r.Namespace, + r.ServiceName, + }, nil) + if err == nil { + // If there is an error, we don't set the key. A blank key forces + // no cache for this request so the request is forwarded directly + // to the server. + info.Key = strconv.FormatUint(v, 10) + } + + return info +} diff --git a/proto/pbpeering/peering.pb.go b/proto/pbpeering/peering.pb.go index c9d2320123..12e0a84fbb 100644 --- a/proto/pbpeering/peering.pb.go +++ b/proto/pbpeering/peering.pb.go @@ -813,10 +813,11 @@ type TrustBundleListByServiceRequest struct { unknownFields protoimpl.UnknownFields ServiceName string `protobuf:"bytes,1,opt,name=ServiceName,proto3" json:"ServiceName,omitempty"` - Partition string `protobuf:"bytes,2,opt,name=Partition,proto3" json:"Partition,omitempty"` + Namespace string `protobuf:"bytes,2,opt,name=Namespace,proto3" json:"Namespace,omitempty"` + Partition string `protobuf:"bytes,3,opt,name=Partition,proto3" json:"Partition,omitempty"` // these are common fields required for implementing structs.RPCInfo methods // that are used to forward requests - Datacenter string `protobuf:"bytes,3,opt,name=Datacenter,proto3" json:"Datacenter,omitempty"` + Datacenter string `protobuf:"bytes,4,opt,name=Datacenter,proto3" json:"Datacenter,omitempty"` } func (x *TrustBundleListByServiceRequest) Reset() { @@ -858,6 +859,13 @@ func (x *TrustBundleListByServiceRequest) GetServiceName() string { return "" } +func (x *TrustBundleListByServiceRequest) GetNamespace() string { + if x != nil { + return x.Namespace + } + return "" +} + func (x *TrustBundleListByServiceRequest) GetPartition() string { if x != nil { return x.Partition @@ -877,7 +885,8 @@ type TrustBundleListByServiceResponse struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Bundles []*PeeringTrustBundle `protobuf:"bytes,1,rep,name=Bundles,proto3" json:"Bundles,omitempty"` + Index uint64 `protobuf:"varint,1,opt,name=Index,proto3" json:"Index,omitempty"` + Bundles []*PeeringTrustBundle `protobuf:"bytes,2,rep,name=Bundles,proto3" json:"Bundles,omitempty"` } func (x *TrustBundleListByServiceResponse) Reset() { @@ -912,6 +921,13 @@ func (*TrustBundleListByServiceResponse) Descriptor() ([]byte, []int) { return file_proto_pbpeering_peering_proto_rawDescGZIP(), []int{11} } +func (x *TrustBundleListByServiceResponse) GetIndex() uint64 { + if x != nil { + return x.Index + } + return 0 +} + func (x *TrustBundleListByServiceResponse) GetBundles() []*PeeringTrustBundle { if x != nil { return x.Bundles @@ -2003,201 +2019,204 @@ var file_proto_pbpeering_peering_proto_rawDesc = []byte{ 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x22, 0x17, 0x0a, 0x15, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x81, 0x01, 0x0a, 0x1f, 0x54, 0x72, 0x75, 0x73, 0x74, 0x42, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x9f, 0x01, 0x0a, 0x1f, 0x54, 0x72, 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x42, 0x79, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x20, 0x0a, 0x0b, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, - 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x50, - 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, - 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1e, 0x0a, 0x0a, 0x44, 0x61, 0x74, - 0x61, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x44, - 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x22, 0x59, 0x0a, 0x20, 0x54, 0x72, 0x75, - 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x42, 0x79, 0x53, 0x65, - 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x35, 0x0a, - 0x07, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, - 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, - 0x54, 0x72, 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x07, 0x42, 0x75, 0x6e, - 0x64, 0x6c, 0x65, 0x73, 0x22, 0x6a, 0x0a, 0x16, 0x54, 0x72, 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, - 0x64, 0x6c, 0x65, 0x52, 0x65, 0x61, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, - 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, - 0x6d, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, - 0x12, 0x1e, 0x0a, 0x0a, 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, - 0x22, 0x64, 0x0a, 0x17, 0x54, 0x72, 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, - 0x65, 0x61, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x49, + 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x4e, + 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, + 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x50, 0x61, 0x72, + 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x50, 0x61, + 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1e, 0x0a, 0x0a, 0x44, 0x61, 0x74, 0x61, 0x63, + 0x65, 0x6e, 0x74, 0x65, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x44, 0x61, 0x74, + 0x61, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x22, 0x6f, 0x0a, 0x20, 0x54, 0x72, 0x75, 0x73, 0x74, + 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x42, 0x79, 0x53, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x49, 0x6e, 0x64, 0x65, - 0x78, 0x12, 0x33, 0x0a, 0x06, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x1b, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x65, 0x65, 0x72, - 0x69, 0x6e, 0x67, 0x54, 0x72, 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x06, - 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x22, 0x2d, 0x0a, 0x1b, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, - 0x67, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x65, 0x42, 0x79, 0x49, 0x44, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x02, 0x49, 0x44, 0x22, 0x1e, 0x0a, 0x1c, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, - 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x65, 0x42, 0x79, 0x49, 0x44, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x8d, 0x01, 0x0a, 0x1e, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, - 0x67, 0x54, 0x72, 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x57, 0x72, 0x69, 0x74, - 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x4b, 0x0a, 0x12, 0x50, 0x65, 0x65, 0x72, - 0x69, 0x6e, 0x67, 0x54, 0x72, 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x50, - 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x54, 0x72, 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, - 0x65, 0x52, 0x12, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x54, 0x72, 0x75, 0x73, 0x74, 0x42, - 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, - 0x74, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x44, 0x61, 0x74, 0x61, 0x63, - 0x65, 0x6e, 0x74, 0x65, 0x72, 0x22, 0x21, 0x0a, 0x1f, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, - 0x54, 0x72, 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x57, 0x72, 0x69, 0x74, 0x65, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x73, 0x0a, 0x1f, 0x50, 0x65, 0x65, 0x72, - 0x69, 0x6e, 0x67, 0x54, 0x72, 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x44, 0x65, - 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x4e, - 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, - 0x1c, 0x0a, 0x09, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x09, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1e, 0x0a, - 0x0a, 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0a, 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x22, 0x22, 0x0a, - 0x20, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x54, 0x72, 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, - 0x64, 0x6c, 0x65, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x22, 0xfc, 0x01, 0x0a, 0x14, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x54, 0x6f, - 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x50, 0x65, - 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x50, 0x65, - 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, + 0x78, 0x12, 0x35, 0x0a, 0x07, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x65, 0x65, + 0x72, 0x69, 0x6e, 0x67, 0x54, 0x72, 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, + 0x07, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x73, 0x22, 0x6a, 0x0a, 0x16, 0x54, 0x72, 0x75, 0x73, + 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x61, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1e, 0x0a, 0x0a, 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, - 0x6e, 0x74, 0x65, 0x72, 0x12, 0x14, 0x0a, 0x05, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x05, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x3b, 0x0a, 0x04, 0x4d, 0x65, - 0x74, 0x61, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, - 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, - 0x79, 0x52, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x1a, 0x37, 0x0a, 0x09, 0x4d, 0x65, 0x74, 0x61, 0x45, - 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, - 0x22, 0x3b, 0x0a, 0x15, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x54, 0x6f, 0x6b, 0x65, - 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x22, 0x0a, 0x0c, 0x50, 0x65, 0x65, - 0x72, 0x69, 0x6e, 0x67, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x0c, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x22, 0x96, 0x02, - 0x0a, 0x0f, 0x49, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x50, 0x65, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x08, 0x50, 0x65, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x22, 0x0a, - 0x0c, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, + 0x6e, 0x74, 0x65, 0x72, 0x22, 0x64, 0x0a, 0x17, 0x54, 0x72, 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, + 0x64, 0x6c, 0x65, 0x52, 0x65, 0x61, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x14, 0x0a, 0x05, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, + 0x49, 0x6e, 0x64, 0x65, 0x78, 0x12, 0x33, 0x0a, 0x06, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, + 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x54, 0x72, 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, + 0x6c, 0x65, 0x52, 0x06, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x22, 0x2d, 0x0a, 0x1b, 0x50, 0x65, + 0x65, 0x72, 0x69, 0x6e, 0x67, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x65, 0x42, 0x79, + 0x49, 0x44, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x44, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x49, 0x44, 0x22, 0x1e, 0x0a, 0x1c, 0x50, 0x65, 0x65, + 0x72, 0x69, 0x6e, 0x67, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x65, 0x42, 0x79, 0x49, + 0x44, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x8d, 0x01, 0x0a, 0x1e, 0x50, 0x65, + 0x65, 0x72, 0x69, 0x6e, 0x67, 0x54, 0x72, 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, + 0x57, 0x72, 0x69, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x4b, 0x0a, 0x12, + 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x54, 0x72, 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, + 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, + 0x6e, 0x67, 0x2e, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x54, 0x72, 0x75, 0x73, 0x74, 0x42, + 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x12, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x54, 0x72, + 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x44, 0x61, 0x74, + 0x61, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x44, + 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x22, 0x21, 0x0a, 0x1f, 0x50, 0x65, 0x65, + 0x72, 0x69, 0x6e, 0x67, 0x54, 0x72, 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x57, + 0x72, 0x69, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x73, 0x0a, 0x1f, + 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x54, 0x72, 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, + 0x6c, 0x65, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x12, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, + 0x61, 0x6d, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, + 0x6e, 0x12, 0x1e, 0x0a, 0x0a, 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, 0x74, 0x65, + 0x72, 0x22, 0x22, 0x0a, 0x20, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x54, 0x72, 0x75, 0x73, + 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0xfc, 0x01, 0x0a, 0x14, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, + 0x74, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, + 0x0a, 0x08, 0x50, 0x65, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x08, 0x50, 0x65, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x50, 0x61, + 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x50, + 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1e, 0x0a, 0x0a, 0x44, 0x61, 0x74, 0x61, + 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x44, 0x61, + 0x74, 0x61, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x12, 0x14, 0x0a, 0x05, 0x54, 0x6f, 0x6b, 0x65, + 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x3b, + 0x0a, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x70, + 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x54, + 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x4d, 0x65, 0x74, 0x61, + 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x1a, 0x37, 0x0a, 0x09, 0x4d, + 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x3a, 0x02, 0x38, 0x01, 0x22, 0x3b, 0x0a, 0x15, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, + 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x22, 0x0a, + 0x0c, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x54, 0x6f, 0x6b, 0x65, - 0x6e, 0x12, 0x1c, 0x0a, 0x09, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, - 0x1e, 0x0a, 0x0a, 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x0a, 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x12, - 0x14, 0x0a, 0x05, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, - 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x36, 0x0a, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x18, 0x06, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x49, 0x6e, - 0x69, 0x74, 0x69, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x4d, 0x65, - 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x1a, 0x37, 0x0a, - 0x09, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, - 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x12, 0x0a, 0x10, 0x49, 0x6e, 0x69, 0x74, 0x69, 0x61, - 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x94, 0x05, 0x0a, 0x12, 0x52, - 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, - 0x65, 0x12, 0x3f, 0x0a, 0x07, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x52, 0x65, 0x70, - 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x2e, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, 0x00, 0x52, 0x07, 0x72, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x12, 0x42, 0x0a, 0x08, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x52, - 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, - 0x65, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x00, 0x52, 0x08, 0x72, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x48, 0x0a, 0x0a, 0x74, 0x65, 0x72, 0x6d, 0x69, 0x6e, - 0x61, 0x74, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x70, 0x65, 0x65, - 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x2e, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, - 0x65, 0x64, 0x48, 0x00, 0x52, 0x0a, 0x74, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x65, 0x64, - 0x1a, 0x7f, 0x0a, 0x07, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x50, - 0x65, 0x65, 0x72, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x50, 0x65, 0x65, - 0x72, 0x49, 0x44, 0x12, 0x14, 0x0a, 0x05, 0x4e, 0x6f, 0x6e, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x05, 0x4e, 0x6f, 0x6e, 0x63, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x52, 0x65, 0x73, - 0x6f, 0x75, 0x72, 0x63, 0x65, 0x55, 0x52, 0x4c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, - 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x55, 0x52, 0x4c, 0x12, 0x24, 0x0a, 0x05, 0x45, - 0x72, 0x72, 0x6f, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x73, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x05, 0x45, 0x72, 0x72, 0x6f, - 0x72, 0x1a, 0x94, 0x02, 0x0a, 0x08, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, - 0x0a, 0x05, 0x4e, 0x6f, 0x6e, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x4e, - 0x6f, 0x6e, 0x63, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, - 0x55, 0x52, 0x4c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x52, 0x65, 0x73, 0x6f, 0x75, - 0x72, 0x63, 0x65, 0x55, 0x52, 0x4c, 0x12, 0x1e, 0x0a, 0x0a, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, - 0x63, 0x65, 0x49, 0x44, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x52, 0x65, 0x73, 0x6f, - 0x75, 0x72, 0x63, 0x65, 0x49, 0x44, 0x12, 0x30, 0x0a, 0x08, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, - 0x63, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, - 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x41, 0x6e, 0x79, 0x52, 0x08, - 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x4c, 0x0a, 0x09, 0x6f, 0x70, 0x65, 0x72, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2e, 0x2e, 0x70, 0x65, - 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x6f, 0x70, 0x65, - 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x30, 0x0a, 0x09, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x6e, 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x10, 0x00, - 0x12, 0x0a, 0x0a, 0x06, 0x55, 0x50, 0x53, 0x45, 0x52, 0x54, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, - 0x44, 0x45, 0x4c, 0x45, 0x54, 0x45, 0x10, 0x02, 0x1a, 0x0c, 0x0a, 0x0a, 0x54, 0x65, 0x72, 0x6d, - 0x69, 0x6e, 0x61, 0x74, 0x65, 0x64, 0x42, 0x09, 0x0a, 0x07, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, - 0x64, 0x2a, 0x53, 0x0a, 0x0c, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x53, 0x74, 0x61, 0x74, - 0x65, 0x12, 0x0d, 0x0a, 0x09, 0x55, 0x4e, 0x44, 0x45, 0x46, 0x49, 0x4e, 0x45, 0x44, 0x10, 0x00, - 0x12, 0x0b, 0x0a, 0x07, 0x49, 0x4e, 0x49, 0x54, 0x49, 0x41, 0x4c, 0x10, 0x01, 0x12, 0x0a, 0x0a, - 0x06, 0x41, 0x43, 0x54, 0x49, 0x56, 0x45, 0x10, 0x02, 0x12, 0x0b, 0x0a, 0x07, 0x46, 0x41, 0x49, - 0x4c, 0x49, 0x4e, 0x47, 0x10, 0x03, 0x12, 0x0e, 0x0a, 0x0a, 0x54, 0x45, 0x52, 0x4d, 0x49, 0x4e, - 0x41, 0x54, 0x45, 0x44, 0x10, 0x04, 0x32, 0xea, 0x05, 0x0a, 0x0e, 0x50, 0x65, 0x65, 0x72, 0x69, - 0x6e, 0x67, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x4e, 0x0a, 0x0d, 0x47, 0x65, 0x6e, - 0x65, 0x72, 0x61, 0x74, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1d, 0x2e, 0x70, 0x65, 0x65, - 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x54, 0x6f, 0x6b, - 0x65, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x70, 0x65, 0x65, 0x72, - 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x54, 0x6f, 0x6b, 0x65, - 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3f, 0x0a, 0x08, 0x49, 0x6e, 0x69, - 0x74, 0x69, 0x61, 0x74, 0x65, 0x12, 0x18, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, - 0x49, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x19, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x49, 0x6e, 0x69, 0x74, 0x69, 0x61, - 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x48, 0x0a, 0x0b, 0x50, 0x65, - 0x65, 0x72, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x61, 0x64, 0x12, 0x1b, 0x2e, 0x70, 0x65, 0x65, 0x72, - 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x61, 0x64, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, - 0x2e, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x61, 0x64, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x48, 0x0a, 0x0b, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x4c, - 0x69, 0x73, 0x74, 0x12, 0x1b, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x65, - 0x65, 0x72, 0x69, 0x6e, 0x67, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x1c, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x65, 0x65, 0x72, 0x69, - 0x6e, 0x67, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4e, - 0x0a, 0x0d, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x12, - 0x1d, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, - 0x67, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, - 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, - 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4b, - 0x0a, 0x0c, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x57, 0x72, 0x69, 0x74, 0x65, 0x12, 0x1c, - 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, - 0x57, 0x72, 0x69, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x70, - 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x57, 0x72, - 0x69, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6f, 0x0a, 0x18, 0x54, - 0x72, 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x42, 0x79, - 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x28, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, - 0x67, 0x2e, 0x54, 0x72, 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x4c, 0x69, 0x73, - 0x74, 0x42, 0x79, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x29, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x54, 0x72, 0x75, 0x73, - 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x42, 0x79, 0x53, 0x65, 0x72, - 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, - 0x54, 0x72, 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x61, 0x64, 0x12, - 0x1f, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x54, 0x72, 0x75, 0x73, 0x74, 0x42, - 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x61, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x20, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x54, 0x72, 0x75, 0x73, 0x74, - 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x61, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x4f, 0x0a, 0x0f, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x73, 0x6f, - 0x75, 0x72, 0x63, 0x65, 0x73, 0x12, 0x1b, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, - 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x73, 0x61, - 0x67, 0x65, 0x1a, 0x1b, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x52, 0x65, 0x70, - 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x28, - 0x01, 0x30, 0x01, 0x42, 0x84, 0x01, 0x0a, 0x0b, 0x63, 0x6f, 0x6d, 0x2e, 0x70, 0x65, 0x65, 0x72, - 0x69, 0x6e, 0x67, 0x42, 0x0c, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x50, 0x72, 0x6f, 0x74, - 0x6f, 0x50, 0x01, 0x5a, 0x2b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, - 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2f, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, - 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x70, 0x62, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, - 0xa2, 0x02, 0x03, 0x50, 0x58, 0x58, 0xaa, 0x02, 0x07, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, - 0xca, 0x02, 0x07, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0xe2, 0x02, 0x13, 0x50, 0x65, 0x65, - 0x72, 0x69, 0x6e, 0x67, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, - 0xea, 0x02, 0x07, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x33, + 0x6e, 0x22, 0x96, 0x02, 0x0a, 0x0f, 0x49, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x74, 0x65, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x50, 0x65, 0x65, 0x72, 0x4e, 0x61, 0x6d, + 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x50, 0x65, 0x65, 0x72, 0x4e, 0x61, 0x6d, + 0x65, 0x12, 0x22, 0x0a, 0x0c, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x54, 0x6f, 0x6b, 0x65, + 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, + 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1c, 0x0a, 0x09, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, + 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, + 0x69, 0x6f, 0x6e, 0x12, 0x1e, 0x0a, 0x0a, 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, 0x74, 0x65, + 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, + 0x74, 0x65, 0x72, 0x12, 0x14, 0x0a, 0x05, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x05, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x05, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x36, 0x0a, 0x04, 0x4d, 0x65, 0x74, + 0x61, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, + 0x67, 0x2e, 0x49, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x4d, 0x65, 0x74, + 0x61, 0x1a, 0x37, 0x0a, 0x09, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, + 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, + 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x12, 0x0a, 0x10, 0x49, 0x6e, + 0x69, 0x74, 0x69, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x94, + 0x05, 0x0a, 0x12, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, + 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x3f, 0x0a, 0x07, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, + 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x73, + 0x61, 0x67, 0x65, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, 0x00, 0x52, 0x07, 0x72, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x42, 0x0a, 0x08, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, + 0x6e, 0x67, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, + 0x73, 0x73, 0x61, 0x67, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x00, + 0x52, 0x08, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x48, 0x0a, 0x0a, 0x74, 0x65, + 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x26, + 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x2e, 0x54, 0x65, 0x72, 0x6d, + 0x69, 0x6e, 0x61, 0x74, 0x65, 0x64, 0x48, 0x00, 0x52, 0x0a, 0x74, 0x65, 0x72, 0x6d, 0x69, 0x6e, + 0x61, 0x74, 0x65, 0x64, 0x1a, 0x7f, 0x0a, 0x07, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x16, 0x0a, 0x06, 0x50, 0x65, 0x65, 0x72, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x06, 0x50, 0x65, 0x65, 0x72, 0x49, 0x44, 0x12, 0x14, 0x0a, 0x05, 0x4e, 0x6f, 0x6e, 0x63, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x4e, 0x6f, 0x6e, 0x63, 0x65, 0x12, 0x20, 0x0a, + 0x0b, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x55, 0x52, 0x4c, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x0b, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x55, 0x52, 0x4c, 0x12, + 0x24, 0x0a, 0x05, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, + 0x2e, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x05, + 0x45, 0x72, 0x72, 0x6f, 0x72, 0x1a, 0x94, 0x02, 0x0a, 0x08, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x4e, 0x6f, 0x6e, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x05, 0x4e, 0x6f, 0x6e, 0x63, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x52, 0x65, 0x73, 0x6f, + 0x75, 0x72, 0x63, 0x65, 0x55, 0x52, 0x4c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x52, + 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x55, 0x52, 0x4c, 0x12, 0x1e, 0x0a, 0x0a, 0x52, 0x65, + 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x44, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, + 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x44, 0x12, 0x30, 0x0a, 0x08, 0x52, 0x65, + 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x41, + 0x6e, 0x79, 0x52, 0x08, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x4c, 0x0a, 0x09, + 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0e, 0x32, + 0x2e, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x2e, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, + 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x30, 0x0a, 0x09, 0x4f, 0x70, + 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x6e, 0x6b, 0x6e, 0x6f, + 0x77, 0x6e, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x55, 0x50, 0x53, 0x45, 0x52, 0x54, 0x10, 0x01, + 0x12, 0x0a, 0x0a, 0x06, 0x44, 0x45, 0x4c, 0x45, 0x54, 0x45, 0x10, 0x02, 0x1a, 0x0c, 0x0a, 0x0a, + 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x65, 0x64, 0x42, 0x09, 0x0a, 0x07, 0x50, 0x61, + 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2a, 0x53, 0x0a, 0x0c, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, + 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x0d, 0x0a, 0x09, 0x55, 0x4e, 0x44, 0x45, 0x46, 0x49, 0x4e, + 0x45, 0x44, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x49, 0x4e, 0x49, 0x54, 0x49, 0x41, 0x4c, 0x10, + 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x41, 0x43, 0x54, 0x49, 0x56, 0x45, 0x10, 0x02, 0x12, 0x0b, 0x0a, + 0x07, 0x46, 0x41, 0x49, 0x4c, 0x49, 0x4e, 0x47, 0x10, 0x03, 0x12, 0x0e, 0x0a, 0x0a, 0x54, 0x45, + 0x52, 0x4d, 0x49, 0x4e, 0x41, 0x54, 0x45, 0x44, 0x10, 0x04, 0x32, 0xea, 0x05, 0x0a, 0x0e, 0x50, + 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x4e, 0x0a, + 0x0d, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1d, + 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, + 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, + 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, + 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3f, 0x0a, + 0x08, 0x49, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x74, 0x65, 0x12, 0x18, 0x2e, 0x70, 0x65, 0x65, 0x72, + 0x69, 0x6e, 0x67, 0x2e, 0x49, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x49, 0x6e, + 0x69, 0x74, 0x69, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x48, + 0x0a, 0x0b, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x61, 0x64, 0x12, 0x1b, 0x2e, + 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x52, + 0x65, 0x61, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x70, 0x65, 0x65, + 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x61, 0x64, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x48, 0x0a, 0x0b, 0x50, 0x65, 0x65, 0x72, + 0x69, 0x6e, 0x67, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x1b, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, + 0x67, 0x2e, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x50, + 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x4e, 0x0a, 0x0d, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x44, 0x65, 0x6c, + 0x65, 0x74, 0x65, 0x12, 0x1d, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x65, + 0x65, 0x72, 0x69, 0x6e, 0x67, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x65, 0x65, + 0x72, 0x69, 0x6e, 0x67, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x4b, 0x0a, 0x0c, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x57, 0x72, 0x69, + 0x74, 0x65, 0x12, 0x1c, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x65, 0x65, + 0x72, 0x69, 0x6e, 0x67, 0x57, 0x72, 0x69, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x1d, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x65, 0x65, 0x72, 0x69, + 0x6e, 0x67, 0x57, 0x72, 0x69, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x6f, 0x0a, 0x18, 0x54, 0x72, 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x4c, 0x69, + 0x73, 0x74, 0x42, 0x79, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x28, 0x2e, 0x70, 0x65, + 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x54, 0x72, 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, + 0x65, 0x4c, 0x69, 0x73, 0x74, 0x42, 0x79, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, + 0x54, 0x72, 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x42, + 0x79, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x54, 0x0a, 0x0f, 0x54, 0x72, 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, + 0x65, 0x61, 0x64, 0x12, 0x1f, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x54, 0x72, + 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x61, 0x64, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x2e, 0x54, + 0x72, 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x61, 0x64, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4f, 0x0a, 0x0f, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, + 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x12, 0x1b, 0x2e, 0x70, 0x65, 0x65, 0x72, + 0x69, 0x6e, 0x67, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4d, + 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x1b, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, + 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x73, + 0x61, 0x67, 0x65, 0x28, 0x01, 0x30, 0x01, 0x42, 0x84, 0x01, 0x0a, 0x0b, 0x63, 0x6f, 0x6d, 0x2e, + 0x70, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x42, 0x0c, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, + 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x2b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, + 0x63, 0x6f, 0x6d, 0x2f, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2f, 0x63, 0x6f, + 0x6e, 0x73, 0x75, 0x6c, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x70, 0x62, 0x70, 0x65, 0x65, + 0x72, 0x69, 0x6e, 0x67, 0xa2, 0x02, 0x03, 0x50, 0x58, 0x58, 0xaa, 0x02, 0x07, 0x50, 0x65, 0x65, + 0x72, 0x69, 0x6e, 0x67, 0xca, 0x02, 0x07, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0xe2, 0x02, + 0x13, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, + 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x07, 0x50, 0x65, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x62, 0x06, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/proto/pbpeering/peering.proto b/proto/pbpeering/peering.proto index 2103c3fe63..8ac8e353fe 100644 --- a/proto/pbpeering/peering.proto +++ b/proto/pbpeering/peering.proto @@ -181,15 +181,17 @@ message PeeringDeleteResponse {} // @consul-rpc-glue: Datacenter,ReadTODO message TrustBundleListByServiceRequest { string ServiceName = 1; - string Partition = 2; + string Namespace = 2; + string Partition = 3; // these are common fields required for implementing structs.RPCInfo methods // that are used to forward requests - string Datacenter = 3; + string Datacenter = 4; } message TrustBundleListByServiceResponse { - repeated PeeringTrustBundle Bundles = 1; + uint64 Index = 1; + repeated PeeringTrustBundle Bundles = 2; } // @consul-rpc-glue: Datacenter,ReadTODO diff --git a/proto/prototest/testing.go b/proto/prototest/testing.go index b17f359b37..c196d77b3a 100644 --- a/proto/prototest/testing.go +++ b/proto/prototest/testing.go @@ -7,7 +7,7 @@ import ( "google.golang.org/protobuf/testing/protocmp" ) -func AssertDeepEqual(t *testing.T, x, y interface{}, opts ...cmp.Option) { +func AssertDeepEqual(t testing.TB, x, y interface{}, opts ...cmp.Option) { t.Helper() opts = append(opts, protocmp.Transform())