From 7c19c701e15dd1b40d63c2441149b3d32c3face8 Mon Sep 17 00:00:00 2001 From: Max Bowsher Date: Sun, 19 Jun 2022 11:58:23 +0100 Subject: [PATCH 01/31] Fix incorrect name and doc for kv_entries metric The name of the metric as registered with the metrics library to provide the help string, was incorrect compared with the actual code that sets the metric value - bring them into sync. Also, the help message was incorrect. Rather than copy the help message from telemetry.mdx, which was correct, but felt a bit unnatural in the way it was worded, update both of them to a new wording. --- agent/consul/usagemetrics/usagemetrics.go | 4 ++-- website/content/docs/agent/telemetry.mdx | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/agent/consul/usagemetrics/usagemetrics.go b/agent/consul/usagemetrics/usagemetrics.go index 6733ed3df0..0d74e0e72f 100644 --- a/agent/consul/usagemetrics/usagemetrics.go +++ b/agent/consul/usagemetrics/usagemetrics.go @@ -37,8 +37,8 @@ var Gauges = []prometheus.GaugeDefinition{ Help: "Measures the current number of server agents registered with Consul. It is only emitted by Consul servers. Added in v1.9.6.", }, { - Name: []string{"consul", "kv", "entries"}, - Help: "Measures the current number of server agents registered with Consul. It is only emitted by Consul servers. Added in v1.10.3.", + Name: []string{"consul", "state", "kv_entries"}, + Help: "Measures the current number of entries in the Consul KV store. It is only emitted by Consul servers. Added in v1.10.3.", }, { Name: []string{"consul", "state", "connect_instances"}, diff --git a/website/content/docs/agent/telemetry.mdx b/website/content/docs/agent/telemetry.mdx index 0435a42c84..01adb30a11 100644 --- a/website/content/docs/agent/telemetry.mdx +++ b/website/content/docs/agent/telemetry.mdx @@ -391,7 +391,7 @@ This is a full list of metrics emitted by Consul. | `consul.state.nodes` | Measures the current number of nodes registered with Consul. It is only emitted by Consul servers. Added in v1.9.0. | number of objects | gauge | | `consul.state.services` | Measures the current number of unique services registered with Consul, based on service name. It is only emitted by Consul servers. Added in v1.9.0. | number of objects | gauge | | `consul.state.service_instances` | Measures the current number of unique service instances registered with Consul. It is only emitted by Consul servers. Added in v1.9.0. | number of objects | gauge | -| `consul.state.kv_entries` | Measures the current number of unique KV entries written in Consul. It is only emitted by Consul servers. Added in v1.10.3. | number of objects | gauge | +| `consul.state.kv_entries` | Measures the current number of entries in the Consul KV store. It is only emitted by Consul servers. Added in v1.10.3. | number of objects | gauge | | `consul.state.connect_instances` | Measures the current number of unique connect service instances registered with Consul labeled by Kind (e.g. connect-proxy, connect-native, etc). Added in v1.10.4 | number of objects | gauge | | `consul.state.config_entries` | Measures the current number of configuration entries registered with Consul labeled by Kind (e.g. service-defaults, proxy-defaults, etc). See [Configuration Entries](/docs/connect/config-entries) for more information. Added in v1.10.4 | number of objects | gauge | | `consul.members.clients` | Measures the current number of client agents registered with Consul. It is only emitted by Consul servers. Added in v1.9.6. | number of clients | gauge | From 3c4fa9b4684dc62be1faa9da6a3a3c6cef3d0fb9 Mon Sep 17 00:00:00 2001 From: Daniel Kimsey Date: Wed, 10 Aug 2022 16:52:32 -0500 Subject: [PATCH 02/31] Add support for filtering the 'List Services' API 1. Create a bexpr filter for performing the filtering 2. Change the state store functions to return the raw (not aggregated) list of ServiceNodes. 3. Move the aggregate service tags by name logic out of the state store functions into a new function called from the RPC endpoint 4. Perform the filtering in the endpoint before aggregation. --- .changelog/11742.txt | 3 ++ agent/consul/catalog_endpoint.go | 42 ++++++++++++++++++- agent/consul/catalog_endpoint_test.go | 39 ++++++++++++++++++ agent/consul/state/catalog.go | 55 ++++--------------------- agent/consul/state/catalog_test.go | 56 +++++++++++++------------- agent/consul/state/state_store_test.go | 11 ++--- website/content/api-docs/catalog.mdx | 53 +++++++++++++++++++++++- 7 files changed, 177 insertions(+), 82 deletions(-) create mode 100644 .changelog/11742.txt diff --git a/.changelog/11742.txt b/.changelog/11742.txt new file mode 100644 index 0000000000..6c6d4c2498 --- /dev/null +++ b/.changelog/11742.txt @@ -0,0 +1,3 @@ +```release-note:improvement +api: Add filtering support to Catalog's List Services (v1/catalog/services) +``` diff --git a/agent/consul/catalog_endpoint.go b/agent/consul/catalog_endpoint.go index 111ee7b2ba..696ae314a7 100644 --- a/agent/consul/catalog_endpoint.go +++ b/agent/consul/catalog_endpoint.go @@ -565,6 +565,11 @@ func (c *Catalog) ListServices(args *structs.DCSpecificRequest, reply *structs.I return err } + filter, err := bexpr.CreateFilter(args.Filter, nil, []*structs.ServiceNode{}) + if err != nil { + return err + } + // Set reply enterprise metadata after resolving and validating the token so // that we can properly infer metadata from the token. reply.EnterpriseMeta = args.EnterpriseMeta @@ -574,10 +579,11 @@ func (c *Catalog) ListServices(args *structs.DCSpecificRequest, reply *structs.I &reply.QueryMeta, func(ws memdb.WatchSet, state *state.Store) error { var err error + var serviceNodes structs.ServiceNodes if len(args.NodeMetaFilters) > 0 { - reply.Index, reply.Services, err = state.ServicesByNodeMeta(ws, args.NodeMetaFilters, &args.EnterpriseMeta, args.PeerName) + reply.Index, serviceNodes, err = state.ServicesByNodeMeta(ws, args.NodeMetaFilters, &args.EnterpriseMeta, args.PeerName) } else { - reply.Index, reply.Services, err = state.Services(ws, &args.EnterpriseMeta, args.PeerName) + reply.Index, serviceNodes, err = state.Services(ws, &args.EnterpriseMeta, args.PeerName) } if err != nil { return err @@ -588,11 +594,43 @@ func (c *Catalog) ListServices(args *structs.DCSpecificRequest, reply *structs.I return nil } + raw, err := filter.Execute(serviceNodes) + if err != nil { + return err + } + + reply.Services = servicesTagsByName(raw.(structs.ServiceNodes)) + c.srv.filterACLWithAuthorizer(authz, reply) + return nil }) } +func servicesTagsByName(services []*structs.ServiceNode) structs.Services { + unique := make(map[string]map[string]struct{}) + for _, svc := range services { + tags, ok := unique[svc.ServiceName] + if !ok { + unique[svc.ServiceName] = make(map[string]struct{}) + tags = unique[svc.ServiceName] + } + for _, tag := range svc.ServiceTags { + tags[tag] = struct{}{} + } + } + + // Generate the output structure. + var results = make(structs.Services) + for service, tags := range unique { + results[service] = make([]string, 0, len(tags)) + for tag := range tags { + results[service] = append(results[service], tag) + } + } + return results +} + // ServiceList is used to query the services in a DC. // Returns services as a list of ServiceNames. func (c *Catalog) ServiceList(args *structs.DCSpecificRequest, reply *structs.IndexedServiceList) error { diff --git a/agent/consul/catalog_endpoint_test.go b/agent/consul/catalog_endpoint_test.go index ca00efaea2..daa22c90c1 100644 --- a/agent/consul/catalog_endpoint_test.go +++ b/agent/consul/catalog_endpoint_test.go @@ -1523,6 +1523,45 @@ func TestCatalog_ListServices_NodeMetaFilter(t *testing.T) { } } +func TestCatalog_ListServices_Filter(t *testing.T) { + t.Parallel() + _, s1 := testServer(t) + codec := rpcClient(t, s1) + + testrpc.WaitForTestAgent(t, s1.RPC, "dc1") + + // prep the cluster with some data we can use in our filters + registerTestCatalogEntries(t, codec) + + // Run the tests against the test server + + t.Run("ListServices", func(t *testing.T) { + args := structs.DCSpecificRequest{ + Datacenter: "dc1", + } + + args.Filter = "ServiceName == redis" + out := new(structs.IndexedServices) + require.NoError(t, msgpackrpc.CallWithCodec(codec, "Catalog.ListServices", &args, out)) + require.Contains(t, out.Services, "redis") + require.ElementsMatch(t, []string{"v1", "v2"}, out.Services["redis"]) + + args.Filter = "NodeMeta.os == NoSuchOS" + out = new(structs.IndexedServices) + require.NoError(t, msgpackrpc.CallWithCodec(codec, "Catalog.ListServices", &args, out)) + require.Len(t, out.Services, 0) + + args.Filter = "NodeMeta.NoSuchMetadata == linux" + out = new(structs.IndexedServices) + require.NoError(t, msgpackrpc.CallWithCodec(codec, "Catalog.ListServices", &args, out)) + require.Len(t, out.Services, 0) + + args.Filter = "InvalidField == linux" + out = new(structs.IndexedServices) + require.Error(t, msgpackrpc.CallWithCodec(codec, "Catalog.ListServices", &args, out)) + }) +} + func TestCatalog_ListServices_Blocking(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") diff --git a/agent/consul/state/catalog.go b/agent/consul/state/catalog.go index 258519d5ba..879c59f747 100644 --- a/agent/consul/state/catalog.go +++ b/agent/consul/state/catalog.go @@ -1134,7 +1134,7 @@ func terminatingGatewayVirtualIPsSupported(tx ReadTxn, ws memdb.WatchSet) (bool, } // Services returns all services along with a list of associated tags. -func (s *Store) Services(ws memdb.WatchSet, entMeta *acl.EnterpriseMeta, peerName string) (uint64, structs.Services, error) { +func (s *Store) Services(ws memdb.WatchSet, entMeta *acl.EnterpriseMeta, peerName string) (uint64, []*structs.ServiceNode, error) { tx := s.db.Txn(false) defer tx.Abort() @@ -1148,30 +1148,11 @@ func (s *Store) Services(ws memdb.WatchSet, entMeta *acl.EnterpriseMeta, peerNam } ws.Add(services.WatchCh()) - // Rip through the services and enumerate them and their unique set of - // tags. - unique := make(map[string]map[string]struct{}) + var result []*structs.ServiceNode for service := services.Next(); service != nil; service = services.Next() { - svc := service.(*structs.ServiceNode) - tags, ok := unique[svc.ServiceName] - if !ok { - unique[svc.ServiceName] = make(map[string]struct{}) - tags = unique[svc.ServiceName] - } - for _, tag := range svc.ServiceTags { - tags[tag] = struct{}{} - } + result = append(result, service.(*structs.ServiceNode)) } - - // Generate the output structure. - var results = make(structs.Services) - for service, tags := range unique { - results[service] = make([]string, 0, len(tags)) - for tag := range tags { - results[service] = append(results[service], tag) - } - } - return idx, results, nil + return idx, result, nil } func (s *Store) ServiceList(ws memdb.WatchSet, entMeta *acl.EnterpriseMeta, peerName string) (uint64, structs.ServiceList, error) { @@ -1212,7 +1193,7 @@ func serviceListTxn(tx ReadTxn, ws memdb.WatchSet, entMeta *acl.EnterpriseMeta, } // ServicesByNodeMeta returns all services, filtered by the given node metadata. -func (s *Store) ServicesByNodeMeta(ws memdb.WatchSet, filters map[string]string, entMeta *acl.EnterpriseMeta, peerName string) (uint64, structs.Services, error) { +func (s *Store) ServicesByNodeMeta(ws memdb.WatchSet, filters map[string]string, entMeta *acl.EnterpriseMeta, peerName string) (uint64, []*structs.ServiceNode, error) { tx := s.db.Txn(false) defer tx.Abort() @@ -1259,8 +1240,7 @@ func (s *Store) ServicesByNodeMeta(ws memdb.WatchSet, filters map[string]string, } allServicesCh := allServices.WatchCh() - // Populate the services map - unique := make(map[string]map[string]struct{}) + var result structs.ServiceNodes for node := nodes.Next(); node != nil; node = nodes.Next() { n := node.(*structs.Node) if len(filters) > 1 && !structs.SatisfiesMetaFilters(n.Meta, filters) { @@ -1274,30 +1254,11 @@ func (s *Store) ServicesByNodeMeta(ws memdb.WatchSet, filters map[string]string, } ws.AddWithLimit(watchLimit, services.WatchCh(), allServicesCh) - // Rip through the services and enumerate them and their unique set of - // tags. for service := services.Next(); service != nil; service = services.Next() { - svc := service.(*structs.ServiceNode) - tags, ok := unique[svc.ServiceName] - if !ok { - unique[svc.ServiceName] = make(map[string]struct{}) - tags = unique[svc.ServiceName] - } - for _, tag := range svc.ServiceTags { - tags[tag] = struct{}{} - } + result = append(result, service.(*structs.ServiceNode)) } } - - // Generate the output structure. - var results = make(structs.Services) - for service, tags := range unique { - results[service] = make([]string, 0, len(tags)) - for tag := range tags { - results[service] = append(results[service], tag) - } - } - return idx, results, nil + return idx, result, nil } // maxIndexForService return the maximum Raft Index for a service diff --git a/agent/consul/state/catalog_test.go b/agent/consul/state/catalog_test.go index 10e7af6dba..ca2bded03b 100644 --- a/agent/consul/state/catalog_test.go +++ b/agent/consul/state/catalog_test.go @@ -12,6 +12,8 @@ import ( "github.com/hashicorp/consul/acl" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" "github.com/hashicorp/go-memdb" "github.com/hashicorp/go-uuid" "github.com/stretchr/testify/assert" @@ -2105,7 +2107,7 @@ func TestStateStore_Services(t *testing.T) { if err := s.EnsureService(2, "node1", ns1); err != nil { t.Fatalf("err: %s", err) } - testRegisterService(t, s, 3, "node1", "dogs") + ns1Dogs := testRegisterService(t, s, 3, "node1", "dogs") testRegisterNode(t, s, 4, "node2") ns2 := &structs.NodeService{ ID: "service3", @@ -2131,19 +2133,13 @@ func TestStateStore_Services(t *testing.T) { t.Fatalf("bad index: %d", idx) } - // Verify the result. We sort the lists since the order is - // non-deterministic (it's built using a map internally). - expected := structs.Services{ - "redis": []string{"prod", "primary", "replica"}, - "dogs": []string{}, - } - sort.Strings(expected["redis"]) - for _, tags := range services { - sort.Strings(tags) - } - if !reflect.DeepEqual(expected, services) { - t.Fatalf("bad: %#v", services) + // Verify the result. + expected := []*structs.ServiceNode{ + ns1Dogs.ToServiceNode("node1"), + ns1.ToServiceNode("node1"), + ns2.ToServiceNode("node2"), } + assertDeepEqual(t, services, expected, cmpopts.IgnoreFields(structs.ServiceNode{}, "RaftIndex")) // Deleting a node with a service should fire the watch. if err := s.DeleteNode(6, "node1", nil, ""); err != nil { @@ -2206,11 +2202,10 @@ func TestStateStore_ServicesByNodeMeta(t *testing.T) { if err != nil { t.Fatalf("err: %s", err) } - expected := structs.Services{ - "redis": []string{"primary", "prod"}, + expected := []*structs.ServiceNode{ + ns1.ToServiceNode("node0"), } - sort.Strings(res["redis"]) - require.Equal(t, expected, res) + assertDeepEqual(t, res, expected, cmpopts.IgnoreFields(structs.ServiceNode{}, "RaftIndex")) }) t.Run("Get all services using the common meta value", func(t *testing.T) { @@ -2218,11 +2213,12 @@ func TestStateStore_ServicesByNodeMeta(t *testing.T) { if err != nil { t.Fatalf("err: %s", err) } - expected := structs.Services{ - "redis": []string{"primary", "prod", "replica"}, + require.Len(t, res, 2) + expected := []*structs.ServiceNode{ + ns1.ToServiceNode("node0"), + ns2.ToServiceNode("node1"), } - sort.Strings(res["redis"]) - require.Equal(t, expected, res) + assertDeepEqual(t, res, expected, cmpopts.IgnoreFields(structs.ServiceNode{}, "RaftIndex")) }) t.Run("Get an empty list for an invalid meta value", func(t *testing.T) { @@ -2230,8 +2226,8 @@ func TestStateStore_ServicesByNodeMeta(t *testing.T) { if err != nil { t.Fatalf("err: %s", err) } - expected := structs.Services{} - require.Equal(t, expected, res) + var expected []*structs.ServiceNode + assertDeepEqual(t, res, expected, cmpopts.IgnoreFields(structs.ServiceNode{}, "RaftIndex")) }) t.Run("Get the first node's service instance using multiple meta filters", func(t *testing.T) { @@ -2239,11 +2235,10 @@ func TestStateStore_ServicesByNodeMeta(t *testing.T) { if err != nil { t.Fatalf("err: %s", err) } - expected := structs.Services{ - "redis": []string{"primary", "prod"}, + expected := []*structs.ServiceNode{ + ns1.ToServiceNode("node0"), } - sort.Strings(res["redis"]) - require.Equal(t, expected, res) + assertDeepEqual(t, res, expected, cmpopts.IgnoreFields(structs.ServiceNode{}, "RaftIndex")) }) t.Run("Registering some unrelated node + service should not fire the watch.", func(t *testing.T) { @@ -8807,3 +8802,10 @@ func setVirtualIPFlags(t *testing.T, s *Store) { Value: "true", })) } + +func assertDeepEqual(t *testing.T, x, y interface{}, opts ...cmp.Option) { + t.Helper() + if diff := cmp.Diff(x, y, opts...); diff != "" { + t.Fatalf("assertion failed: values are not equal\n--- expected\n+++ actual\n%v", diff) + } +} diff --git a/agent/consul/state/state_store_test.go b/agent/consul/state/state_store_test.go index c8460ca821..88e5418c8d 100644 --- a/agent/consul/state/state_store_test.go +++ b/agent/consul/state/state_store_test.go @@ -146,13 +146,13 @@ func testRegisterServiceOpts(t *testing.T, s *Store, idx uint64, nodeID, service // testRegisterServiceWithChange registers a service and allow ensuring the consul index is updated // even if service already exists if using `modifyAccordingIndex`. // This is done by setting the transaction ID in "version" meta so service will be updated if it already exists -func testRegisterServiceWithChange(t *testing.T, s *Store, idx uint64, nodeID, serviceID string, modifyAccordingIndex bool) { - testRegisterServiceWithChangeOpts(t, s, idx, nodeID, serviceID, modifyAccordingIndex) +func testRegisterServiceWithChange(t *testing.T, s *Store, idx uint64, nodeID, serviceID string, modifyAccordingIndex bool) *structs.NodeService { + return testRegisterServiceWithChangeOpts(t, s, idx, nodeID, serviceID, modifyAccordingIndex) } // testRegisterServiceWithChangeOpts is the same as testRegisterServiceWithChange with the addition of opts that can // modify the service prior to writing. -func testRegisterServiceWithChangeOpts(t *testing.T, s *Store, idx uint64, nodeID, serviceID string, modifyAccordingIndex bool, opts ...func(service *structs.NodeService)) { +func testRegisterServiceWithChangeOpts(t *testing.T, s *Store, idx uint64, nodeID, serviceID string, modifyAccordingIndex bool, opts ...func(service *structs.NodeService)) *structs.NodeService { meta := make(map[string]string) if modifyAccordingIndex { meta["version"] = fmt.Sprint(idx) @@ -183,14 +183,15 @@ func testRegisterServiceWithChangeOpts(t *testing.T, s *Store, idx uint64, nodeI result.ServiceID != serviceID { t.Fatalf("bad service: %#v", result) } + return svc } // testRegisterService register a service with given transaction idx // If the service already exists, transaction number might not be increased // Use `testRegisterServiceWithChange()` if you want perform a registration that // ensures the transaction is updated by setting idx in Meta of Service -func testRegisterService(t *testing.T, s *Store, idx uint64, nodeID, serviceID string) { - testRegisterServiceWithChange(t, s, idx, nodeID, serviceID, false) +func testRegisterService(t *testing.T, s *Store, idx uint64, nodeID, serviceID string) *structs.NodeService { + return testRegisterServiceWithChange(t, s, idx, nodeID, serviceID, false) } func testRegisterConnectService(t *testing.T, s *Store, idx uint64, nodeID, serviceID string) { diff --git a/website/content/api-docs/catalog.mdx b/website/content/api-docs/catalog.mdx index b259176850..86480ab79b 100644 --- a/website/content/api-docs/catalog.mdx +++ b/website/content/api-docs/catalog.mdx @@ -410,13 +410,64 @@ The corresponding CLI command is [`consul catalog services`](/commands/catalog/s - `dc` `(string: "")` - Specifies the datacenter to query. This will default to the datacenter of the agent being queried. -- `node-meta` `(string: "")` - Specifies a desired node metadata key/value pair +- `node-meta` `(string: "")` **Deprecated** - Use `filter` with the `NodeMeta` selector instead. + This parameter will be removed in a future version of Consul. + Specifies a desired node metadata key/value pair of the form `key:value`. This parameter can be specified multiple times, and filters the results to nodes with the specified key/value pairs. - `ns` `(string: "")` - Specifies the namespace of the services you lookup. You can also [specify the namespace through other methods](#methods-to-specify-namespace). +- `filter` `(string: "")` - Specifies the expression used to filter the + queries results prior to returning the data. + +### Filtering + +The filter will be executed against each Service mapping within the catalog. +The following selectors and filter operations are supported: + +| Selector | Supported Operations | +| ---------------------------------------------------- | -------------------------------------------------- | +| `Address` | Equal, Not Equal, In, Not In, Matches, Not Matches | +| `Datacenter` | Equal, Not Equal, In, Not In, Matches, Not Matches | +| `ID` | Equal, Not Equal, In, Not In, Matches, Not Matches | +| `Node` | Equal, Not Equal, In, Not In, Matches, Not Matches | +| `NodeMeta.` | Equal, Not Equal, In, Not In, Matches, Not Matches | +| `NodeMeta` | Is Empty, Is Not Empty, In, Not In | +| `ServiceAddress` | Equal, Not Equal, In, Not In, Matches, Not Matches | +| `ServiceConnect.Native` | Equal, Not Equal | +| `ServiceEnableTagOverride` | Equal, Not Equal | +| `ServiceID` | Equal, Not Equal, In, Not In, Matches, Not Matches | +| `ServiceKind` | Equal, Not Equal, In, Not In, Matches, Not Matches | +| `ServiceMeta.` | Equal, Not Equal, In, Not In, Matches, Not Matches | +| `ServiceMeta` | Is Empty, Is Not Empty, In, Not In | +| `ServiceName` | Equal, Not Equal, In, Not In, Matches, Not Matches | +| `ServicePort` | Equal, Not Equal | +| `ServiceProxy.DestinationServiceID` | Equal, Not Equal, In, Not In, Matches, Not Matches | +| `ServiceProxy.DestinationServiceName` | Equal, Not Equal, In, Not In, Matches, Not Matches | +| `ServiceProxy.LocalServiceAddress` | Equal, Not Equal, In, Not In, Matches, Not Matches | +| `ServiceProxy.LocalServicePort` | Equal, Not Equal | +| `ServiceProxy.Mode` | Equal, Not Equal, In, Not In, Matches, Not Matches | +| `ServiceProxy.TransparentProxy.OutboundListenerPort` | Equal, Not Equal | +| `ServiceProxy.MeshGateway.Mode` | Equal, Not Equal, In, Not In, Matches, Not Matches | +| `ServiceProxy.Upstreams.Datacenter` | Equal, Not Equal, In, Not In, Matches, Not Matches | +| `ServiceProxy.Upstreams.DestinationName` | Equal, Not Equal, In, Not In, Matches, Not Matches | +| `ServiceProxy.Upstreams.DestinationNamespace` | Equal, Not Equal, In, Not In, Matches, Not Matches | +| `ServiceProxy.Upstreams.DestinationType` | Equal, Not Equal, In, Not In, Matches, Not Matches | +| `ServiceProxy.Upstreams.LocalBindAddress` | Equal, Not Equal, In, Not In, Matches, Not Matches | +| `ServiceProxy.Upstreams.LocalBindPort` | Equal, Not Equal | +| `ServiceProxy.Upstreams.MeshGateway.Mode` | Equal, Not Equal, In, Not In, Matches, Not Matches | +| `ServiceProxy.Upstreams` | Is Empty, Is Not Empty | +| `ServiceTaggedAddresses..Address` | Equal, Not Equal, In, Not In, Matches, Not Matches | +| `ServiceTaggedAddresses..Port` | Equal, Not Equal | +| `ServiceTaggedAddresses` | Is Empty, Is Not Empty, In, Not In | +| `ServiceTags` | In, Not In, Is Empty, Is Not Empty | +| `ServiceWeights.Passing` | Equal, Not Equal | +| `ServiceWeights.Warning` | Equal, Not Equal | +| `TaggedAddresses.` | Equal, Not Equal, In, Not In, Matches, Not Matches | +| `TaggedAddresses` | Is Empty, Is Not Empty, In, Not In | + ### Sample Request ```shell-session From 25675c8bc6227fe13f557479329856f58b121d36 Mon Sep 17 00:00:00 2001 From: Max Bowsher Date: Sun, 14 Aug 2022 16:16:41 +0100 Subject: [PATCH 03/31] Correct problem with merge from master, including reformat of table --- website/content/docs/agent/telemetry.mdx | 110 +++++++++++------------ 1 file changed, 55 insertions(+), 55 deletions(-) diff --git a/website/content/docs/agent/telemetry.mdx b/website/content/docs/agent/telemetry.mdx index c297090575..12f04e8825 100644 --- a/website/content/docs/agent/telemetry.mdx +++ b/website/content/docs/agent/telemetry.mdx @@ -349,59 +349,59 @@ populated free list structure. This is a full list of metrics emitted by Consul. -| Metric | Description | Unit | Type | -| -------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------- | ------- | -| `consul.acl.blocked.{check,service}.deregistration` | Increments whenever a deregistration fails for an entity (check or service) is blocked by an ACL. | requests | counter | -| `consul.acl.blocked.{check,node,service}.registration` | Increments whenever a registration fails for an entity (check, node or service) is blocked by an ACL. | requests | counter | -| `consul.api.http` | This samples how long it takes to service the given HTTP request for the given verb and path. Includes labels for `path` and `method`. `path` does not include details like service or key names, for these an underscore will be present as a placeholder (eg. path=`v1.kv._`) | ms | timer | -| `consul.client.rpc` | Increments whenever a Consul agent in client mode makes an RPC request to a Consul server. This gives a measure of how much a given agent is loading the Consul servers. Currently, this is only generated by agents in client mode, not Consul servers. | requests | counter | -| `consul.client.rpc.exceeded` | Increments whenever a Consul agent in client mode makes an RPC request to a Consul server gets rate limited by that agent's [`limits`](/docs/agent/config/config-files#limits) configuration. This gives an indication that there's an abusive application making too many requests on the agent, or that the rate limit needs to be increased. Currently, this only applies to agents in client mode, not Consul servers. | rejected requests | counter | -| `consul.client.rpc.failed` | Increments whenever a Consul agent in client mode makes an RPC request to a Consul server and fails. | requests | counter | -| `consul.client.api.catalog_register.` | Increments whenever a Consul agent receives a catalog register request. | requests | counter | -| `consul.client.api.success.catalog_register.` | Increments whenever a Consul agent successfully responds to a catalog register request. | requests | counter | -| `consul.client.rpc.error.catalog_register.` | Increments whenever a Consul agent receives an RPC error for a catalog register request. | errors | counter | -| `consul.client.api.catalog_deregister.` | Increments whenever a Consul agent receives a catalog deregister request. | requests | counter | -| `consul.client.api.success.catalog_deregister.` | Increments whenever a Consul agent successfully responds to a catalog deregister request. | requests | counter | -| `consul.client.rpc.error.catalog_deregister.` | Increments whenever a Consul agent receives an RPC error for a catalog deregister request. | errors | counter | -| `consul.client.api.catalog_datacenters.` | Increments whenever a Consul agent receives a request to list datacenters in the catalog. | requests | counter | -| `consul.client.api.success.catalog_datacenters.` | Increments whenever a Consul agent successfully responds to a request to list datacenters. | requests | counter | -| `consul.client.rpc.error.catalog_datacenters.` | Increments whenever a Consul agent receives an RPC error for a request to list datacenters. | errors | counter | -| `consul.client.api.catalog_nodes.` | Increments whenever a Consul agent receives a request to list nodes from the catalog. | requests | counter | -| `consul.client.api.success.catalog_nodes.` | Increments whenever a Consul agent successfully responds to a request to list nodes. | requests | counter | -| `consul.client.rpc.error.catalog_nodes.` | Increments whenever a Consul agent receives an RPC error for a request to list nodes. | errors | counter | -| `consul.client.api.catalog_services.` | Increments whenever a Consul agent receives a request to list services from the catalog. | requests | counter | -| `consul.client.api.success.catalog_services.` | Increments whenever a Consul agent successfully responds to a request to list services. | requests | counter | -| `consul.client.rpc.error.catalog_services.` | Increments whenever a Consul agent receives an RPC error for a request to list services. | errors | counter | -| `consul.client.api.catalog_service_nodes.` | Increments whenever a Consul agent receives a request to list nodes offering a service. | requests | counter | -| `consul.client.api.success.catalog_service_nodes.` | Increments whenever a Consul agent successfully responds to a request to list nodes offering a service. | requests | counter | -| `consul.client.api.error.catalog_service_nodes.` | Increments whenever a Consul agent receives an RPC error for request to list nodes offering a service. | requests | counter | -| `consul.client.rpc.error.catalog_service_nodes.` | Increments whenever a Consul agent receives an RPC error for a request to list nodes offering a service.   | errors | counter | -| `consul.client.api.catalog_node_services.` | Increments whenever a Consul agent receives a request to list services registered in a node.   | requests | counter | -| `consul.client.api.success.catalog_node_services.` | Increments whenever a Consul agent successfully responds to a request to list services in a node.   | requests | counter | -| `consul.client.rpc.error.catalog_node_services.` | Increments whenever a Consul agent receives an RPC error for a request to list services in a node.   | errors | counter | -| `consul.client.api.catalog_node_service_list` | Increments whenever a Consul agent receives a request to list a node's registered services. | requests | counter | -| `consul.client.rpc.error.catalog_node_service_list` | Increments whenever a Consul agent receives an RPC error for request to list a node's registered services. | errors | counter | -| `consul.client.api.success.catalog_node_service_list` | Increments whenever a Consul agent successfully responds to a request to list a node's registered services. | requests | counter | -| `consul.client.api.catalog_gateway_services.` | Increments whenever a Consul agent receives a request to list services associated with a gateway. | requests | counter | -| `consul.client.api.success.catalog_gateway_services.` | Increments whenever a Consul agent successfully responds to a request to list services associated with a gateway. | requests | counter | -| `consul.client.rpc.error.catalog_gateway_services.` | Increments whenever a Consul agent receives an RPC error for a request to list services associated with a gateway. | errors | counter | -| `consul.runtime.num_goroutines` | Tracks the number of running goroutines and is a general load pressure indicator. This may burst from time to time but should return to a steady state value. | number of goroutines | gauge | -| `consul.runtime.alloc_bytes` | Measures the number of bytes allocated by the Consul process. This may burst from time to time but should return to a steady state value. | bytes | gauge | -| `consul.runtime.heap_objects` | Measures the number of objects allocated on the heap and is a general memory pressure indicator. This may burst from time to time but should return to a steady state value. | number of objects | gauge | -| `consul.state.nodes` | Measures the current number of nodes registered with Consul. It is only emitted by Consul servers. Added in v1.9.0. | number of objects | gauge | -| `consul.state.peerings` | Measures the current number of peerings registered with Consul. It is only emitted by Consul servers. Added in v1.13.0. | number of objects | gauge | -| `consul.state.services` | Measures the current number of unique services registered with Consul, based on service name. It is only emitted by Consul servers. Added in v1.9.0. | number of objects | gauge | -| `consul.state.service_instances` | Measures the current number of unique service instances registered with Consul. It is only emitted by Consul servers. Added in v1.9.0. | number of objects | gauge | -| `consul.state.kv_entries` | Measures the current number of entries in the Consul KV store. It is only emitted by Consul servers. Added in v1.10.3. | number of objects | gauge | -| `consul.state.connect_instances` | Measures the current number of unique connect service instances registered with Consul labeled by Kind (e.g. connect-proxy, connect-native, etc). Added in v1.10.4 | number of objects | gauge | -| `consul.state.config_entries` | Measures the current number of configuration entries registered with Consul labeled by Kind (e.g. service-defaults, proxy-defaults, etc). See [Configuration Entries](/docs/connect/config-entries) for more information. Added in v1.10.4 | number of objects | gauge | -| `consul.members.clients` | Measures the current number of client agents registered with Consul. It is only emitted by Consul servers. Added in v1.9.6. | number of clients | gauge | -| `consul.members.servers` | Measures the current number of server agents registered with Consul. It is only emitted by Consul servers. Added in v1.9.6. | number of servers | gauge | -| `consul.dns.stale_queries` | Increments when an agent serves a query within the allowed stale threshold. | queries | counter | -| `consul.dns.ptr_query.` | Measures the time spent handling a reverse DNS query for the given node. | ms | timer | -| `consul.dns.domain_query.` | Measures the time spent handling a domain query for the given node. | ms | timer | -| `consul.system.licenseExpiration` | This measures the number of hours remaining on the agents license. | hours | gauge | -| `consul.version` | Represents the Consul version. +| Metric | Description | Unit | Type | +|--------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------|---------| +| `consul.acl.blocked.{check,service}.deregistration` | Increments whenever a deregistration fails for an entity (check or service) is blocked by an ACL. | requests | counter | +| `consul.acl.blocked.{check,node,service}.registration` | Increments whenever a registration fails for an entity (check, node or service) is blocked by an ACL. | requests | counter | +| `consul.api.http` | This samples how long it takes to service the given HTTP request for the given verb and path. Includes labels for `path` and `method`. `path` does not include details like service or key names, for these an underscore will be present as a placeholder (eg. path=`v1.kv._`) | ms | timer | +| `consul.client.rpc` | Increments whenever a Consul agent in client mode makes an RPC request to a Consul server. This gives a measure of how much a given agent is loading the Consul servers. Currently, this is only generated by agents in client mode, not Consul servers. | requests | counter | +| `consul.client.rpc.exceeded` | Increments whenever a Consul agent in client mode makes an RPC request to a Consul server gets rate limited by that agent's [`limits`](/docs/agent/config/config-files#limits) configuration. This gives an indication that there's an abusive application making too many requests on the agent, or that the rate limit needs to be increased. Currently, this only applies to agents in client mode, not Consul servers. | rejected requests | counter | +| `consul.client.rpc.failed` | Increments whenever a Consul agent in client mode makes an RPC request to a Consul server and fails. | requests | counter | +| `consul.client.api.catalog_register.` | Increments whenever a Consul agent receives a catalog register request. | requests | counter | +| `consul.client.api.success.catalog_register.` | Increments whenever a Consul agent successfully responds to a catalog register request. | requests | counter | +| `consul.client.rpc.error.catalog_register.` | Increments whenever a Consul agent receives an RPC error for a catalog register request. | errors | counter | +| `consul.client.api.catalog_deregister.` | Increments whenever a Consul agent receives a catalog deregister request. | requests | counter | +| `consul.client.api.success.catalog_deregister.` | Increments whenever a Consul agent successfully responds to a catalog deregister request. | requests | counter | +| `consul.client.rpc.error.catalog_deregister.` | Increments whenever a Consul agent receives an RPC error for a catalog deregister request. | errors | counter | +| `consul.client.api.catalog_datacenters.` | Increments whenever a Consul agent receives a request to list datacenters in the catalog. | requests | counter | +| `consul.client.api.success.catalog_datacenters.` | Increments whenever a Consul agent successfully responds to a request to list datacenters. | requests | counter | +| `consul.client.rpc.error.catalog_datacenters.` | Increments whenever a Consul agent receives an RPC error for a request to list datacenters. | errors | counter | +| `consul.client.api.catalog_nodes.` | Increments whenever a Consul agent receives a request to list nodes from the catalog. | requests | counter | +| `consul.client.api.success.catalog_nodes.` | Increments whenever a Consul agent successfully responds to a request to list nodes. | requests | counter | +| `consul.client.rpc.error.catalog_nodes.` | Increments whenever a Consul agent receives an RPC error for a request to list nodes. | errors | counter | +| `consul.client.api.catalog_services.` | Increments whenever a Consul agent receives a request to list services from the catalog. | requests | counter | +| `consul.client.api.success.catalog_services.` | Increments whenever a Consul agent successfully responds to a request to list services. | requests | counter | +| `consul.client.rpc.error.catalog_services.` | Increments whenever a Consul agent receives an RPC error for a request to list services. | errors | counter | +| `consul.client.api.catalog_service_nodes.` | Increments whenever a Consul agent receives a request to list nodes offering a service. | requests | counter | +| `consul.client.api.success.catalog_service_nodes.` | Increments whenever a Consul agent successfully responds to a request to list nodes offering a service. | requests | counter | +| `consul.client.api.error.catalog_service_nodes.` | Increments whenever a Consul agent receives an RPC error for request to list nodes offering a service. | requests | counter | +| `consul.client.rpc.error.catalog_service_nodes.` | Increments whenever a Consul agent receives an RPC error for a request to list nodes offering a service.   | errors | counter | +| `consul.client.api.catalog_node_services.` | Increments whenever a Consul agent receives a request to list services registered in a node.   | requests | counter | +| `consul.client.api.success.catalog_node_services.` | Increments whenever a Consul agent successfully responds to a request to list services in a node.   | requests | counter | +| `consul.client.rpc.error.catalog_node_services.` | Increments whenever a Consul agent receives an RPC error for a request to list services in a node.   | errors | counter | +| `consul.client.api.catalog_node_service_list` | Increments whenever a Consul agent receives a request to list a node's registered services. | requests | counter | +| `consul.client.rpc.error.catalog_node_service_list` | Increments whenever a Consul agent receives an RPC error for request to list a node's registered services. | errors | counter | +| `consul.client.api.success.catalog_node_service_list` | Increments whenever a Consul agent successfully responds to a request to list a node's registered services. | requests | counter | +| `consul.client.api.catalog_gateway_services.` | Increments whenever a Consul agent receives a request to list services associated with a gateway. | requests | counter | +| `consul.client.api.success.catalog_gateway_services.` | Increments whenever a Consul agent successfully responds to a request to list services associated with a gateway. | requests | counter | +| `consul.client.rpc.error.catalog_gateway_services.` | Increments whenever a Consul agent receives an RPC error for a request to list services associated with a gateway. | errors | counter | +| `consul.runtime.num_goroutines` | Tracks the number of running goroutines and is a general load pressure indicator. This may burst from time to time but should return to a steady state value. | number of goroutines | gauge | +| `consul.runtime.alloc_bytes` | Measures the number of bytes allocated by the Consul process. This may burst from time to time but should return to a steady state value. | bytes | gauge | +| `consul.runtime.heap_objects` | Measures the number of objects allocated on the heap and is a general memory pressure indicator. This may burst from time to time but should return to a steady state value. | number of objects | gauge | +| `consul.state.nodes` | Measures the current number of nodes registered with Consul. It is only emitted by Consul servers. Added in v1.9.0. | number of objects | gauge | +| `consul.state.peerings` | Measures the current number of peerings registered with Consul. It is only emitted by Consul servers. Added in v1.13.0. | number of objects | gauge | +| `consul.state.services` | Measures the current number of unique services registered with Consul, based on service name. It is only emitted by Consul servers. Added in v1.9.0. | number of objects | gauge | +| `consul.state.service_instances` | Measures the current number of unique service instances registered with Consul. It is only emitted by Consul servers. Added in v1.9.0. | number of objects | gauge | +| `consul.state.kv_entries` | Measures the current number of entries in the Consul KV store. It is only emitted by Consul servers. Added in v1.10.3. | number of objects | gauge | +| `consul.state.connect_instances` | Measures the current number of unique connect service instances registered with Consul labeled by Kind (e.g. connect-proxy, connect-native, etc). Added in v1.10.4 | number of objects | gauge | +| `consul.state.config_entries` | Measures the current number of configuration entries registered with Consul labeled by Kind (e.g. service-defaults, proxy-defaults, etc). See [Configuration Entries](/docs/connect/config-entries) for more information. Added in v1.10.4 | number of objects | gauge | +| `consul.members.clients` | Measures the current number of client agents registered with Consul. It is only emitted by Consul servers. Added in v1.9.6. | number of clients | gauge | +| `consul.members.servers` | Measures the current number of server agents registered with Consul. It is only emitted by Consul servers. Added in v1.9.6. | number of servers | gauge | +| `consul.dns.stale_queries` | Increments when an agent serves a query within the allowed stale threshold. | queries | counter | +| `consul.dns.ptr_query.` | Measures the time spent handling a reverse DNS query for the given node. | ms | timer | +| `consul.dns.domain_query.` | Measures the time spent handling a domain query for the given node. | ms | timer | +| `consul.system.licenseExpiration` | This measures the number of hours remaining on the agents license. | hours | gauge | +| `consul.version` | Represents the Consul version. | agents | gauge | ## Server Health @@ -691,14 +691,14 @@ agent. The table below describes the additional metrics exported by the proxy. **Requirements:** - Consul 1.13.0+ -[Cluster peering](/docs/connect/cluster-peering) refers to enabling communication between Consul clusters through a peer connection, as opposed to a federated connection. Consul collects metrics that describe the number of services exported to a peered cluster. Peering metrics are only emitted by the leader server. +[Cluster peering](/docs/connect/cluster-peering) refers to enabling communication between Consul clusters through a peer connection, as opposed to a federated connection. Consul collects metrics that describe the number of services exported to a peered cluster. Peering metrics are only emitted by the leader server. | Metric | Description | Unit | Type | | ------------------------------------- | ----------------------------------------------------------------------| ------ | ------- | | `consul.peering.exported_services` | Counts the number of services exported to a peer cluster. | count | gauge | ### Labels -Consul attaches the following labels to metric values. +Consul attaches the following labels to metric values. | Label Name | Description | Possible values | | ------------------------------------- | ---------------------------------------------------------------------- | ------------------------------------------ | | `peer_name` | The name of the peering on the reporting cluster or leader. | Any defined peer name in the cluster | From 4d40d02c73d8daeff833f451e9e1714e6d0a7cde Mon Sep 17 00:00:00 2001 From: Thomas Eckert Date: Thu, 25 Aug 2022 12:45:57 -0400 Subject: [PATCH 04/31] Remove warning about 1.9 --- website/content/docs/k8s/connect/terminating-gateways.mdx | 2 -- 1 file changed, 2 deletions(-) diff --git a/website/content/docs/k8s/connect/terminating-gateways.mdx b/website/content/docs/k8s/connect/terminating-gateways.mdx index e82bd773fb..9ff16b5c64 100644 --- a/website/content/docs/k8s/connect/terminating-gateways.mdx +++ b/website/content/docs/k8s/connect/terminating-gateways.mdx @@ -6,8 +6,6 @@ description: Configuring Terminating Gateways on Kubernetes # Terminating Gateways on Kubernetes --> 1.9.0+: This feature is available in Consul versions 1.9.0 and higher - ~> This topic requires familiarity with [Terminating Gateways](/docs/connect/gateways/terminating-gateway). Adding a terminating gateway is a multi-step process: From ac129339f8ec9e3081b915c9e4cf4a575f23dad2 Mon Sep 17 00:00:00 2001 From: Thomas Eckert Date: Thu, 25 Aug 2022 12:49:54 -0400 Subject: [PATCH 05/31] Instruct users to use the CLI --- website/content/docs/k8s/connect/terminating-gateways.mdx | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/website/content/docs/k8s/connect/terminating-gateways.mdx b/website/content/docs/k8s/connect/terminating-gateways.mdx index 9ff16b5c64..c5607d4380 100644 --- a/website/content/docs/k8s/connect/terminating-gateways.mdx +++ b/website/content/docs/k8s/connect/terminating-gateways.mdx @@ -36,9 +36,11 @@ terminatingGateways: ## Deploying the Helm chart -Ensure you have the latest consul-helm chart and install Consul via helm using the following -[guide](/docs/k8s/installation/install#installing-consul) while being sure to provide the yaml configuration -as previously discussed. +The Helm chart may be deployed using the [Consul on Kubernetes CLI](/docs/k8s/k8s-cli). + +```shell-session +consul-k8s install -f config.yaml +``` ## Accessing the Consul agent From 884dda25c26caced01ae4a00ca89f5a277acf26e Mon Sep 17 00:00:00 2001 From: Thomas Eckert Date: Thu, 25 Aug 2022 13:02:55 -0400 Subject: [PATCH 06/31] Use tabs for with and without TLS --- .../docs/k8s/connect/terminating-gateways.mdx | 32 +++++++++++-------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/website/content/docs/k8s/connect/terminating-gateways.mdx b/website/content/docs/k8s/connect/terminating-gateways.mdx index c5607d4380..9fb2b149c4 100644 --- a/website/content/docs/k8s/connect/terminating-gateways.mdx +++ b/website/content/docs/k8s/connect/terminating-gateways.mdx @@ -6,8 +6,6 @@ description: Configuring Terminating Gateways on Kubernetes # Terminating Gateways on Kubernetes -~> This topic requires familiarity with [Terminating Gateways](/docs/connect/gateways/terminating-gateway). - Adding a terminating gateway is a multi-step process: - Update the Helm chart with terminating gateway config options @@ -15,6 +13,12 @@ Adding a terminating gateway is a multi-step process: - Access the Consul agent - Register external services with Consul +## Requirements + +- [Consul]() +- [Consul on Kubernetes CLI]() +- Familiarity with [Terminating Gateways](/docs/connect/gateways/terminating-gateway) + ## Update the helm chart with terminating gateway config options Minimum required Helm options: @@ -39,36 +43,38 @@ terminatingGateways: The Helm chart may be deployed using the [Consul on Kubernetes CLI](/docs/k8s/k8s-cli). ```shell-session -consul-k8s install -f config.yaml +$ consul-k8s install -f config.yaml ``` ## Accessing the Consul agent -You can access the Consul server directly from your host via `kubectl port-forward`. This is helpful for interacting with your Consul UI locally as well as to validate connectivity of the application. +You can access the Consul server directly from your host via `kubectl port-forward`. This is helpful for interacting with your Consul UI locally as well as for validating the connectivity of the application. + + + ```shell-session $ kubectl port-forward consul-server-0 8500 & ``` +```shell-session +$ export CONSUL_HTTP_ADDR=http://localhost:8500 +``` + + + If TLS is enabled use port 8501: ```shell-session $ kubectl port-forward consul-server-0 8501 & ``` --> Be sure the latest consul binary is installed locally on your host. -[https://releases.hashicorp.com/consul/](https://releases.hashicorp.com/consul/) - -```shell-session -$ export CONSUL_HTTP_ADDR=http://localhost:8500 -``` - -If TLS is enabled set: - ```shell-session $ export CONSUL_HTTP_ADDR=https://localhost:8501 $ export CONSUL_HTTP_SSL_VERIFY=false ``` + + If ACLs are enabled also set: From 65dce3476f5e5f3d1a6e55b8e2448c8b525071e8 Mon Sep 17 00:00:00 2001 From: Thomas Eckert Date: Thu, 25 Aug 2022 13:27:43 -0400 Subject: [PATCH 07/31] Clean up copy for registration --- .../docs/k8s/connect/terminating-gateways.mdx | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/website/content/docs/k8s/connect/terminating-gateways.mdx b/website/content/docs/k8s/connect/terminating-gateways.mdx index 9fb2b149c4..188da110f0 100644 --- a/website/content/docs/k8s/connect/terminating-gateways.mdx +++ b/website/content/docs/k8s/connect/terminating-gateways.mdx @@ -94,14 +94,15 @@ Registering the external services with Consul is a multi-step process: ### Register external services with Consul -There are two ways to register an external service with Consul: -1. If [`TransparentProxy`](/docs/connect/transparent-proxy) is enabled, the preferred method is to declare external endpoints in the [`destination`](/docs/connect/config-entries/service-defaults#terminating-gateway-destination) field of `ServiceDefaults`. -1. You can add the service as a node in the Consul catalog. +You may register an external service with Consul using `ServiceDefaults` if +[`TransparentProxy`](/docs/connect/transparent-proxy) is enabled. Otherwise, +you may register the service as a node in the Consul catalog. -#### Register an external service as a destination + + -The [`destination`](/docs/connect/config-entries/service-defaults#terminating-gateway-destination) field of the `ServiceDefaults` Custom Resource Definition (CRD) allows clients to dial the external service directly. It is valid only in [`TransparentProxy`](/docs/connect/transparent-proxy)) mode. -The following table describes traffic behaviors when using `destination`s to route traffic through a terminating gateway: +The [`destination`](/docs/connect/config-entries/service-defaults#terminating-gateway-destination) field of the `ServiceDefaults` Custom Resource Definition (CRD) allows clients to dial an external service directly. For this method to work, [`TransparentProxy`](/docs/connect/transparent-proxy) must be enabled. +The following table describes traffic behaviors when using the `destination` field to route traffic through a terminating gateway: | External Services Layer | Client dials | Client uses TLS | Allowed | Notes | |---|---|---|---|---| @@ -145,8 +146,8 @@ $ kubectl apply --filename serviceDefaults.yaml ``` All other terminating gateway operations can use the name of the `ServiceDefaults` in place of a typical Consul service name. - -#### Register an external service as a Catalog Node + + -> **Note:** Normal Consul services are registered with the Consul client on the node that they're running on. Since this is an external service, there is no Consul node @@ -197,6 +198,10 @@ If ACLs and TLS are enabled : $ curl --request PUT --header "X-Consul-Token: $CONSUL_HTTP_TOKEN" --data @external.json --insecure $CONSUL_HTTP_ADDR/v1/catalog/register true ``` + + + + ### Update terminating gateway ACL role if ACLs are enabled From a2a7b56292b0f64d3e94d3fbc72562420c4589b4 Mon Sep 17 00:00:00 2001 From: Thomas Eckert Date: Thu, 25 Aug 2022 13:37:52 -0400 Subject: [PATCH 08/31] Format traffic behaviors table --- .../docs/k8s/connect/terminating-gateways.mdx | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/website/content/docs/k8s/connect/terminating-gateways.mdx b/website/content/docs/k8s/connect/terminating-gateways.mdx index 188da110f0..c685c07c20 100644 --- a/website/content/docs/k8s/connect/terminating-gateways.mdx +++ b/website/content/docs/k8s/connect/terminating-gateways.mdx @@ -104,16 +104,16 @@ you may register the service as a node in the Consul catalog. The [`destination`](/docs/connect/config-entries/service-defaults#terminating-gateway-destination) field of the `ServiceDefaults` Custom Resource Definition (CRD) allows clients to dial an external service directly. For this method to work, [`TransparentProxy`](/docs/connect/transparent-proxy) must be enabled. The following table describes traffic behaviors when using the `destination` field to route traffic through a terminating gateway: -| External Services Layer | Client dials | Client uses TLS | Allowed | Notes | -|---|---|---|---|---| -| L4 | Hostname | Yes | Allowed | `CAFiles` are not allowed because traffic is already end-to-end encrypted by the client. | -| L4 | IP | Yes | Allowed | `CAFiles` are not allowed because traffic is already end-to-end encrypted by the client. | -| L4 | Hostname | No | Not allowed | The sidecar is not protocol aware and can not identify traffic going to the external service. | -| L4 | IP | No | Allowed | There are no limitations on dialing IPs without TLS. | -| L7 | Hostname | Yes | Not allowed | Because traffic is already encrypted before the sidecar, it cannot route as L7 traffic. | -| L7 | IP | Yes | Not allowed | Because traffic is already encrypted before the sidecar, it cannot route as L7 traffic. | -| L7 | Hostname | No | Allowed | A `Host` or `:authority` header is required. | -| L7 | IP | No | Allowed | There are no limitations on dialing IPs without TLS. | +| External Services Layer | Client dials | Client uses TLS | Allowed | Notes | +| ----------------------- | ------------ | --------------- | ----------- | --------------------------------------------------------------------------------------------- | +| L4 | Hostname | Yes | Allowed | `CAFiles` are not allowed because traffic is already end-to-end encrypted by the client. | +| L4 | IP | Yes | Allowed | `CAFiles` are not allowed because traffic is already end-to-end encrypted by the client. | +| L4 | Hostname | No | Not allowed | The sidecar is not protocol aware and can not identify traffic going to the external service. | +| L4 | IP | No | Allowed | There are no limitations on dialing IPs without TLS. | +| L7 | Hostname | Yes | Not allowed | Because traffic is already encrypted before the sidecar, it cannot route as L7 traffic. | +| L7 | IP | Yes | Not allowed | Because traffic is already encrypted before the sidecar, it cannot route as L7 traffic. | +| L7 | Hostname | No | Allowed | A `Host` or `:authority` header is required. | +| L7 | IP | No | Allowed | There are no limitations on dialing IPs without TLS. | You can provide a `caFile` to secure traffic between unencrypted clients that connect to external services through the terminating gateway. Refer to [Create the configuration entry for the terminating gateway](#create-the-configuration-entry-for-the-terminating-gateway) for details. From e990b03d5cd0af5136703070694a08ce6d8faa42 Mon Sep 17 00:00:00 2001 From: Thomas Eckert Date: Thu, 25 Aug 2022 13:56:13 -0400 Subject: [PATCH 09/31] Normalize table with nobrs --- .../docs/k8s/connect/terminating-gateways.mdx | 31 ++++++++++--------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/website/content/docs/k8s/connect/terminating-gateways.mdx b/website/content/docs/k8s/connect/terminating-gateways.mdx index c685c07c20..80654698a3 100644 --- a/website/content/docs/k8s/connect/terminating-gateways.mdx +++ b/website/content/docs/k8s/connect/terminating-gateways.mdx @@ -104,25 +104,25 @@ you may register the service as a node in the Consul catalog. The [`destination`](/docs/connect/config-entries/service-defaults#terminating-gateway-destination) field of the `ServiceDefaults` Custom Resource Definition (CRD) allows clients to dial an external service directly. For this method to work, [`TransparentProxy`](/docs/connect/transparent-proxy) must be enabled. The following table describes traffic behaviors when using the `destination` field to route traffic through a terminating gateway: -| External Services Layer | Client dials | Client uses TLS | Allowed | Notes | -| ----------------------- | ------------ | --------------- | ----------- | --------------------------------------------------------------------------------------------- | -| L4 | Hostname | Yes | Allowed | `CAFiles` are not allowed because traffic is already end-to-end encrypted by the client. | -| L4 | IP | Yes | Allowed | `CAFiles` are not allowed because traffic is already end-to-end encrypted by the client. | -| L4 | Hostname | No | Not allowed | The sidecar is not protocol aware and can not identify traffic going to the external service. | -| L4 | IP | No | Allowed | There are no limitations on dialing IPs without TLS. | -| L7 | Hostname | Yes | Not allowed | Because traffic is already encrypted before the sidecar, it cannot route as L7 traffic. | -| L7 | IP | Yes | Not allowed | Because traffic is already encrypted before the sidecar, it cannot route as L7 traffic. | -| L7 | Hostname | No | Allowed | A `Host` or `:authority` header is required. | -| L7 | IP | No | Allowed | There are no limitations on dialing IPs without TLS. | +| External Services Layer | Client dials | Client uses TLS | Allowed | Notes | +|--------------------------------------|---------------------------|------------------------------|--------------------------|-----------------------------------------------------------------------------------------------| +| L4 | Hostname | Yes | Allowed | `CAFiles` are not allowed because traffic is already end-to-end encrypted by the client. | +| L4 | IP | Yes | Allowed | `CAFiles` are not allowed because traffic is already end-to-end encrypted by the client. | +| L4 | Hostname | No | Not allowed | The sidecar is not protocol aware and can not identify traffic going to the external service. | +| L4 | IP | No | Allowed | There are no limitations on dialing IPs without TLS. | +| L7 | Hostname | Yes | Not allowed | Because traffic is already encrypted before the sidecar, it cannot route as L7 traffic. | +| L7 | IP | Yes | Not allowed | Because traffic is already encrypted before the sidecar, it cannot route as L7 traffic. | +| L7 | Hostname | No | Allowed | A `Host` or `:authority` header is required. | +| L7 | IP | No | Allowed | There are no limitations on dialing IPs without TLS. | You can provide a `caFile` to secure traffic between unencrypted clients that connect to external services through the terminating gateway. Refer to [Create the configuration entry for the terminating gateway](#create-the-configuration-entry-for-the-terminating-gateway) for details. -Also note that regardless of the `protocol` specified in the `ServiceDefaults`, [L7 intentions](/docs/connect/config-entries/service-intentions#permissions) are not currently supported with `ServiceDefaults` destinations. +-> **Note:** Regardless of the `protocol` specified in the `ServiceDefaults`, [L7 intentions](/docs/connect/config-entries/service-intentions#permissions) are not currently supported with `ServiceDefaults` destinations. Create a `ServiceDefaults` custom resource for the external service: - + ```yaml apiVersion: consul.hashicorp.com/v1alpha1 @@ -142,14 +142,15 @@ Create a `ServiceDefaults` custom resource for the external service: Apply the `ServiceDefaults` resource with `kubectl apply`: ```shell-session -$ kubectl apply --filename serviceDefaults.yaml +$ kubectl apply --filename service-defaults.yaml ``` -All other terminating gateway operations can use the name of the `ServiceDefaults` in place of a typical Consul service name. +All other terminating gateway operations can use the name of the `ServiceDefaults` component, in this case "example-https", as a Consul service name. + --> **Note:** Normal Consul services are registered with the Consul client on the node that +Normally, Consul services are registered with the Consul client on the node that they're running on. Since this is an external service, there is no Consul node to register it onto. Instead, we will make up a node name and register the service to that node. From 6d9872388bcb4dc7a9b3f8f6cde67a70731f0037 Mon Sep 17 00:00:00 2001 From: Thomas Eckert Date: Thu, 25 Aug 2022 14:03:43 -0400 Subject: [PATCH 10/31] Clean up copy in ACL role update --- .../content/docs/k8s/connect/terminating-gateways.mdx | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/website/content/docs/k8s/connect/terminating-gateways.mdx b/website/content/docs/k8s/connect/terminating-gateways.mdx index 80654698a3..9ed9865a3a 100644 --- a/website/content/docs/k8s/connect/terminating-gateways.mdx +++ b/website/content/docs/k8s/connect/terminating-gateways.mdx @@ -202,15 +202,12 @@ true - - ### Update terminating gateway ACL role if ACLs are enabled If ACLs are enabled, update the terminating gateway acl role to have `service: write` permissions on all of the services -being represented by the gateway: +being represented by the gateway. -- Create a new policy that includes these permissions -- Update the existing role to include the new policy +Create a new policy that includes the write permission for the service you created. @@ -242,7 +239,7 @@ consul acl role list | grep -B 6 -- "- RELEASE_NAME-terminating-gateway-policy" ID: ``` -Update the terminating gateway acl token with the new policy +Update the terminating gateway ACL token with the new policy. ```shell-session $ consul acl role update -id -policy-name example-https-write-policy From 77c9995a8e1634c6bcb72e6f65e000f2c6252e5b Mon Sep 17 00:00:00 2001 From: Thomas Eckert Date: Thu, 25 Aug 2022 14:04:33 -0400 Subject: [PATCH 11/31] Lil' more cleanup --- website/content/docs/k8s/connect/terminating-gateways.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/content/docs/k8s/connect/terminating-gateways.mdx b/website/content/docs/k8s/connect/terminating-gateways.mdx index 9ed9865a3a..01937f6aa4 100644 --- a/website/content/docs/k8s/connect/terminating-gateways.mdx +++ b/website/content/docs/k8s/connect/terminating-gateways.mdx @@ -231,7 +231,7 @@ service "example-https" { } ``` -Now fetch the ID of the terminating gateway token +Fetch the ID of the terminating gateway token. ```shell-session consul acl role list | grep -B 6 -- "- RELEASE_NAME-terminating-gateway-policy" | grep ID From ed4a430b3ec17d7d70b671fb4e01deab63fbc8a4 Mon Sep 17 00:00:00 2001 From: Thomas Eckert Date: Thu, 25 Aug 2022 14:40:18 -0400 Subject: [PATCH 12/31] Use tabs for destinations --- .../docs/k8s/connect/terminating-gateways.mdx | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/website/content/docs/k8s/connect/terminating-gateways.mdx b/website/content/docs/k8s/connect/terminating-gateways.mdx index 01937f6aa4..2119e54e4f 100644 --- a/website/content/docs/k8s/connect/terminating-gateways.mdx +++ b/website/content/docs/k8s/connect/terminating-gateways.mdx @@ -278,8 +278,6 @@ Configure the [`caFile`](https://www.consul.io/docs/connect/config-entries/termi - Consul Helm chart 0.43 or older - An Envoy image with an alpine base image -For `ServiceDefaults` destinations, refer to [Register an external service as a destination](#register-an-external-service-as-a-destination). - Apply the `TerminatingGateway` resource with `kubectl apply`: ```shell-session @@ -315,7 +313,7 @@ $ kubectl apply --filename service-intentions.yaml ### Define the external services as upstreams for services in the mesh -Finally define and deploy the external services as upstreams for the internal mesh services that wish to talk to them. +As a final step, you may define and deploy the external services as upstreams for the internal mesh services that wish to talk to them. An example deployment is provided which will serve as a static client for the terminating gateway service. @@ -364,33 +362,35 @@ spec: -Run the service via `kubectl apply`: +Deploy the service with `kubectl apply`. ```shell-session $ kubectl apply --filename static-client.yaml ``` -Wait for the service to be ready: +Wait for the service to be ready. ```shell-session $ kubectl rollout status deploy static-client --watch deployment "static-client" successfully rolled out ``` -You can verify connectivity of the static-client and terminating gateway via a curl command: +You can verify connectivity of the static-client and terminating gateway via a curl command. - - -```shell-session -$ kubectl exec deploy/static-client -- curl -vvvs --header "Host: example-https.com" http://localhost:1234/ -``` - - - - + + ```shell-session $ kubectl exec deploy/static-client -- curl -vvvs https://example.com/ ``` - + + + +```shell-session +$ kubectl exec deploy/static-client -- curl -vvvs --header "Host: example-https.com" http://localhost:1234/ +``` + + + + From 5064fbc2545706dec7c4a8b87bf7a7f9ba05e354 Mon Sep 17 00:00:00 2001 From: Thomas Eckert Date: Thu, 25 Aug 2022 14:44:33 -0400 Subject: [PATCH 13/31] Add links to requirements --- website/content/docs/k8s/connect/terminating-gateways.mdx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/website/content/docs/k8s/connect/terminating-gateways.mdx b/website/content/docs/k8s/connect/terminating-gateways.mdx index 2119e54e4f..ed1b11e8c7 100644 --- a/website/content/docs/k8s/connect/terminating-gateways.mdx +++ b/website/content/docs/k8s/connect/terminating-gateways.mdx @@ -15,8 +15,8 @@ Adding a terminating gateway is a multi-step process: ## Requirements -- [Consul]() -- [Consul on Kubernetes CLI]() +- [Consul](https://www.consul.io/docs/install#install-consul) +- [Consul on Kubernetes CLI](/docs/k8s/k8s-cli) - Familiarity with [Terminating Gateways](/docs/connect/gateways/terminating-gateway) ## Update the helm chart with terminating gateway config options From 70a1cbd8ea3a1efef2373cb15c4657d590c31615 Mon Sep 17 00:00:00 2001 From: Thomas Eckert Date: Thu, 25 Aug 2022 14:44:45 -0400 Subject: [PATCH 14/31] Capitalize Helm --- website/content/docs/k8s/connect/terminating-gateways.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/content/docs/k8s/connect/terminating-gateways.mdx b/website/content/docs/k8s/connect/terminating-gateways.mdx index ed1b11e8c7..06316f5f51 100644 --- a/website/content/docs/k8s/connect/terminating-gateways.mdx +++ b/website/content/docs/k8s/connect/terminating-gateways.mdx @@ -19,7 +19,7 @@ Adding a terminating gateway is a multi-step process: - [Consul on Kubernetes CLI](/docs/k8s/k8s-cli) - Familiarity with [Terminating Gateways](/docs/connect/gateways/terminating-gateway) -## Update the helm chart with terminating gateway config options +## Update the Helm chart with terminating gateway config options Minimum required Helm options: From e2fe8b8d65638877ac4cc64c1957b69dac82ed7d Mon Sep 17 00:00:00 2001 From: "Chris S. Kim" Date: Fri, 26 Aug 2022 11:14:02 -0400 Subject: [PATCH 15/31] Fix tests for enterprise --- agent/consul/state/catalog_test.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/agent/consul/state/catalog_test.go b/agent/consul/state/catalog_test.go index ca2bded03b..e2310dbb7c 100644 --- a/agent/consul/state/catalog_test.go +++ b/agent/consul/state/catalog_test.go @@ -2104,10 +2104,13 @@ func TestStateStore_Services(t *testing.T) { Address: "1.1.1.1", Port: 1111, } + ns1.EnterpriseMeta.Normalize() if err := s.EnsureService(2, "node1", ns1); err != nil { t.Fatalf("err: %s", err) } ns1Dogs := testRegisterService(t, s, 3, "node1", "dogs") + ns1Dogs.EnterpriseMeta.Normalize() + testRegisterNode(t, s, 4, "node2") ns2 := &structs.NodeService{ ID: "service3", @@ -2116,6 +2119,7 @@ func TestStateStore_Services(t *testing.T) { Address: "1.1.1.1", Port: 1111, } + ns2.EnterpriseMeta.Normalize() if err := s.EnsureService(5, "node2", ns2); err != nil { t.Fatalf("err: %s", err) } @@ -2139,7 +2143,7 @@ func TestStateStore_Services(t *testing.T) { ns1.ToServiceNode("node1"), ns2.ToServiceNode("node2"), } - assertDeepEqual(t, services, expected, cmpopts.IgnoreFields(structs.ServiceNode{}, "RaftIndex")) + assertDeepEqual(t, expected, services, cmpopts.IgnoreFields(structs.ServiceNode{}, "RaftIndex")) // Deleting a node with a service should fire the watch. if err := s.DeleteNode(6, "node1", nil, ""); err != nil { @@ -2178,6 +2182,7 @@ func TestStateStore_ServicesByNodeMeta(t *testing.T) { Address: "1.1.1.1", Port: 1111, } + ns1.EnterpriseMeta.Normalize() if err := s.EnsureService(2, "node0", ns1); err != nil { t.Fatalf("err: %s", err) } @@ -2188,6 +2193,7 @@ func TestStateStore_ServicesByNodeMeta(t *testing.T) { Address: "1.1.1.1", Port: 1111, } + ns2.EnterpriseMeta.Normalize() if err := s.EnsureService(3, "node1", ns2); err != nil { t.Fatalf("err: %s", err) } From eb0c5bb9a17ea2a6643eaea240fc3526bc8f129c Mon Sep 17 00:00:00 2001 From: smamindla57 <106655516+smamindla57@users.noreply.github.com> Date: Fri, 26 Aug 2022 21:43:46 +0530 Subject: [PATCH 16/31] Updated consul monitoring with Newrelic APM (#14360) * added newrelic consul quickstart link * adding HCP Consul Co-authored-by: David Yu --- website/content/docs/integrate/partnerships.mdx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/website/content/docs/integrate/partnerships.mdx b/website/content/docs/integrate/partnerships.mdx index e7c5bb2228..057e3464cf 100644 --- a/website/content/docs/integrate/partnerships.mdx +++ b/website/content/docs/integrate/partnerships.mdx @@ -99,12 +99,13 @@ Here are links to resources, documentation, examples and best practices to guide - [Consul Telemetry Documentation](/docs/agent/telemetry) - [Monitoring Consul with Datadog APM](https://www.datadoghq.com/blog/consul-datadog/) - [Monitoring Consul with Dynatrace APM](https://www.dynatrace.com/news/blog/automatic-intelligent-observability-into-your-hashicorp-consul-service-mesh/) +- [Monitoring Consul with New Relic APM](https://newrelic.com/instant-observability/consul/b65825cc-faee-47b5-8d7c-6d60d6ab3c59) +- [Monitoring HCP Consul with New Relic APM](https://newrelic.com/instant-observability/hcp-consul/bc99ad15-7aba-450e-8236-6ea667d50cae) **Logging** - [Monitor Consul with Logz.io](https://www.hashicorp.com/integrations/logz-io/consul) - [Monitor Consul with Splunk SignalFx](https://www.hashicorp.com/integrations/splunksignalfx/consul) -- [Consul Datacenter Monitoring with New Relic](https://www.hashicorp.com/integrations/new-relic/consul) #### Platform: From 650e48624df0d5f7dcc9dabf088709f87c2390a9 Mon Sep 17 00:00:00 2001 From: freddygv Date: Fri, 26 Aug 2022 10:52:47 -0600 Subject: [PATCH 17/31] Allow terminated peerings to be deleted Peerings are terminated when a peer decides to delete the peering from their end. Deleting a peering sends a termination message to the peer and triggers them to mark the peering as terminated but does NOT delete the peering itself. This is to prevent peerings from disappearing from both sides just because one side deleted them. Previously the Delete endpoint was skipping the deletion if the peering was not marked as active. However, terminated peerings are also inactive. This PR makes some updates so that peerings marked as terminated can be deleted by users. --- agent/consul/state/peering.go | 23 ++- agent/consul/state/peering_test.go | 228 +++++++++++++++++++++++++---- agent/rpc/peering/service.go | 3 +- agent/rpc/peering/service_test.go | 64 ++++---- 4 files changed, 258 insertions(+), 60 deletions(-) diff --git a/agent/consul/state/peering.go b/agent/consul/state/peering.go index 287e822919..9457dd811a 100644 --- a/agent/consul/state/peering.go +++ b/agent/consul/state/peering.go @@ -549,8 +549,25 @@ func (s *Store) PeeringWrite(idx uint64, req *pbpeering.PeeringWriteRequest) err if req.Peering.ID != existing.ID { return fmt.Errorf("A peering already exists with the name %q and a different ID %q", req.Peering.Name, existing.ID) } + + // Nothing to do if our peer wants to terminate the peering but the peering is already marked for deletion. + if existing.State == pbpeering.PeeringState_DELETING && req.Peering.State == pbpeering.PeeringState_TERMINATED { + return nil + } + + // No-op deletion + if existing.State == pbpeering.PeeringState_DELETING && req.Peering.State == pbpeering.PeeringState_DELETING { + return nil + } + + // No-op termination + if existing.State == pbpeering.PeeringState_TERMINATED && req.Peering.State == pbpeering.PeeringState_TERMINATED { + return nil + } + // Prevent modifications to Peering marked for deletion. - if !existing.IsActive() { + // This blocks generating new peering tokens or re-establishing the peering until the peering is done deleting. + if existing.State == pbpeering.PeeringState_DELETING { return fmt.Errorf("cannot write to peering that is marked for deletion") } @@ -582,8 +599,8 @@ func (s *Store) PeeringWrite(idx uint64, req *pbpeering.PeeringWriteRequest) err req.Peering.ModifyIndex = idx } - // Ensure associated secrets are cleaned up when a peering is marked for deletion. - if req.Peering.State == pbpeering.PeeringState_DELETING { + // Ensure associated secrets are cleaned up when a peering is marked for deletion or terminated. + if !req.Peering.IsActive() { if err := peeringSecretsDeleteTxn(tx, req.Peering.ID, req.Peering.ShouldDial()); err != nil { return fmt.Errorf("failed to delete peering secrets: %w", err) } diff --git a/agent/consul/state/peering_test.go b/agent/consul/state/peering_test.go index bfce75295c..1dc2446fe1 100644 --- a/agent/consul/state/peering_test.go +++ b/agent/consul/state/peering_test.go @@ -1112,16 +1112,22 @@ func TestStore_PeeringWrite(t *testing.T) { // Each case depends on the previous. s := NewStateStore(nil) + testTime := time.Now() + + type expectations struct { + peering *pbpeering.Peering + secrets *pbpeering.PeeringSecrets + err string + } type testcase struct { - name string - input *pbpeering.PeeringWriteRequest - expectSecrets *pbpeering.PeeringSecrets - expectErr string + name string + input *pbpeering.PeeringWriteRequest + expect expectations } run := func(t *testing.T, tc testcase) { err := s.PeeringWrite(10, tc.input) - if tc.expectErr != "" { - testutil.RequireErrorContains(t, err, tc.expectErr) + if tc.expect.err != "" { + testutil.RequireErrorContains(t, err, tc.expect.err) return } require.NoError(t, err) @@ -1133,57 +1139,185 @@ func TestStore_PeeringWrite(t *testing.T) { _, p, err := s.PeeringRead(nil, q) require.NoError(t, err) require.NotNil(t, p) - require.Equal(t, tc.input.Peering.State, p.State) - require.Equal(t, tc.input.Peering.Name, p.Name) + require.Equal(t, tc.expect.peering.State, p.State) + require.Equal(t, tc.expect.peering.Name, p.Name) + require.Equal(t, tc.expect.peering.Meta, p.Meta) + if tc.expect.peering.DeletedAt != nil { + require.Equal(t, tc.expect.peering.DeletedAt, p.DeletedAt) + } secrets, err := s.PeeringSecretsRead(nil, tc.input.Peering.ID) require.NoError(t, err) - prototest.AssertDeepEqual(t, tc.expectSecrets, secrets) + prototest.AssertDeepEqual(t, tc.expect.secrets, secrets) } tcs := []testcase{ { name: "create baz", input: &pbpeering.PeeringWriteRequest{ Peering: &pbpeering.Peering{ - ID: testBazPeerID, - Name: "baz", - Partition: structs.NodeEnterpriseMetaInDefaultPartition().PartitionOrEmpty(), + ID: testBazPeerID, + Name: "baz", + State: pbpeering.PeeringState_ESTABLISHING, + PeerServerAddresses: []string{"localhost:8502"}, + Partition: structs.NodeEnterpriseMetaInDefaultPartition().PartitionOrEmpty(), }, SecretsRequest: &pbpeering.SecretsWriteRequest{ PeerID: testBazPeerID, - Request: &pbpeering.SecretsWriteRequest_GenerateToken{ - GenerateToken: &pbpeering.SecretsWriteRequest_GenerateTokenRequest{ - EstablishmentSecret: testBazSecretID, + Request: &pbpeering.SecretsWriteRequest_Establish{ + Establish: &pbpeering.SecretsWriteRequest_EstablishRequest{ + ActiveStreamSecret: testBazSecretID, }, }, }, }, - expectSecrets: &pbpeering.PeeringSecrets{ - PeerID: testBazPeerID, - Establishment: &pbpeering.PeeringSecrets_Establishment{ - SecretID: testBazSecretID, + expect: expectations{ + peering: &pbpeering.Peering{ + ID: testBazPeerID, + Name: "baz", + State: pbpeering.PeeringState_ESTABLISHING, }, + secrets: &pbpeering.PeeringSecrets{ + PeerID: testBazPeerID, + Stream: &pbpeering.PeeringSecrets_Stream{ + ActiveSecretID: testBazSecretID, + }, + }, + }, + }, + { + name: "cannot change ID for baz", + input: &pbpeering.PeeringWriteRequest{ + Peering: &pbpeering.Peering{ + ID: "123", + Name: "baz", + State: pbpeering.PeeringState_FAILING, + PeerServerAddresses: []string{"localhost:8502"}, + Partition: structs.NodeEnterpriseMetaInDefaultPartition().PartitionOrEmpty(), + }, + }, + expect: expectations{ + err: `A peering already exists with the name "baz" and a different ID`, }, }, { name: "update baz", input: &pbpeering.PeeringWriteRequest{ Peering: &pbpeering.Peering{ - ID: testBazPeerID, - Name: "baz", - State: pbpeering.PeeringState_FAILING, + ID: testBazPeerID, + Name: "baz", + State: pbpeering.PeeringState_FAILING, + PeerServerAddresses: []string{"localhost:8502"}, + Partition: structs.NodeEnterpriseMetaInDefaultPartition().PartitionOrEmpty(), + }, + }, + expect: expectations{ + peering: &pbpeering.Peering{ + ID: testBazPeerID, + Name: "baz", + State: pbpeering.PeeringState_FAILING, + }, + secrets: &pbpeering.PeeringSecrets{ + PeerID: testBazPeerID, + Stream: &pbpeering.PeeringSecrets_Stream{ + ActiveSecretID: testBazSecretID, + }, + }, + }, + }, + { + name: "if no state was included in request it is inherited from existing", + input: &pbpeering.PeeringWriteRequest{ + Peering: &pbpeering.Peering{ + ID: testBazPeerID, + Name: "baz", + // Send undefined state. + // State: pbpeering.PeeringState_FAILING, + PeerServerAddresses: []string{"localhost:8502"}, + Partition: structs.NodeEnterpriseMetaInDefaultPartition().PartitionOrEmpty(), + }, + }, + expect: expectations{ + peering: &pbpeering.Peering{ + ID: testBazPeerID, + Name: "baz", + // Previous failing state is picked up. + State: pbpeering.PeeringState_FAILING, + }, + secrets: &pbpeering.PeeringSecrets{ + PeerID: testBazPeerID, + Stream: &pbpeering.PeeringSecrets_Stream{ + ActiveSecretID: testBazSecretID, + }, + }, + }, + }, + { + name: "mark baz as terminated", + input: &pbpeering.PeeringWriteRequest{ + Peering: &pbpeering.Peering{ + ID: testBazPeerID, + Name: "baz", + State: pbpeering.PeeringState_TERMINATED, + PeerServerAddresses: []string{"localhost:8502"}, + Partition: structs.NodeEnterpriseMetaInDefaultPartition().PartitionOrEmpty(), + }, + }, + expect: expectations{ + peering: &pbpeering.Peering{ + ID: testBazPeerID, + Name: "baz", + State: pbpeering.PeeringState_TERMINATED, + }, + // Secrets for baz should have been deleted + secrets: nil, + }, + }, + { + name: "cannot edit data during no-op termination", + input: &pbpeering.PeeringWriteRequest{ + Peering: &pbpeering.Peering{ + ID: testBazPeerID, + Name: "baz", + State: pbpeering.PeeringState_TERMINATED, + // Attempt to modify the addresses + Meta: map[string]string{"foo": "bar"}, Partition: structs.NodeEnterpriseMetaInDefaultPartition().PartitionOrEmpty(), }, }, - expectSecrets: &pbpeering.PeeringSecrets{ - PeerID: testBazPeerID, - Establishment: &pbpeering.PeeringSecrets_Establishment{ - SecretID: testBazSecretID, + expect: expectations{ + peering: &pbpeering.Peering{ + ID: testBazPeerID, + Name: "baz", + State: pbpeering.PeeringState_TERMINATED, + // Meta should be unchanged. + Meta: nil, }, }, }, { name: "mark baz for deletion", + input: &pbpeering.PeeringWriteRequest{ + Peering: &pbpeering.Peering{ + ID: testBazPeerID, + Name: "baz", + State: pbpeering.PeeringState_DELETING, + PeerServerAddresses: []string{"localhost:8502"}, + DeletedAt: structs.TimeToProto(testTime), + Partition: structs.NodeEnterpriseMetaInDefaultPartition().PartitionOrEmpty(), + }, + }, + expect: expectations{ + peering: &pbpeering.Peering{ + ID: testBazPeerID, + Name: "baz", + State: pbpeering.PeeringState_DELETING, + DeletedAt: structs.TimeToProto(testTime), + }, + secrets: nil, + }, + }, + { + name: "deleting a deleted peering is a no-op", input: &pbpeering.PeeringWriteRequest{ Peering: &pbpeering.Peering{ ID: testBazPeerID, @@ -1193,8 +1327,38 @@ func TestStore_PeeringWrite(t *testing.T) { Partition: structs.NodeEnterpriseMetaInDefaultPartition().PartitionOrEmpty(), }, }, - // Secrets for baz should have been deleted - expectSecrets: nil, + expect: expectations{ + peering: &pbpeering.Peering{ + ID: testBazPeerID, + Name: "baz", + // Still marked as deleting at the original testTime + State: pbpeering.PeeringState_DELETING, + DeletedAt: structs.TimeToProto(testTime), + }, + // Secrets for baz should have been deleted + secrets: nil, + }, + }, + { + name: "terminating a peering marked for deletion is a no-op", + input: &pbpeering.PeeringWriteRequest{ + Peering: &pbpeering.Peering{ + ID: testBazPeerID, + Name: "baz", + State: pbpeering.PeeringState_TERMINATED, + Partition: structs.NodeEnterpriseMetaInDefaultPartition().PartitionOrEmpty(), + }, + }, + expect: expectations{ + peering: &pbpeering.Peering{ + ID: testBazPeerID, + Name: "baz", + // Still marked as deleting + State: pbpeering.PeeringState_DELETING, + }, + // Secrets for baz should have been deleted + secrets: nil, + }, }, { name: "cannot update peering marked for deletion", @@ -1209,7 +1373,9 @@ func TestStore_PeeringWrite(t *testing.T) { Partition: structs.NodeEnterpriseMetaInDefaultPartition().PartitionOrEmpty(), }, }, - expectErr: "cannot write to peering that is marked for deletion", + expect: expectations{ + err: "cannot write to peering that is marked for deletion", + }, }, { name: "cannot create peering marked for deletion", @@ -1221,7 +1387,9 @@ func TestStore_PeeringWrite(t *testing.T) { Partition: structs.NodeEnterpriseMetaInDefaultPartition().PartitionOrEmpty(), }, }, - expectErr: "cannot create a new peering marked for deletion", + expect: expectations{ + err: "cannot create a new peering marked for deletion", + }, }, } for _, tc := range tcs { diff --git a/agent/rpc/peering/service.go b/agent/rpc/peering/service.go index 20bbafc1cf..6c0950d9e9 100644 --- a/agent/rpc/peering/service.go +++ b/agent/rpc/peering/service.go @@ -726,11 +726,12 @@ func (s *Server) PeeringDelete(ctx context.Context, req *pbpeering.PeeringDelete return nil, err } - if !existing.IsActive() { + if existing == nil || existing.State == pbpeering.PeeringState_DELETING { // Return early when the Peering doesn't exist or is already marked for deletion. // We don't return nil because the pb will fail to marshal. return &pbpeering.PeeringDeleteResponse{}, nil } + // We are using a write request due to needing to perform a deferred deletion. // The peering gets marked for deletion by setting the DeletedAt field, // and a leader routine will handle deleting the peering. diff --git a/agent/rpc/peering/service_test.go b/agent/rpc/peering/service_test.go index 54770d6a61..6b26db7d04 100644 --- a/agent/rpc/peering/service_test.go +++ b/agent/rpc/peering/service_test.go @@ -621,38 +621,50 @@ func TestPeeringService_Read_ACLEnforcement(t *testing.T) { } func TestPeeringService_Delete(t *testing.T) { - // TODO(peering): see note on newTestServer, refactor to not use this - s := newTestServer(t, nil) - - p := &pbpeering.Peering{ - ID: testUUID(t), - Name: "foo", - State: pbpeering.PeeringState_ESTABLISHING, - PeerCAPems: nil, - PeerServerName: "test", - PeerServerAddresses: []string{"addr1"}, + tt := map[string]pbpeering.PeeringState{ + "active peering": pbpeering.PeeringState_ACTIVE, + "terminated peering": pbpeering.PeeringState_TERMINATED, } - err := s.Server.FSM().State().PeeringWrite(10, &pbpeering.PeeringWriteRequest{Peering: p}) - require.NoError(t, err) - require.Nil(t, p.DeletedAt) - require.True(t, p.IsActive()) - client := pbpeering.NewPeeringServiceClient(s.ClientConn(t)) + for name, overrideState := range tt { + t.Run(name, func(t *testing.T) { + // TODO(peering): see note on newTestServer, refactor to not use this + s := newTestServer(t, nil) - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - t.Cleanup(cancel) + // A pointer is kept for the following peering so that we can modify the object without another PeeringWrite. + p := &pbpeering.Peering{ + ID: testUUID(t), + Name: "foo", + PeerCAPems: nil, + PeerServerName: "test", + PeerServerAddresses: []string{"addr1"}, + } + err := s.Server.FSM().State().PeeringWrite(10, &pbpeering.PeeringWriteRequest{Peering: p}) + require.NoError(t, err) + require.Nil(t, p.DeletedAt) + require.True(t, p.IsActive()) - _, err = client.PeeringDelete(ctx, &pbpeering.PeeringDeleteRequest{Name: "foo"}) - require.NoError(t, err) + // Overwrite the peering state to simulate deleting from a non-initial state. + p.State = overrideState - retry.Run(t, func(r *retry.R) { - _, resp, err := s.Server.FSM().State().PeeringRead(nil, state.Query{Value: "foo"}) - require.NoError(r, err) + client := pbpeering.NewPeeringServiceClient(s.ClientConn(t)) - // Initially the peering will be marked for deletion but eventually the leader - // routine will clean it up. - require.Nil(r, resp) - }) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + t.Cleanup(cancel) + + _, err = client.PeeringDelete(ctx, &pbpeering.PeeringDeleteRequest{Name: "foo"}) + require.NoError(t, err) + + retry.Run(t, func(r *retry.R) { + _, resp, err := s.Server.FSM().State().PeeringRead(nil, state.Query{Value: "foo"}) + require.NoError(r, err) + + // Initially the peering will be marked for deletion but eventually the leader + // routine will clean it up. + require.Nil(r, resp) + }) + }) + } } func TestPeeringService_Delete_ACLEnforcement(t *testing.T) { From b1ba0a89bc4c577fc65a21995ff7a25c199beaa0 Mon Sep 17 00:00:00 2001 From: David Yu Date: Fri, 26 Aug 2022 13:37:41 -0700 Subject: [PATCH 18/31] docs: Release notes for Consul 1.12, 1.13 and Consul K8s 0.47.0 (#14352) * consul 1.12, consul 1.13, and consul-k8s release notes Co-authored-by: Jeff Boruszak <104028618+boruszak@users.noreply.github.com> --- website/content/docs/lambda/invocation.mdx | 2 +- .../consul-api-gateway/v0_1_x.mdx | 2 +- .../docs/release-notes/consul-k8s/v0_47_x.mdx | 48 +++++++++++++++++ .../docs/release-notes/consul/v1_10_x.mdx | 2 + .../docs/release-notes/consul/v1_11_x.mdx | 2 + .../docs/release-notes/consul/v1_12_x.mdx | 54 +++++++++++++++++++ .../docs/release-notes/consul/v1_13_x.mdx | 44 +++++++++++++++ website/data/docs-nav-data.json | 17 ++++++ 8 files changed, 169 insertions(+), 2 deletions(-) create mode 100644 website/content/docs/release-notes/consul-k8s/v0_47_x.mdx create mode 100644 website/content/docs/release-notes/consul/v1_12_x.mdx create mode 100644 website/content/docs/release-notes/consul/v1_13_x.mdx diff --git a/website/content/docs/lambda/invocation.mdx b/website/content/docs/lambda/invocation.mdx index 991c8eb92d..4789c0adac 100644 --- a/website/content/docs/lambda/invocation.mdx +++ b/website/content/docs/lambda/invocation.mdx @@ -72,7 +72,7 @@ service mesh. } ``` 1. Issue the `consul services register` command to store the configuration: - ```shell-sesion + ```shell-session $ consul services register api-sidecar-proxy.hcl ``` 1. Call the upstream service to invoke the Lambda function. In the following example, the `api` service invokes the `authentication` service at `localhost:2345`: diff --git a/website/content/docs/release-notes/consul-api-gateway/v0_1_x.mdx b/website/content/docs/release-notes/consul-api-gateway/v0_1_x.mdx index 357fd7032d..b191e77fd2 100644 --- a/website/content/docs/release-notes/consul-api-gateway/v0_1_x.mdx +++ b/website/content/docs/release-notes/consul-api-gateway/v0_1_x.mdx @@ -8,7 +8,7 @@ description: >- # Consul API Gateway 0.1.0 -## OVerview +## Overview This is the first general availability (GA) release of Consul API Gateway. It provides controlled access for network traffic from outside a Consul service diff --git a/website/content/docs/release-notes/consul-k8s/v0_47_x.mdx b/website/content/docs/release-notes/consul-k8s/v0_47_x.mdx new file mode 100644 index 0000000000..b040787c5c --- /dev/null +++ b/website/content/docs/release-notes/consul-k8s/v0_47_x.mdx @@ -0,0 +1,48 @@ +--- +layout: docs +page_title: 0.47.x +description: >- + Consul on Kubernetes release notes for version 0.47.x +--- + +# Consul on Kubernetes 0.47.0 + +## Release Highlights + +- **Cluster Peering (Beta)**: This release introduces support for Cluster Peering, which allows service connectivity between two independent clusters. Enabling peering will deploy the peering controllers and PeeringAcceptor and PeeringDialer CRDs. The new CRDs are used to establish a peering connection between two clusters. Refer to [Cluster Peering on Kubernetes](/docs/connect/cluster-peering/k8s) for full instructions on using Cluster Peering on Kubernetes. + +- **Envoy Proxy Debugging CLI Commands**: This release introduces new commands to quickly identify proxies and troubleshoot Envoy proxies for sidecars and gateways. + * Add `consul-k8s proxy list` command for displaying pods running Envoy managed by Consul. + * Add `consul-k8s proxy read podname` command for displaying Envoy configuration for a given pod + +- **Transparent Proxy Egress**: Adds support for destinations on the Service Defaults CRD when using transparent proxy for terminating gateways. + +## Supported Software + +- Consul 1.11.x, Consul 1.12.x and Consul 1.13.1+ +- Kubernetes 1.19+ + - Kubernetes 1.24 is not supported at this time. +- Kubectl 1.21+ +- Envoy proxy support is determined by the Consul version deployed. Refer to + [Envoy Integration](/docs/connect/proxies/envoy) for details. + +## Upgrading + +For detailed information on upgrading, please refer to the [Upgrades page](/docs/k8s/upgrade) + +## Known Issues + +The following issues are know to exist in the v0.47.0 and v0.47.1 releases + +- Kubernetes 1.24 is not supported because secret-based tokens are no longer autocreated by default for service accounts. Refer to GitHub issue + [[GH-1145](https://github.com/hashicorp/consul-k8s/issues/1145)] for + details. + +## Changelogs + +The changelogs for this major release version and any maintenance versions are listed below. + +~> **Note:** The following link takes you to the changelogs on the GitHub website. + +- [0.47.0](https://github.com/hashicorp/consul-k8s/releases/tag/v0.47.0) +- [0.47.1](https://github.com/hashicorp/consul-k8s/releases/tag/v0.47.1) diff --git a/website/content/docs/release-notes/consul/v1_10_x.mdx b/website/content/docs/release-notes/consul/v1_10_x.mdx index e4531dec08..e55b2ce32c 100644 --- a/website/content/docs/release-notes/consul/v1_10_x.mdx +++ b/website/content/docs/release-notes/consul/v1_10_x.mdx @@ -24,6 +24,8 @@ description: >- - Drops support for Envoy version 1.13.x. - (Enterprise Only) Consul Enterprise has removed support for temporary licensing. All server agents must have a valid license at startup and client agents must have a license at startup or be able to retrieve one from the servers. +## Upgrading + For more detailed information, please refer to the [upgrade details page](/docs/upgrading/upgrade-specific#consul-1-10-0) and the changelogs. ## Changelogs diff --git a/website/content/docs/release-notes/consul/v1_11_x.mdx b/website/content/docs/release-notes/consul/v1_11_x.mdx index eeb468003d..d26cd6a804 100644 --- a/website/content/docs/release-notes/consul/v1_11_x.mdx +++ b/website/content/docs/release-notes/consul/v1_11_x.mdx @@ -27,6 +27,8 @@ description: >- - Drops support for Envoy versions 1.15.x and 1.16.x +## Upgrading + For more detailed information, please refer to the [upgrade details page](/docs/upgrading/upgrade-specific#consul-1-11-0) and the changelogs. ## Changelogs diff --git a/website/content/docs/release-notes/consul/v1_12_x.mdx b/website/content/docs/release-notes/consul/v1_12_x.mdx new file mode 100644 index 0000000000..842dfb31c8 --- /dev/null +++ b/website/content/docs/release-notes/consul/v1_12_x.mdx @@ -0,0 +1,54 @@ +--- +layout: docs +page_title: 1.12.x +description: >- + Consul release notes for version 1.12.x +--- + +# Consul 1.12.0 + +## Release Highlights + +- **AWS IAM Auth Method**: Consul now provides an AWS IAM auth method that allows AWS IAM roles and users to authenticate with Consul to obtain ACL tokens. Refer to [AWS IAM Auth Method](/docs/security/acl/auth-methods/aws-iam) for detailed configuration information. + +- **Per listener TLS Config**: It is now possible to configure TLS differently for each of Consul's listeners, such as HTTPS, gRPC, and the internal multiplexed RPC listener, using the `tls` stanza. Refer to [TLS Configuration Reference](/docs/agent/config/config-files#tls-configuration-reference) for more details. + +- **AWS Lambda**: Adds the ability to invoke AWS Lambdas through terminating gateways, which allows for cross-datacenter communication, transparent proxy, and intentions with Consul Service Mesh. Refer to [AWS Lambda](/docs]/lambda) and [Invoke Lambda Functions](/docs/lambda/invocation) for more details. + +- **Mesh-wide TLS min/max versions and cipher suites:** Using the [Mesh](/docs/connect/config-entries/mesh#tls) Config Entry or CRD, it is now possible to set TLS min/max versions and cipher suites for both inbound and outbound mTLS connections. + +- **Expanded details for ACL Permission Denied errors**: Details are now provided when a permission denied errors surface for RPC calls. Details include the accessor ID of the ACL token, the missing permission, and any namespace or partition that the error occurred on. + +- **ACL token read**: The `consul acl token read -rules` command now includes an `-expanded` option to display detailed info about any policies and rules affecting the token. Refer to [Consul ACL Token read](/commands/acl/token/read) for more details. + +- **Automatically reload agent config when watching agent config file changes**: When using the `auto-reload-config` CLI flag or `auto_reload_config` agent config option, Consul now automatically reloads the [reloadable configuration options](/docs/agent/config#reloadable-configuration) when configuration files change. Refer to [auto_reload_config](/docs/agent/config/cli-flags#_auto_reload_config) for more details. + + +## What's Changed + +- Removes support for Envoy 1.17.x and Envoy 1.18.x, and adds support for Envoy 1.21.x and Envoy 1.22.x. Refer to the [Envoy Compatibility matrix](/docs/connect/proxies/envoy) for more details. + +- The `disable_compat_1.9` option now defaults to true. Metrics formatted in the style of version 1.9, such as `consul.http...`, can still be enabled by setting disable_compat_1.9 = false. However, these metrics will be removed in 1.13. + +- The `agent_master` ACL token has been renamed to `agent_recovery` ACL token. In addition, the `consul acl set-agent-token master` command has been replaced with `consul acl set-agent-token recovery`. Refer to [ACL Agent Recovery Token](/docs/security/acl/acl-tokens#acl-agent-recovery-token) and [Consul ACL Set Agent Token](/commands/acl/set-agent-token) for more information. + +- If TLS min versions and max versions are not specified, the TLS min/max versions default to the following values. For details on how to configure TLS min and max, refer to the [Mesh TLS config entry](/docs/connect/config-entries/mesh#tls) or CRD documentation. + - Incoming connections: TLS 1.2 for min0 version, TLS 1.3 for max version + - Outgoing connections: TLS 1.2 for both TLS min and TLS max versions. + +## Upgrading + +For more detailed information, please refer to the [upgrade details page](/docs/upgrading/upgrade-specific#consul-1-12-0) and the changelogs. + +## Changelogs + +The changelogs for this major release version and any maintenance versions are listed below. + +-> **Note**: These links take you to the changelogs on the GitHub website. + +- [1.12.0](https://github.com/hashicorp/consul/releases/tag/v1.12.0) +- [1.12.1](https://github.com/hashicorp/consul/releases/tag/v1.12.1) +- [1.12.2](https://github.com/hashicorp/consul/releases/tag/v1.12.2) +- [1.12.3](https://github.com/hashicorp/consul/releases/tag/v1.12.3) +- [1.12.4](https://github.com/hashicorp/consul/releases/tag/v1.12.4) + diff --git a/website/content/docs/release-notes/consul/v1_13_x.mdx b/website/content/docs/release-notes/consul/v1_13_x.mdx new file mode 100644 index 0000000000..23b694a913 --- /dev/null +++ b/website/content/docs/release-notes/consul/v1_13_x.mdx @@ -0,0 +1,44 @@ +--- +layout: docs +page_title: 1.13.x +description: >- + Consul release notes for version 1.13.x +--- + +# Consul 1.13.0 + +## Release Highlights + +- **Cluster Peering (Beta)**: This version adds a new model to federate Consul clusters for both service mesh and traditional service discovery. Cluster peering allows for service interconnectivity with looser coupling than the existing WAN federation. For more information, refer to the [cluster peering](/docs/connect/cluster-peering) documentation. + +- **Transparent proxying through terminating gateways**: This version adds egress traffic control to destinations outside of Consul's catalog, such as APIs on the public internet. Transparent proxies can dial [destinations defined in service-defaults](/docs/connect/config-entries/service-defaults#destination) and have the traffic routed through terminating gateways. For more information, refer to the [terminating gateway](/docs/connect/gateways/terminating-gateway#terminating-gateway-configuration) documentation. + +- **Enables TLS on the Envoy Prometheus endpoint**: The Envoy prometheus endpoint can be enabled when `envoy_prometheus_bind_addr` is set and then secured over TLS using new CLI flags for the `consul connect envoy` command. These commands are: `-prometheus-ca-file`, `-prometheus-ca-path`, `-prometheus-cert-file` and `-prometheus-key-file`. The CA, cert, and key can be provided to Envoy by a Kubernetes mounted volume so that Envoy can watch the files and dynamically reload the certs when the volume is updated. + +- **UDP Health Checks**: Adds the ability to register service discovery health checks that periodically send UDP datagrams to the specified IP/hostname and port. Refer to [UDP checks](/docs/discovery/checks#udp-interval). + +## What's Changed + +- Removes support for Envoy 1.19.x and adds suport for Envoy 1.23. Refer to the [Envoy Compatibility matrix](/docs/connect/proxies/envoy) for more details. + +- The [`disable_compat_19`](/docs/agent/options#telemetry-disable_compat_1.9) telemetry configuration option is now removed. In Consul versions 1.10.x through 1.11.x, the config defaulted to `false`. In version 1.12.x it defaulted to `true`. Before upgrading you should remove this flag from your config if the flag is being used. + +## Upgrading + +For more detailed information, please refer to the [upgrade details page](/docs/upgrading/upgrade-specific#consul-1-13-0) and the changelogs. + +## Known Issues +The following issues are know to exist in the 1.13.0 release: + +- Consul 1.13.1 fixes a compatibility issue when restoring snapshots from pre-1.13.0 versions of Consul. Refer to GitHub issue [[GH-14149](https://github.com/hashicorp/consul/issues/14149)] for more details. +- Consul 1.13.0 and Consul 1.13.1 default to requiring TLS for gRPC communication with Envoy proxies when auto-encrypt and auto-config are enabled. In environments where Envoy proxies are not already configured to use TLS for gRPC, upgrading Consul 1.13 will cause Envoy proxies to disconnect from the control plane (Consul agents). A future patch release will default to disabling TLS by default for GRPC communication with Envoy proxies when using Service Mesh and auto-config or auto-encrypt. Refer to GitHub issue [GH-14253](https://github.com/hashicorp/consul/issues/14253) and [Service Mesh deployments using auto-config and auto-enrypt](https://www.consul.io/docs/upgrading/upgrade-specific#service-mesh-deployments-using-auto-encrypt-or-auto-config) for more details. + + +## Changelogs + +The changelogs for this major release version and any maintenance versions are listed below. + +-> **Note**: These links take you to the changelogs on the GitHub website. + +- [1.13.0](https://github.com/hashicorp/consul/releases/tag/v1.13.0) +- [1.13.1](https://github.com/hashicorp/consul/releases/tag/v1.13.1) diff --git a/website/data/docs-nav-data.json b/website/data/docs-nav-data.json index 7f64ed3b45..49b1d91100 100644 --- a/website/data/docs-nav-data.json +++ b/website/data/docs-nav-data.json @@ -1248,6 +1248,14 @@ { "title": "Consul", "routes": [ + { + "title": "v1.13.x", + "path": "release-notes/consul/v1_13_x" + }, + { + "title": "v1.12.x", + "path": "release-notes/consul/v1_12_x" + }, { "title": "v1.11.x", "path": "release-notes/consul/v1_11_x" @@ -1262,6 +1270,15 @@ } ] }, + { + "title": "Consul K8s", + "routes": [ + { + "title": "v0.47.x", + "path": "release-notes/consul-k8s/v0_47_x" + } + ] + }, { "title": "Consul API Gateway", "routes": [ From 4d97e2f9364bfdae39eda1297df3f47fbd7c2aca Mon Sep 17 00:00:00 2001 From: "Chris S. Kim" Date: Fri, 26 Aug 2022 16:49:03 -0400 Subject: [PATCH 19/31] Adjust metrics reporting for peering tracker --- agent/consul/server.go | 11 ++- .../services/peerstream/server.go | 8 +- .../services/peerstream/stream_resources.go | 6 +- .../services/peerstream/stream_test.go | 38 +++------- .../services/peerstream/stream_tracker.go | 51 +++++-------- .../peerstream/stream_tracker_test.go | 76 +++++++------------ 6 files changed, 68 insertions(+), 122 deletions(-) diff --git a/agent/consul/server.go b/agent/consul/server.go index 8f2986c3eb..f92a03ccd2 100644 --- a/agent/consul/server.go +++ b/agent/consul/server.go @@ -742,7 +742,6 @@ func NewServer(config *Config, flat Deps, externalGRPCServer *grpc.Server) (*Ser return s.ForwardGRPC(s.grpcConnPool, info, fn) }, }) - s.peerStreamTracker.SetHeartbeatTimeout(s.peerStreamServer.Config.IncomingHeartbeatTimeout) s.peerStreamServer.Register(s.externalGRPCServer) // Initialize internal gRPC server. @@ -1575,12 +1574,12 @@ func (s *Server) Stats() map[string]map[string]string { // GetLANCoordinate returns the coordinate of the node in the LAN gossip // pool. // -// - Clients return a single coordinate for the single gossip pool they are -// in (default, segment, or partition). +// - Clients return a single coordinate for the single gossip pool they are +// in (default, segment, or partition). // -// - Servers return one coordinate for their canonical gossip pool (i.e. -// default partition/segment) and one per segment they are also ancillary -// members of. +// - Servers return one coordinate for their canonical gossip pool (i.e. +// default partition/segment) and one per segment they are also ancillary +// members of. // // NOTE: servers do not emit coordinates for partitioned gossip pools they // are ancillary members of. diff --git a/agent/grpc-external/services/peerstream/server.go b/agent/grpc-external/services/peerstream/server.go index 6568d7bf80..7254c60c7c 100644 --- a/agent/grpc-external/services/peerstream/server.go +++ b/agent/grpc-external/services/peerstream/server.go @@ -42,8 +42,8 @@ type Config struct { // outgoingHeartbeatInterval is how often we send a heartbeat. outgoingHeartbeatInterval time.Duration - // IncomingHeartbeatTimeout is how long we'll wait between receiving heartbeats before we close the connection. - IncomingHeartbeatTimeout time.Duration + // incomingHeartbeatTimeout is how long we'll wait between receiving heartbeats before we close the connection. + incomingHeartbeatTimeout time.Duration } //go:generate mockery --name ACLResolver --inpackage @@ -63,8 +63,8 @@ func NewServer(cfg Config) *Server { if cfg.outgoingHeartbeatInterval == 0 { cfg.outgoingHeartbeatInterval = defaultOutgoingHeartbeatInterval } - if cfg.IncomingHeartbeatTimeout == 0 { - cfg.IncomingHeartbeatTimeout = defaultIncomingHeartbeatTimeout + if cfg.incomingHeartbeatTimeout == 0 { + cfg.incomingHeartbeatTimeout = defaultIncomingHeartbeatTimeout } return &Server{ Config: cfg, diff --git a/agent/grpc-external/services/peerstream/stream_resources.go b/agent/grpc-external/services/peerstream/stream_resources.go index 0e6b28f45a..ad5d9d4631 100644 --- a/agent/grpc-external/services/peerstream/stream_resources.go +++ b/agent/grpc-external/services/peerstream/stream_resources.go @@ -406,7 +406,7 @@ func (s *Server) realHandleStream(streamReq HandleStreamRequest) error { // incomingHeartbeatCtx will complete if incoming heartbeats time out. incomingHeartbeatCtx, incomingHeartbeatCtxCancel := - context.WithTimeout(context.Background(), s.IncomingHeartbeatTimeout) + context.WithTimeout(context.Background(), s.incomingHeartbeatTimeout) // NOTE: It's important that we wrap the call to cancel in a wrapper func because during the loop we're // re-assigning the value of incomingHeartbeatCtxCancel and we want the defer to run on the last assigned // value, not the current value. @@ -575,6 +575,7 @@ func (s *Server) realHandleStream(streamReq HandleStreamRequest) error { status.TrackRecvResourceSuccess() } + // We are replying ACK or NACK depending on whether we successfully processed the response. if err := streamSend(reply); err != nil { return fmt.Errorf("failed to send to stream: %v", err) } @@ -605,7 +606,7 @@ func (s *Server) realHandleStream(streamReq HandleStreamRequest) error { // They just can't trace the execution properly for some reason (possibly golang/go#29587). //nolint:govet incomingHeartbeatCtx, incomingHeartbeatCtxCancel = - context.WithTimeout(context.Background(), s.IncomingHeartbeatTimeout) + context.WithTimeout(context.Background(), s.incomingHeartbeatTimeout) } case update := <-subCh: @@ -642,7 +643,6 @@ func (s *Server) realHandleStream(streamReq HandleStreamRequest) error { if err := streamSend(replResp); err != nil { return fmt.Errorf("failed to push data for %q: %w", update.CorrelationID, err) } - status.TrackSendSuccess() } } } diff --git a/agent/grpc-external/services/peerstream/stream_test.go b/agent/grpc-external/services/peerstream/stream_test.go index be4a44ec87..fcdd07422b 100644 --- a/agent/grpc-external/services/peerstream/stream_test.go +++ b/agent/grpc-external/services/peerstream/stream_test.go @@ -572,7 +572,7 @@ func TestStreamResources_Server_StreamTracker(t *testing.T) { }) }) - var lastSendAck, lastSendSuccess time.Time + var lastSendAck time.Time testutil.RunStep(t, "ack tracked as success", func(t *testing.T) { ack := &pbpeerstream.ReplicationMessage{ @@ -587,16 +587,13 @@ func TestStreamResources_Server_StreamTracker(t *testing.T) { }, } - lastSendAck = time.Date(2000, time.January, 1, 0, 0, 2, 0, time.UTC) - lastSendSuccess = time.Date(2000, time.January, 1, 0, 0, 3, 0, time.UTC) + lastSendAck = it.FutureNow(1) err := client.Send(ack) require.NoError(t, err) expect := Status{ - Connected: true, - LastAck: lastSendAck, - heartbeatTimeout: defaultIncomingHeartbeatTimeout, - LastSendSuccess: lastSendSuccess, + Connected: true, + LastAck: lastSendAck, } retry.Run(t, func(r *retry.R) { @@ -624,20 +621,17 @@ func TestStreamResources_Server_StreamTracker(t *testing.T) { }, } - lastSendAck = time.Date(2000, time.January, 1, 0, 0, 4, 0, time.UTC) - lastNack = time.Date(2000, time.January, 1, 0, 0, 5, 0, time.UTC) + lastNack = it.FutureNow(1) err := client.Send(nack) require.NoError(t, err) lastNackMsg = "client peer was unable to apply resource: bad bad not good" expect := Status{ - Connected: true, - LastAck: lastSendAck, - LastNack: lastNack, - LastNackMessage: lastNackMsg, - heartbeatTimeout: defaultIncomingHeartbeatTimeout, - LastSendSuccess: lastSendSuccess, + Connected: true, + LastAck: lastSendAck, + LastNack: lastNack, + LastNackMessage: lastNackMsg, } retry.Run(t, func(r *retry.R) { @@ -661,7 +655,7 @@ func TestStreamResources_Server_StreamTracker(t *testing.T) { }, }, } - lastRecvResourceSuccess = it.FutureNow(1) + lastRecvResourceSuccess = it.FutureNow(2) err := client.Send(resp) require.NoError(t, err) @@ -707,8 +701,6 @@ func TestStreamResources_Server_StreamTracker(t *testing.T) { ImportedServices: map[string]struct{}{ api.String(): {}, }, - heartbeatTimeout: defaultIncomingHeartbeatTimeout, - LastSendSuccess: lastSendSuccess, } retry.Run(t, func(r *retry.R) { @@ -770,8 +762,6 @@ func TestStreamResources_Server_StreamTracker(t *testing.T) { ImportedServices: map[string]struct{}{ api.String(): {}, }, - heartbeatTimeout: defaultIncomingHeartbeatTimeout, - LastSendSuccess: lastSendSuccess, } retry.Run(t, func(r *retry.R) { @@ -805,8 +795,6 @@ func TestStreamResources_Server_StreamTracker(t *testing.T) { ImportedServices: map[string]struct{}{ api.String(): {}, }, - heartbeatTimeout: defaultIncomingHeartbeatTimeout, - LastSendSuccess: lastSendSuccess, } retry.Run(t, func(r *retry.R) { @@ -839,8 +827,6 @@ func TestStreamResources_Server_StreamTracker(t *testing.T) { ImportedServices: map[string]struct{}{ api.String(): {}, }, - heartbeatTimeout: defaultIncomingHeartbeatTimeout, - LastSendSuccess: lastSendSuccess, } retry.Run(t, func(r *retry.R) { @@ -1143,7 +1129,7 @@ func TestStreamResources_Server_DisconnectsOnHeartbeatTimeout(t *testing.T) { srv, store := newTestServer(t, func(c *Config) { c.Tracker.SetClock(it.Now) - c.IncomingHeartbeatTimeout = 5 * time.Millisecond + c.incomingHeartbeatTimeout = 5 * time.Millisecond }) p := writePeeringToBeDialed(t, store, 1, "my-peer") @@ -1250,7 +1236,7 @@ func TestStreamResources_Server_KeepsConnectionOpenWithHeartbeat(t *testing.T) { srv, store := newTestServer(t, func(c *Config) { c.Tracker.SetClock(it.Now) - c.IncomingHeartbeatTimeout = incomingHeartbeatTimeout + c.incomingHeartbeatTimeout = incomingHeartbeatTimeout }) p := writePeeringToBeDialed(t, store, 1, "my-peer") diff --git a/agent/grpc-external/services/peerstream/stream_tracker.go b/agent/grpc-external/services/peerstream/stream_tracker.go index ffde98ba32..d0dcd39770 100644 --- a/agent/grpc-external/services/peerstream/stream_tracker.go +++ b/agent/grpc-external/services/peerstream/stream_tracker.go @@ -16,8 +16,6 @@ type Tracker struct { // timeNow is a shim for testing. timeNow func() time.Time - - heartbeatTimeout time.Duration } func NewTracker() *Tracker { @@ -35,12 +33,6 @@ func (t *Tracker) SetClock(clock func() time.Time) { } } -func (t *Tracker) SetHeartbeatTimeout(heartbeatTimeout time.Duration) { - t.mu.Lock() - defer t.mu.Unlock() - t.heartbeatTimeout = heartbeatTimeout -} - // Register a stream for a given peer but do not mark it as connected. func (t *Tracker) Register(id string) (*MutableStatus, error) { t.mu.Lock() @@ -52,7 +44,7 @@ func (t *Tracker) Register(id string) (*MutableStatus, error) { func (t *Tracker) registerLocked(id string, initAsConnected bool) (*MutableStatus, bool, error) { status, ok := t.streams[id] if !ok { - status = newMutableStatus(t.timeNow, t.heartbeatTimeout, initAsConnected) + status = newMutableStatus(t.timeNow, initAsConnected) t.streams[id] = status return status, true, nil } @@ -152,8 +144,6 @@ type MutableStatus struct { // Status contains information about the replication stream to a peer cluster. // TODO(peering): There's a lot of fields here... type Status struct { - heartbeatTimeout time.Duration - // Connected is true when there is an open stream for the peer. Connected bool @@ -182,9 +172,6 @@ type Status struct { // LastSendErrorMessage tracks the last error message when sending into the stream. LastSendErrorMessage string - // LastSendSuccess tracks the time of the last success response sent into the stream. - LastSendSuccess time.Time - // LastRecvHeartbeat tracks when we last received a heartbeat from our peer. LastRecvHeartbeat time.Time @@ -216,38 +203,40 @@ func (s *Status) GetExportedServicesCount() uint64 { // IsHealthy is a convenience func that returns true/ false for a peering status. // We define a peering as unhealthy if its status satisfies one of the following: -// - If heartbeat hasn't been received within the IncomingHeartbeatTimeout -// - If the last sent error is newer than last sent success +// - If it is disconnected +// - If the last received Nack is newer than last received Ack // - If the last received error is newer than last received success // If none of these conditions apply, we call the peering healthy. func (s *Status) IsHealthy() bool { - if time.Now().Sub(s.LastRecvHeartbeat) > s.heartbeatTimeout { - // 1. If heartbeat hasn't been received for a while - report unhealthy + if !s.Connected { return false } - if s.LastSendError.After(s.LastSendSuccess) { - // 2. If last sent error is newer than last sent success - report unhealthy + // If stream is in a disconnected state, report unhealthy. + // This should be logically equivalent to s.Connected above. + if !s.DisconnectTime.IsZero() { return false } + // If last Nack is after last Ack, it means the peer is unable to + // handle our replication messages. + if s.LastNack.After(s.LastAck) { + return false + } + + // If last recv error is newer than last recv success - report unhealthy if s.LastRecvError.After(s.LastRecvResourceSuccess) { - // 3. If last recv error is newer than last recv success - report unhealthy return false } return true } -func newMutableStatus(now func() time.Time, heartbeatTimeout time.Duration, connected bool) *MutableStatus { - if heartbeatTimeout.Microseconds() == 0 { - heartbeatTimeout = defaultIncomingHeartbeatTimeout - } +func newMutableStatus(now func() time.Time, connected bool) *MutableStatus { return &MutableStatus{ Status: Status{ - Connected: connected, - heartbeatTimeout: heartbeatTimeout, - NeverConnected: !connected, + Connected: connected, + NeverConnected: !connected, }, timeNow: now, doneCh: make(chan struct{}), @@ -271,12 +260,6 @@ func (s *MutableStatus) TrackSendError(error string) { s.mu.Unlock() } -func (s *MutableStatus) TrackSendSuccess() { - s.mu.Lock() - s.LastSendSuccess = s.timeNow().UTC() - s.mu.Unlock() -} - // TrackRecvResourceSuccess tracks receiving a replicated resource. func (s *MutableStatus) TrackRecvResourceSuccess() { s.mu.Lock() diff --git a/agent/grpc-external/services/peerstream/stream_tracker_test.go b/agent/grpc-external/services/peerstream/stream_tracker_test.go index 8cdcbc79a2..7500ccd4b8 100644 --- a/agent/grpc-external/services/peerstream/stream_tracker_test.go +++ b/agent/grpc-external/services/peerstream/stream_tracker_test.go @@ -16,11 +16,10 @@ const ( func TestStatus_IsHealthy(t *testing.T) { type testcase struct { - name string - dontConnect bool - modifierFunc func(status *MutableStatus) - expectedVal bool - heartbeatTimeout time.Duration + name string + dontConnect bool + modifierFunc func(status *MutableStatus) + expectedVal bool } tcs := []testcase{ @@ -30,56 +29,39 @@ func TestStatus_IsHealthy(t *testing.T) { dontConnect: true, }, { - name: "no heartbeat, unhealthy", - expectedVal: false, - }, - { - name: "heartbeat is not received, unhealthy", + name: "disconnect time not zero", expectedVal: false, modifierFunc: func(status *MutableStatus) { - // set heartbeat - status.LastRecvHeartbeat = time.Now().Add(-1 * time.Second) - }, - heartbeatTimeout: 1 * time.Second, - }, - { - name: "send error before send success", - expectedVal: false, - modifierFunc: func(status *MutableStatus) { - // set heartbeat - status.LastRecvHeartbeat = time.Now() - - status.LastSendSuccess = time.Now() - status.LastSendError = time.Now() + status.DisconnectTime = time.Now() }, }, { - name: "received error before received success", + name: "receive error before receive success", expectedVal: false, modifierFunc: func(status *MutableStatus) { - // set heartbeat - status.LastRecvHeartbeat = time.Now() - - status.LastRecvResourceSuccess = time.Now() - status.LastRecvError = time.Now() + now := time.Now() + status.LastRecvResourceSuccess = now + status.LastRecvError = now.Add(1 * time.Second) + }, + }, + { + name: "receive error before receive success", + expectedVal: false, + modifierFunc: func(status *MutableStatus) { + now := time.Now() + status.LastAck = now + status.LastNack = now.Add(1 * time.Second) }, }, { name: "healthy", expectedVal: true, - modifierFunc: func(status *MutableStatus) { - // set heartbeat - status.LastRecvHeartbeat = time.Now() - }, }, } for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { tracker := NewTracker() - if tc.heartbeatTimeout.Microseconds() != 0 { - tracker.SetHeartbeatTimeout(tc.heartbeatTimeout) - } if !tc.dontConnect { st, err := tracker.Connected(aPeerID) @@ -120,8 +102,7 @@ func TestTracker_EnsureConnectedDisconnected(t *testing.T) { require.NoError(t, err) expect := Status{ - Connected: true, - heartbeatTimeout: defaultIncomingHeartbeatTimeout, + Connected: true, } status, ok := tracker.StreamStatus(peerID) @@ -147,9 +128,8 @@ func TestTracker_EnsureConnectedDisconnected(t *testing.T) { lastSuccess = it.base.Add(time.Duration(sequence) * time.Second).UTC() expect := Status{ - Connected: true, - LastAck: lastSuccess, - heartbeatTimeout: defaultIncomingHeartbeatTimeout, + Connected: true, + LastAck: lastSuccess, } require.Equal(t, expect, status) }) @@ -159,10 +139,9 @@ func TestTracker_EnsureConnectedDisconnected(t *testing.T) { sequence++ expect := Status{ - Connected: false, - DisconnectTime: it.base.Add(time.Duration(sequence) * time.Second).UTC(), - LastAck: lastSuccess, - heartbeatTimeout: defaultIncomingHeartbeatTimeout, + Connected: false, + DisconnectTime: it.base.Add(time.Duration(sequence) * time.Second).UTC(), + LastAck: lastSuccess, } status, ok := tracker.StreamStatus(peerID) require.True(t, ok) @@ -174,9 +153,8 @@ func TestTracker_EnsureConnectedDisconnected(t *testing.T) { require.NoError(t, err) expect := Status{ - Connected: true, - LastAck: lastSuccess, - heartbeatTimeout: defaultIncomingHeartbeatTimeout, + Connected: true, + LastAck: lastSuccess, // DisconnectTime gets cleared on re-connect. } From 10996654736710e57581443a1bceaef533c57c6a Mon Sep 17 00:00:00 2001 From: Eric Haberkorn Date: Mon, 29 Aug 2022 09:51:32 -0400 Subject: [PATCH 20/31] Update the structs and discovery chain for service resolver redirects to cluster peers. (#14366) --- agent/consul/discoverychain/compile_test.go | 42 ++ agent/structs/config_entry_discoverychain.go | 29 +- .../config_entry_discoverychain_oss_test.go | 22 + .../config_entry_discoverychain_test.go | 46 ++ api/config_entry_discoverychain.go | 1 + api/config_entry_discoverychain_test.go | 14 + proto/pbconfigentry/config_entry.gen.go | 2 + proto/pbconfigentry/config_entry.pb.go | 661 +++++++++--------- proto/pbconfigentry/config_entry.proto | 1 + 9 files changed, 487 insertions(+), 331 deletions(-) diff --git a/agent/consul/discoverychain/compile_test.go b/agent/consul/discoverychain/compile_test.go index 6505fdb9ea..a4c9c65ed7 100644 --- a/agent/consul/discoverychain/compile_test.go +++ b/agent/consul/discoverychain/compile_test.go @@ -39,6 +39,7 @@ func TestCompile(t *testing.T) { "service redirect": testcase_ServiceRedirect(), "service and subset redirect": testcase_ServiceAndSubsetRedirect(), "datacenter redirect": testcase_DatacenterRedirect(), + "redirect to cluster peer": testcase_PeerRedirect(), "datacenter redirect with mesh gateways": testcase_DatacenterRedirect_WithMeshGateways(), "service failover": testcase_ServiceFailover(), "service failover through redirect": testcase_ServiceFailoverThroughRedirect(), @@ -1084,6 +1085,47 @@ func testcase_DatacenterRedirect() compileTestCase { return compileTestCase{entries: entries, expect: expect} } +func testcase_PeerRedirect() compileTestCase { + entries := newEntries() + entries.AddResolvers( + &structs.ServiceResolverConfigEntry{ + Kind: "service-resolver", + Name: "main", + Redirect: &structs.ServiceResolverRedirect{ + Service: "other", + Peer: "cluster-01", + }, + }, + ) + + expect := &structs.CompiledDiscoveryChain{ + Protocol: "tcp", + StartNode: "resolver:other.default.default.external.cluster-01", + Nodes: map[string]*structs.DiscoveryGraphNode{ + "resolver:other.default.default.external.cluster-01": { + Type: structs.DiscoveryGraphNodeTypeResolver, + Name: "other.default.default.external.cluster-01", + Resolver: &structs.DiscoveryResolver{ + Default: true, + ConnectTimeout: 5 * time.Second, + Target: "other.default.default.external.cluster-01", + }, + }, + }, + Targets: map[string]*structs.DiscoveryTarget{ + "other.default.default.external.cluster-01": newTarget(structs.DiscoveryTargetOpts{ + Service: "other", + Peer: "cluster-01", + }, func(t *structs.DiscoveryTarget) { + t.SNI = "" + t.Name = "" + t.Datacenter = "" + }), + }, + } + return compileTestCase{entries: entries, expect: expect} +} + func testcase_DatacenterRedirect_WithMeshGateways() compileTestCase { entries := newEntries() entries.AddProxyDefaults(&structs.ProxyConfigEntry{ diff --git a/agent/structs/config_entry_discoverychain.go b/agent/structs/config_entry_discoverychain.go index 0ea2609551..7867602cae 100644 --- a/agent/structs/config_entry_discoverychain.go +++ b/agent/structs/config_entry_discoverychain.go @@ -964,11 +964,18 @@ func (e *ServiceResolverConfigEntry) Validate() error { // TODO(rb): prevent subsets and default subsets from being defined? - if r.Service == "" && r.ServiceSubset == "" && r.Namespace == "" && r.Partition == "" && r.Datacenter == "" { + if r.isEmpty() { return fmt.Errorf("Redirect is empty") } - if r.Service == "" { + switch { + case r.Peer != "" && r.ServiceSubset != "": + return fmt.Errorf("Redirect.Peer cannot be set with Redirect.ServiceSubset") + case r.Peer != "" && r.Partition != "": + return fmt.Errorf("Redirect.Partition cannot be set with Redirect.Peer") + case r.Peer != "" && r.Datacenter != "": + return fmt.Errorf("Redirect.Peer cannot be set with Redirect.Datacenter") + case r.Service == "": if r.ServiceSubset != "" { return fmt.Errorf("Redirect.ServiceSubset defined without Redirect.Service") } @@ -978,9 +985,12 @@ func (e *ServiceResolverConfigEntry) Validate() error { if r.Partition != "" { return fmt.Errorf("Redirect.Partition defined without Redirect.Service") } - } else if r.Service == e.Name { - if r.ServiceSubset != "" && !isSubset(r.ServiceSubset) { - return fmt.Errorf("Redirect.ServiceSubset %q is not a valid subset of %q", r.ServiceSubset, r.Service) + if r.Peer != "" { + return fmt.Errorf("Redirect.Peer defined without Redirect.Service") + } + case r.ServiceSubset != "" && (r.Service == "" || r.Service == e.Name): + if !isSubset(r.ServiceSubset) { + return fmt.Errorf("Redirect.ServiceSubset %q is not a valid subset of %q", r.ServiceSubset, e.Name) } } } @@ -1231,6 +1241,10 @@ type ServiceResolverRedirect struct { // Datacenter is the datacenter to resolve the service from instead of the // current one (optional). Datacenter string `json:",omitempty"` + + // Peer is the name of the cluster peer to resolve the service from instead + // of the current one (optional). + Peer string `json:",omitempty"` } func (r *ServiceResolverRedirect) ToDiscoveryTargetOpts() DiscoveryTargetOpts { @@ -1240,9 +1254,14 @@ func (r *ServiceResolverRedirect) ToDiscoveryTargetOpts() DiscoveryTargetOpts { Namespace: r.Namespace, Partition: r.Partition, Datacenter: r.Datacenter, + Peer: r.Peer, } } +func (r *ServiceResolverRedirect) isEmpty() bool { + return r.Service == "" && r.ServiceSubset == "" && r.Namespace == "" && r.Partition == "" && r.Datacenter == "" && r.Peer == "" +} + // There are some restrictions on what is allowed in here: // // - Service, ServiceSubset, Namespace, Datacenters, and Targets cannot all be diff --git a/agent/structs/config_entry_discoverychain_oss_test.go b/agent/structs/config_entry_discoverychain_oss_test.go index 81bf6541a1..9f962c8bd4 100644 --- a/agent/structs/config_entry_discoverychain_oss_test.go +++ b/agent/structs/config_entry_discoverychain_oss_test.go @@ -72,6 +72,28 @@ func TestServiceResolverConfigEntry_OSS(t *testing.T) { }, validateErr: `Bad Failover["*"]: Setting Namespace requires Consul Enterprise`, }, + { + name: "setting redirect Namespace on OSS", + entry: &ServiceResolverConfigEntry{ + Kind: ServiceResolver, + Name: "test", + Redirect: &ServiceResolverRedirect{ + Namespace: "ns1", + }, + }, + validateErr: `Redirect: Setting Namespace requires Consul Enterprise`, + }, + { + name: "setting redirect Partition on OSS", + entry: &ServiceResolverConfigEntry{ + Kind: ServiceResolver, + Name: "test", + Redirect: &ServiceResolverRedirect{ + Partition: "ap1", + }, + }, + validateErr: `Redirect: Setting Partition requires Consul Enterprise`, + }, } // Bulk add a bunch of similar validation cases. diff --git a/agent/structs/config_entry_discoverychain_test.go b/agent/structs/config_entry_discoverychain_test.go index 2580ed4c1b..0d6691fa02 100644 --- a/agent/structs/config_entry_discoverychain_test.go +++ b/agent/structs/config_entry_discoverychain_test.go @@ -655,6 +655,41 @@ func TestServiceResolverConfigEntry(t *testing.T) { }, validateErr: `Redirect.ServiceSubset "gone" is not a valid subset of "test"`, }, + { + name: "redirect with peer and subset", + entry: &ServiceResolverConfigEntry{ + Kind: ServiceResolver, + Name: "test", + Redirect: &ServiceResolverRedirect{ + Peer: "cluster-01", + ServiceSubset: "gone", + }, + }, + validateErr: `Redirect.Peer cannot be set with Redirect.ServiceSubset`, + }, + { + name: "redirect with peer and datacenter", + entry: &ServiceResolverConfigEntry{ + Kind: ServiceResolver, + Name: "test", + Redirect: &ServiceResolverRedirect{ + Peer: "cluster-01", + Datacenter: "dc2", + }, + }, + validateErr: `Redirect.Peer cannot be set with Redirect.Datacenter`, + }, + { + name: "redirect with peer and datacenter", + entry: &ServiceResolverConfigEntry{ + Kind: ServiceResolver, + Name: "test", + Redirect: &ServiceResolverRedirect{ + Peer: "cluster-01", + }, + }, + validateErr: `Redirect.Peer defined without Redirect.Service`, + }, { name: "self redirect with valid subset", entry: &ServiceResolverConfigEntry{ @@ -669,6 +704,17 @@ func TestServiceResolverConfigEntry(t *testing.T) { }, }, }, + { + name: "redirect to peer", + entry: &ServiceResolverConfigEntry{ + Kind: ServiceResolver, + Name: "test", + Redirect: &ServiceResolverRedirect{ + Service: "other", + Peer: "cluster-01", + }, + }, + }, { name: "simple wildcard failover", entry: &ServiceResolverConfigEntry{ diff --git a/api/config_entry_discoverychain.go b/api/config_entry_discoverychain.go index fb22baabdb..f827708ee2 100644 --- a/api/config_entry_discoverychain.go +++ b/api/config_entry_discoverychain.go @@ -219,6 +219,7 @@ type ServiceResolverRedirect struct { Namespace string `json:",omitempty"` Partition string `json:",omitempty"` Datacenter string `json:",omitempty"` + Peer string `json:",omitempty"` } type ServiceResolverFailover struct { diff --git a/api/config_entry_discoverychain_test.go b/api/config_entry_discoverychain_test.go index 672aae1af5..c990fa0c68 100644 --- a/api/config_entry_discoverychain_test.go +++ b/api/config_entry_discoverychain_test.go @@ -193,6 +193,20 @@ func TestAPI_ConfigEntry_DiscoveryChain(t *testing.T) { }, verify: verifyResolver, }, + { + name: "redirect to peer", + entry: &ServiceResolverConfigEntry{ + Kind: ServiceResolver, + Name: "test-redirect", + Partition: splitDefaultPartition, + Namespace: splitDefaultNamespace, + Redirect: &ServiceResolverRedirect{ + Service: "test-failover", + Peer: "cluster-01", + }, + }, + verify: verifyResolver, + }, { name: "mega splitter", // use one mega object to avoid multiple trips entry: &ServiceSplitterConfigEntry{ diff --git a/proto/pbconfigentry/config_entry.gen.go b/proto/pbconfigentry/config_entry.gen.go index 7c01387dfb..6bb8fdb0d0 100644 --- a/proto/pbconfigentry/config_entry.gen.go +++ b/proto/pbconfigentry/config_entry.gen.go @@ -689,6 +689,7 @@ func ServiceResolverRedirectToStructs(s *ServiceResolverRedirect, t *structs.Ser t.Namespace = s.Namespace t.Partition = s.Partition t.Datacenter = s.Datacenter + t.Peer = s.Peer } func ServiceResolverRedirectFromStructs(t *structs.ServiceResolverRedirect, s *ServiceResolverRedirect) { if s == nil { @@ -699,6 +700,7 @@ func ServiceResolverRedirectFromStructs(t *structs.ServiceResolverRedirect, s *S s.Namespace = t.Namespace s.Partition = t.Partition s.Datacenter = t.Datacenter + s.Peer = t.Peer } func ServiceResolverSubsetToStructs(s *ServiceResolverSubset, t *structs.ServiceResolverSubset) { if s == nil { diff --git a/proto/pbconfigentry/config_entry.pb.go b/proto/pbconfigentry/config_entry.pb.go index bfa51c8cc6..0a27021543 100644 --- a/proto/pbconfigentry/config_entry.pb.go +++ b/proto/pbconfigentry/config_entry.pb.go @@ -796,6 +796,7 @@ type ServiceResolverRedirect struct { Namespace string `protobuf:"bytes,3,opt,name=Namespace,proto3" json:"Namespace,omitempty"` Partition string `protobuf:"bytes,4,opt,name=Partition,proto3" json:"Partition,omitempty"` Datacenter string `protobuf:"bytes,5,opt,name=Datacenter,proto3" json:"Datacenter,omitempty"` + Peer string `protobuf:"bytes,6,opt,name=Peer,proto3" json:"Peer,omitempty"` } func (x *ServiceResolverRedirect) Reset() { @@ -865,6 +866,13 @@ func (x *ServiceResolverRedirect) GetDatacenter() string { return "" } +func (x *ServiceResolverRedirect) GetPeer() string { + if x != nil { + return x.Peer + } + return "" +} + // mog annotation: // // target=github.com/hashicorp/consul/agent/structs.ServiceResolverFailover @@ -2521,7 +2529,7 @@ var file_proto_pbconfigentry_config_entry_proto_rawDesc = []byte{ 0x65, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x12, 0x20, 0x0a, 0x0b, 0x4f, 0x6e, 0x6c, 0x79, 0x50, 0x61, 0x73, 0x73, 0x69, 0x6e, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, - 0x0b, 0x4f, 0x6e, 0x6c, 0x79, 0x50, 0x61, 0x73, 0x73, 0x69, 0x6e, 0x67, 0x22, 0xb5, 0x01, 0x0a, + 0x0b, 0x4f, 0x6e, 0x6c, 0x79, 0x50, 0x61, 0x73, 0x73, 0x69, 0x6e, 0x67, 0x22, 0xc9, 0x01, 0x0a, 0x17, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x72, 0x52, 0x65, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x53, 0x65, 0x72, 0x76, 0x69, @@ -2533,341 +2541,342 @@ var file_proto_pbconfigentry_config_entry_proto_rawDesc = []byte{ 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1e, 0x0a, 0x0a, 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, - 0x6e, 0x74, 0x65, 0x72, 0x22, 0xf9, 0x01, 0x0a, 0x17, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, - 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x72, 0x46, 0x61, 0x69, 0x6c, 0x6f, 0x76, 0x65, 0x72, - 0x12, 0x18, 0x0a, 0x07, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x07, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x24, 0x0a, 0x0d, 0x53, 0x65, - 0x72, 0x76, 0x69, 0x63, 0x65, 0x53, 0x75, 0x62, 0x73, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0d, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x53, 0x75, 0x62, 0x73, 0x65, 0x74, - 0x12, 0x1c, 0x0a, 0x09, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x09, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x20, - 0x0a, 0x0b, 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x73, 0x18, 0x04, 0x20, - 0x03, 0x28, 0x09, 0x52, 0x0b, 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x73, - 0x12, 0x5e, 0x0a, 0x07, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x44, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, - 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, - 0x65, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x72, 0x46, 0x61, 0x69, 0x6c, 0x6f, 0x76, 0x65, - 0x72, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x07, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, - 0x22, 0xcf, 0x01, 0x0a, 0x1d, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x6f, - 0x6c, 0x76, 0x65, 0x72, 0x46, 0x61, 0x69, 0x6c, 0x6f, 0x76, 0x65, 0x72, 0x54, 0x61, 0x72, 0x67, - 0x65, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x07, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x24, 0x0a, 0x0d, - 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x53, 0x75, 0x62, 0x73, 0x65, 0x74, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x0d, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x53, 0x75, 0x62, 0x73, - 0x65, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, - 0x12, 0x1c, 0x0a, 0x09, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x09, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x1e, - 0x0a, 0x0a, 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x18, 0x05, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x0a, 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x12, 0x12, - 0x0a, 0x04, 0x50, 0x65, 0x65, 0x72, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x50, 0x65, - 0x65, 0x72, 0x22, 0xc7, 0x02, 0x0a, 0x0c, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, - 0x63, 0x65, 0x72, 0x12, 0x16, 0x0a, 0x06, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x06, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x12, 0x5d, 0x0a, 0x0e, 0x52, - 0x69, 0x6e, 0x67, 0x48, 0x61, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x35, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, - 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, - 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x52, 0x69, 0x6e, 0x67, - 0x48, 0x61, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0e, 0x52, 0x69, 0x6e, 0x67, - 0x48, 0x61, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x69, 0x0a, 0x12, 0x4c, 0x65, - 0x61, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x39, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, - 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, - 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x4c, - 0x65, 0x61, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x52, 0x12, 0x4c, 0x65, 0x61, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x43, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x55, 0x0a, 0x0c, 0x48, 0x61, 0x73, 0x68, 0x50, 0x6f, 0x6c, - 0x69, 0x63, 0x69, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x31, 0x2e, 0x68, 0x61, - 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, - 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, - 0x74, 0x72, 0x79, 0x2e, 0x48, 0x61, 0x73, 0x68, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x52, 0x0c, - 0x48, 0x61, 0x73, 0x68, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x69, 0x65, 0x73, 0x22, 0x64, 0x0a, 0x0e, - 0x52, 0x69, 0x6e, 0x67, 0x48, 0x61, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x28, - 0x0a, 0x0f, 0x4d, 0x69, 0x6e, 0x69, 0x6d, 0x75, 0x6d, 0x52, 0x69, 0x6e, 0x67, 0x53, 0x69, 0x7a, - 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0f, 0x4d, 0x69, 0x6e, 0x69, 0x6d, 0x75, 0x6d, - 0x52, 0x69, 0x6e, 0x67, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x28, 0x0a, 0x0f, 0x4d, 0x61, 0x78, 0x69, - 0x6d, 0x75, 0x6d, 0x52, 0x69, 0x6e, 0x67, 0x53, 0x69, 0x7a, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x04, 0x52, 0x0f, 0x4d, 0x61, 0x78, 0x69, 0x6d, 0x75, 0x6d, 0x52, 0x69, 0x6e, 0x67, 0x53, 0x69, - 0x7a, 0x65, 0x22, 0x36, 0x0a, 0x12, 0x4c, 0x65, 0x61, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x20, 0x0a, 0x0b, 0x43, 0x68, 0x6f, 0x69, - 0x63, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0b, 0x43, - 0x68, 0x6f, 0x69, 0x63, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0xd3, 0x01, 0x0a, 0x0a, 0x48, - 0x61, 0x73, 0x68, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x46, 0x69, 0x65, - 0x6c, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x12, - 0x1e, 0x0a, 0x0a, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x0a, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, - 0x57, 0x0a, 0x0c, 0x43, 0x6f, 0x6f, 0x6b, 0x69, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x33, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, + 0x6e, 0x74, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x50, 0x65, 0x65, 0x72, 0x18, 0x06, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x04, 0x50, 0x65, 0x65, 0x72, 0x22, 0xf9, 0x01, 0x0a, 0x17, 0x53, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x72, 0x46, 0x61, 0x69, 0x6c, + 0x6f, 0x76, 0x65, 0x72, 0x12, 0x18, 0x0a, 0x07, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x24, + 0x0a, 0x0d, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x53, 0x75, 0x62, 0x73, 0x65, 0x74, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x53, 0x75, + 0x62, 0x73, 0x65, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, + 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, + 0x63, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, + 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0b, 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, + 0x74, 0x65, 0x72, 0x73, 0x12, 0x5e, 0x0a, 0x07, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x18, + 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x44, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, - 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x43, 0x6f, - 0x6f, 0x6b, 0x69, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0c, 0x43, 0x6f, 0x6f, 0x6b, - 0x69, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1a, 0x0a, 0x08, 0x53, 0x6f, 0x75, 0x72, - 0x63, 0x65, 0x49, 0x50, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x53, 0x6f, 0x75, 0x72, - 0x63, 0x65, 0x49, 0x50, 0x12, 0x1a, 0x0a, 0x08, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x6c, - 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x6c, - 0x22, 0x69, 0x0a, 0x0c, 0x43, 0x6f, 0x6f, 0x6b, 0x69, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x12, 0x18, 0x0a, 0x07, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x08, 0x52, 0x07, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x2b, 0x0a, 0x03, 0x54, 0x54, - 0x4c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x52, 0x03, 0x54, 0x54, 0x4c, 0x12, 0x12, 0x0a, 0x04, 0x50, 0x61, 0x74, 0x68, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x50, 0x61, 0x74, 0x68, 0x22, 0xbf, 0x02, 0x0a, 0x0e, - 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x12, 0x49, - 0x0a, 0x03, 0x54, 0x4c, 0x53, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x37, 0x2e, 0x68, 0x61, - 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, - 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, - 0x74, 0x72, 0x79, 0x2e, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x54, 0x4c, 0x53, 0x43, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x52, 0x03, 0x54, 0x4c, 0x53, 0x12, 0x54, 0x0a, 0x09, 0x4c, 0x69, 0x73, - 0x74, 0x65, 0x6e, 0x65, 0x72, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x36, 0x2e, 0x68, - 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, - 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, - 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x4c, 0x69, 0x73, 0x74, - 0x65, 0x6e, 0x65, 0x72, 0x52, 0x09, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x73, 0x12, - 0x53, 0x0a, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3f, 0x2e, - 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, - 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x47, 0x61, 0x74, - 0x65, 0x77, 0x61, 0x79, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, - 0x4d, 0x65, 0x74, 0x61, 0x1a, 0x37, 0x0a, 0x09, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, - 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, - 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xea, 0x01, - 0x0a, 0x10, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x54, 0x4c, 0x53, 0x43, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x12, 0x18, 0x0a, 0x07, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x08, 0x52, 0x07, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x4c, 0x0a, 0x03, - 0x53, 0x44, 0x53, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x68, 0x61, 0x73, 0x68, + 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x53, 0x65, + 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x72, 0x46, 0x61, 0x69, + 0x6c, 0x6f, 0x76, 0x65, 0x72, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x07, 0x54, 0x61, 0x72, + 0x67, 0x65, 0x74, 0x73, 0x22, 0xcf, 0x01, 0x0a, 0x1d, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x72, 0x46, 0x61, 0x69, 0x6c, 0x6f, 0x76, 0x65, 0x72, + 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, + 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x12, 0x24, 0x0a, 0x0d, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x53, 0x75, 0x62, 0x73, 0x65, + 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x53, 0x75, 0x62, 0x73, 0x65, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, + 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x50, 0x61, 0x72, 0x74, 0x69, + 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1c, 0x0a, 0x09, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, + 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, + 0x63, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, + 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x44, 0x61, 0x74, 0x61, 0x63, 0x65, 0x6e, 0x74, + 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x50, 0x65, 0x65, 0x72, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x04, 0x50, 0x65, 0x65, 0x72, 0x22, 0xc7, 0x02, 0x0a, 0x0c, 0x4c, 0x6f, 0x61, 0x64, 0x42, + 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x12, 0x16, 0x0a, 0x06, 0x50, 0x6f, 0x6c, 0x69, 0x63, + 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x12, + 0x5d, 0x0a, 0x0e, 0x52, 0x69, 0x6e, 0x67, 0x48, 0x61, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x35, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, + 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, + 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, + 0x52, 0x69, 0x6e, 0x67, 0x48, 0x61, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0e, + 0x52, 0x69, 0x6e, 0x67, 0x48, 0x61, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x69, + 0x0a, 0x12, 0x4c, 0x65, 0x61, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x43, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x39, 0x2e, 0x68, 0x61, 0x73, + 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, + 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, + 0x72, 0x79, 0x2e, 0x4c, 0x65, 0x61, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x43, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x12, 0x4c, 0x65, 0x61, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x55, 0x0a, 0x0c, 0x48, 0x61, 0x73, + 0x68, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x69, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x31, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, + 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x61, 0x73, 0x68, 0x50, 0x6f, 0x6c, 0x69, + 0x63, 0x79, 0x52, 0x0c, 0x48, 0x61, 0x73, 0x68, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x69, 0x65, 0x73, + 0x22, 0x64, 0x0a, 0x0e, 0x52, 0x69, 0x6e, 0x67, 0x48, 0x61, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x12, 0x28, 0x0a, 0x0f, 0x4d, 0x69, 0x6e, 0x69, 0x6d, 0x75, 0x6d, 0x52, 0x69, 0x6e, + 0x67, 0x53, 0x69, 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0f, 0x4d, 0x69, 0x6e, + 0x69, 0x6d, 0x75, 0x6d, 0x52, 0x69, 0x6e, 0x67, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x28, 0x0a, 0x0f, + 0x4d, 0x61, 0x78, 0x69, 0x6d, 0x75, 0x6d, 0x52, 0x69, 0x6e, 0x67, 0x53, 0x69, 0x7a, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0f, 0x4d, 0x61, 0x78, 0x69, 0x6d, 0x75, 0x6d, 0x52, 0x69, + 0x6e, 0x67, 0x53, 0x69, 0x7a, 0x65, 0x22, 0x36, 0x0a, 0x12, 0x4c, 0x65, 0x61, 0x73, 0x74, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x20, 0x0a, 0x0b, + 0x43, 0x68, 0x6f, 0x69, 0x63, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0d, 0x52, 0x0b, 0x43, 0x68, 0x6f, 0x69, 0x63, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0xd3, + 0x01, 0x0a, 0x0a, 0x48, 0x61, 0x73, 0x68, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x12, 0x14, 0x0a, + 0x05, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x46, 0x69, + 0x65, 0x6c, 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x56, 0x61, 0x6c, 0x75, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x56, 0x61, + 0x6c, 0x75, 0x65, 0x12, 0x57, 0x0a, 0x0c, 0x43, 0x6f, 0x6f, 0x6b, 0x69, 0x65, 0x43, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x33, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, - 0x79, 0x2e, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x54, 0x4c, 0x53, 0x53, 0x44, 0x53, 0x43, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x03, 0x53, 0x44, 0x53, 0x12, 0x24, 0x0a, 0x0d, 0x54, 0x4c, - 0x53, 0x4d, 0x69, 0x6e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0d, 0x54, 0x4c, 0x53, 0x4d, 0x69, 0x6e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, - 0x12, 0x24, 0x0a, 0x0d, 0x54, 0x4c, 0x53, 0x4d, 0x61, 0x78, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, - 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x54, 0x4c, 0x53, 0x4d, 0x61, 0x78, 0x56, - 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x22, 0x0a, 0x0c, 0x43, 0x69, 0x70, 0x68, 0x65, 0x72, - 0x53, 0x75, 0x69, 0x74, 0x65, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x43, 0x69, - 0x70, 0x68, 0x65, 0x72, 0x53, 0x75, 0x69, 0x74, 0x65, 0x73, 0x22, 0x5b, 0x0a, 0x13, 0x47, 0x61, - 0x74, 0x65, 0x77, 0x61, 0x79, 0x54, 0x4c, 0x53, 0x53, 0x44, 0x53, 0x43, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x12, 0x20, 0x0a, 0x0b, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x4e, - 0x61, 0x6d, 0x65, 0x12, 0x22, 0x0a, 0x0c, 0x43, 0x65, 0x72, 0x74, 0x52, 0x65, 0x73, 0x6f, 0x75, - 0x72, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x43, 0x65, 0x72, 0x74, 0x52, - 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x22, 0xdf, 0x01, 0x0a, 0x0f, 0x49, 0x6e, 0x67, 0x72, - 0x65, 0x73, 0x73, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x50, - 0x6f, 0x72, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x50, 0x6f, 0x72, 0x74, 0x12, - 0x1a, 0x0a, 0x08, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x08, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x51, 0x0a, 0x08, 0x53, - 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x35, 0x2e, - 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, - 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x53, 0x65, 0x72, - 0x76, 0x69, 0x63, 0x65, 0x52, 0x08, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x12, 0x49, - 0x0a, 0x03, 0x54, 0x4c, 0x53, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x37, 0x2e, 0x68, 0x61, - 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, - 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, - 0x74, 0x72, 0x79, 0x2e, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x54, 0x4c, 0x53, 0x43, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x52, 0x03, 0x54, 0x4c, 0x53, 0x22, 0xbe, 0x04, 0x0a, 0x0e, 0x49, 0x6e, - 0x67, 0x72, 0x65, 0x73, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x12, 0x0a, 0x04, - 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, - 0x12, 0x14, 0x0a, 0x05, 0x48, 0x6f, 0x73, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, - 0x05, 0x48, 0x6f, 0x73, 0x74, 0x73, 0x12, 0x50, 0x0a, 0x03, 0x54, 0x4c, 0x53, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x3e, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, - 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, - 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x47, 0x61, 0x74, 0x65, - 0x77, 0x61, 0x79, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x54, 0x4c, 0x53, 0x43, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x52, 0x03, 0x54, 0x4c, 0x53, 0x12, 0x62, 0x0a, 0x0e, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x3a, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, - 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, - 0x64, 0x65, 0x72, 0x4d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x72, 0x73, 0x52, 0x0e, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x12, 0x64, 0x0a, 0x0f, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x18, - 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, - 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, - 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, - 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x4d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x72, - 0x73, 0x52, 0x0f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x65, 0x61, 0x64, 0x65, - 0x72, 0x73, 0x12, 0x53, 0x0a, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x3f, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, + 0x79, 0x2e, 0x43, 0x6f, 0x6f, 0x6b, 0x69, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0c, + 0x43, 0x6f, 0x6f, 0x6b, 0x69, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1a, 0x0a, 0x08, + 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x50, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, + 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x50, 0x12, 0x1a, 0x0a, 0x08, 0x54, 0x65, 0x72, 0x6d, + 0x69, 0x6e, 0x61, 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x54, 0x65, 0x72, 0x6d, + 0x69, 0x6e, 0x61, 0x6c, 0x22, 0x69, 0x0a, 0x0c, 0x43, 0x6f, 0x6f, 0x6b, 0x69, 0x65, 0x43, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x12, 0x18, 0x0a, 0x07, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x2b, + 0x0a, 0x03, 0x54, 0x54, 0x4c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, + 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, + 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x03, 0x54, 0x54, 0x4c, 0x12, 0x12, 0x0a, 0x04, 0x50, + 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x50, 0x61, 0x74, 0x68, 0x22, + 0xbf, 0x02, 0x0a, 0x0e, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x47, 0x61, 0x74, 0x65, 0x77, + 0x61, 0x79, 0x12, 0x49, 0x0a, 0x03, 0x54, 0x4c, 0x53, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x37, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, + 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x54, + 0x4c, 0x53, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x03, 0x54, 0x4c, 0x53, 0x12, 0x54, 0x0a, + 0x09, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x36, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, - 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, - 0x79, 0x52, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x58, 0x0a, 0x0e, 0x45, 0x6e, 0x74, 0x65, 0x72, - 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x30, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, - 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, - 0x6f, 0x6e, 0x2e, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, - 0x61, 0x52, 0x0e, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, - 0x61, 0x1a, 0x37, 0x0a, 0x09, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, - 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, - 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x67, 0x0a, 0x17, 0x47, 0x61, - 0x74, 0x65, 0x77, 0x61, 0x79, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x54, 0x4c, 0x53, 0x43, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x4c, 0x0a, 0x03, 0x53, 0x44, 0x53, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, - 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x47, 0x61, 0x74, 0x65, 0x77, - 0x61, 0x79, 0x54, 0x4c, 0x53, 0x53, 0x44, 0x53, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x03, - 0x53, 0x44, 0x53, 0x22, 0xcb, 0x02, 0x0a, 0x13, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, - 0x65, 0x72, 0x4d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x72, 0x73, 0x12, 0x55, 0x0a, 0x03, 0x41, - 0x64, 0x64, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x43, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, - 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, - 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, - 0x2e, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x4d, 0x6f, 0x64, 0x69, 0x66, - 0x69, 0x65, 0x72, 0x73, 0x2e, 0x41, 0x64, 0x64, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x03, 0x41, - 0x64, 0x64, 0x12, 0x55, 0x0a, 0x03, 0x53, 0x65, 0x74, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, - 0x43, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, - 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, - 0x65, 0x72, 0x4d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x72, 0x73, 0x2e, 0x53, 0x65, 0x74, 0x45, - 0x6e, 0x74, 0x72, 0x79, 0x52, 0x03, 0x53, 0x65, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x52, 0x65, 0x6d, - 0x6f, 0x76, 0x65, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, 0x52, 0x65, 0x6d, 0x6f, 0x76, - 0x65, 0x1a, 0x36, 0x0a, 0x08, 0x41, 0x64, 0x64, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, - 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, - 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x36, 0x0a, 0x08, 0x53, 0x65, 0x74, + 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x52, 0x09, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, + 0x65, 0x72, 0x73, 0x12, 0x53, 0x0a, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x18, 0x03, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x3f, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, + 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, + 0x73, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, + 0x72, 0x79, 0x52, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x1a, 0x37, 0x0a, 0x09, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, - 0x01, 0x22, 0xf6, 0x01, 0x0a, 0x11, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x49, 0x6e, 0x74, - 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x50, 0x0a, 0x07, 0x53, 0x6f, 0x75, 0x72, 0x63, - 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x36, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, - 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, - 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, - 0x2e, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, - 0x52, 0x07, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x12, 0x56, 0x0a, 0x04, 0x4d, 0x65, 0x74, - 0x61, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x42, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, - 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, - 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, - 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, - 0x73, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x4d, 0x65, 0x74, - 0x61, 0x1a, 0x37, 0x0a, 0x09, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, - 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, - 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xa6, 0x06, 0x0a, 0x0f, 0x53, - 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, - 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, - 0x6d, 0x65, 0x12, 0x4e, 0x0a, 0x06, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0e, 0x32, 0x36, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, - 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x6e, - 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x06, 0x41, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x12, 0x5c, 0x0a, 0x0b, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, - 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, - 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, - 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, - 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, - 0x69, 0x6f, 0x6e, 0x52, 0x0b, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x73, - 0x12, 0x1e, 0x0a, 0x0a, 0x50, 0x72, 0x65, 0x63, 0x65, 0x64, 0x65, 0x6e, 0x63, 0x65, 0x18, 0x04, - 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x50, 0x72, 0x65, 0x63, 0x65, 0x64, 0x65, 0x6e, 0x63, 0x65, - 0x12, 0x1a, 0x0a, 0x08, 0x4c, 0x65, 0x67, 0x61, 0x63, 0x79, 0x49, 0x44, 0x18, 0x05, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x08, 0x4c, 0x65, 0x67, 0x61, 0x63, 0x79, 0x49, 0x44, 0x12, 0x4e, 0x0a, 0x04, - 0x54, 0x79, 0x70, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x3a, 0x2e, 0x68, 0x61, 0x73, - 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, - 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, - 0x72, 0x79, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x6f, 0x75, 0x72, - 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x54, 0x79, 0x70, 0x65, 0x12, 0x20, 0x0a, 0x0b, - 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x07, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0b, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x66, - 0x0a, 0x0a, 0x4c, 0x65, 0x67, 0x61, 0x63, 0x79, 0x4d, 0x65, 0x74, 0x61, 0x18, 0x08, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x46, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, - 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x53, 0x6f, 0x75, 0x72, 0x63, - 0x65, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x4c, 0x65, 0x67, 0x61, 0x63, - 0x79, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, 0x4c, 0x65, 0x67, 0x61, - 0x63, 0x79, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x46, 0x0a, 0x10, 0x4c, 0x65, 0x67, 0x61, 0x63, 0x79, - 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, - 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x10, 0x4c, 0x65, - 0x67, 0x61, 0x63, 0x79, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x46, - 0x0a, 0x10, 0x4c, 0x65, 0x67, 0x61, 0x63, 0x79, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x54, 0x69, - 0x6d, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, - 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, - 0x74, 0x61, 0x6d, 0x70, 0x52, 0x10, 0x4c, 0x65, 0x67, 0x61, 0x63, 0x79, 0x55, 0x70, 0x64, 0x61, - 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x58, 0x0a, 0x0e, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, - 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x30, - 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, - 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, - 0x6e, 0x2e, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, - 0x52, 0x0e, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, - 0x12, 0x12, 0x0a, 0x04, 0x50, 0x65, 0x65, 0x72, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, - 0x50, 0x65, 0x65, 0x72, 0x1a, 0x3d, 0x0a, 0x0f, 0x4c, 0x65, 0x67, 0x61, 0x63, 0x79, 0x4d, 0x65, - 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, - 0x02, 0x38, 0x01, 0x22, 0xb9, 0x01, 0x0a, 0x13, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, - 0x6e, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x4e, 0x0a, 0x06, 0x41, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x36, 0x2e, 0x68, 0x61, - 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, - 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, - 0x74, 0x72, 0x79, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x52, 0x06, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x52, 0x0a, 0x04, 0x48, - 0x54, 0x54, 0x50, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3e, 0x2e, 0x68, 0x61, 0x73, 0x68, - 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, - 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, - 0x79, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x48, 0x54, 0x54, 0x50, 0x50, - 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x04, 0x48, 0x54, 0x54, 0x50, 0x22, - 0xed, 0x01, 0x0a, 0x17, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x48, 0x54, 0x54, - 0x50, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1c, 0x0a, 0x09, 0x50, - 0x61, 0x74, 0x68, 0x45, 0x78, 0x61, 0x63, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, - 0x50, 0x61, 0x74, 0x68, 0x45, 0x78, 0x61, 0x63, 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x50, 0x61, 0x74, - 0x68, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x50, - 0x61, 0x74, 0x68, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x12, 0x1c, 0x0a, 0x09, 0x50, 0x61, 0x74, - 0x68, 0x52, 0x65, 0x67, 0x65, 0x78, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x50, 0x61, - 0x74, 0x68, 0x52, 0x65, 0x67, 0x65, 0x78, 0x12, 0x5c, 0x0a, 0x06, 0x48, 0x65, 0x61, 0x64, 0x65, - 0x72, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x44, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, - 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, - 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, - 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, - 0x64, 0x65, 0x72, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x06, 0x48, - 0x65, 0x61, 0x64, 0x65, 0x72, 0x12, 0x18, 0x0a, 0x07, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x73, - 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x73, 0x22, - 0xc1, 0x01, 0x0a, 0x1d, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x48, 0x54, 0x54, - 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, - 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x50, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x74, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x50, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x74, 0x12, - 0x14, 0x0a, 0x05, 0x45, 0x78, 0x61, 0x63, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, - 0x45, 0x78, 0x61, 0x63, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x18, - 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x12, 0x16, 0x0a, - 0x06, 0x53, 0x75, 0x66, 0x66, 0x69, 0x78, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x53, - 0x75, 0x66, 0x66, 0x69, 0x78, 0x12, 0x14, 0x0a, 0x05, 0x52, 0x65, 0x67, 0x65, 0x78, 0x18, 0x06, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x52, 0x65, 0x67, 0x65, 0x78, 0x12, 0x16, 0x0a, 0x06, 0x49, - 0x6e, 0x76, 0x65, 0x72, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x49, 0x6e, 0x76, - 0x65, 0x72, 0x74, 0x2a, 0x77, 0x0a, 0x04, 0x4b, 0x69, 0x6e, 0x64, 0x12, 0x0f, 0x0a, 0x0b, 0x4b, - 0x69, 0x6e, 0x64, 0x55, 0x6e, 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x10, 0x00, 0x12, 0x12, 0x0a, 0x0e, - 0x4b, 0x69, 0x6e, 0x64, 0x4d, 0x65, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x10, 0x01, - 0x12, 0x17, 0x0a, 0x13, 0x4b, 0x69, 0x6e, 0x64, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, - 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x72, 0x10, 0x02, 0x12, 0x16, 0x0a, 0x12, 0x4b, 0x69, 0x6e, - 0x64, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x10, - 0x03, 0x12, 0x19, 0x0a, 0x15, 0x4b, 0x69, 0x6e, 0x64, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, - 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x10, 0x04, 0x2a, 0x26, 0x0a, 0x0f, - 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, - 0x08, 0x0a, 0x04, 0x44, 0x65, 0x6e, 0x79, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x41, 0x6c, 0x6c, - 0x6f, 0x77, 0x10, 0x01, 0x2a, 0x21, 0x0a, 0x13, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, - 0x6e, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0a, 0x0a, 0x06, 0x43, - 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x10, 0x00, 0x42, 0xa6, 0x02, 0x0a, 0x29, 0x63, 0x6f, 0x6d, 0x2e, + 0x01, 0x22, 0xea, 0x01, 0x0a, 0x10, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x54, 0x4c, 0x53, + 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x18, 0x0a, 0x07, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, + 0x12, 0x4c, 0x0a, 0x03, 0x53, 0x44, 0x53, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x65, 0x6e, 0x74, 0x72, 0x79, 0x42, 0x10, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x45, 0x6e, 0x74, - 0x72, 0x79, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, - 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2f, - 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x70, 0x62, 0x63, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0xa2, 0x02, 0x04, 0x48, 0x43, 0x49, - 0x43, 0xaa, 0x02, 0x25, 0x48, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x43, 0x6f, - 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x43, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0xca, 0x02, 0x25, 0x48, 0x61, 0x73, 0x68, - 0x69, 0x63, 0x6f, 0x72, 0x70, 0x5c, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x5c, 0x49, 0x6e, 0x74, - 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5c, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, - 0x79, 0xe2, 0x02, 0x31, 0x48, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x5c, 0x43, 0x6f, - 0x6e, 0x73, 0x75, 0x6c, 0x5c, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5c, 0x43, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, - 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x28, 0x48, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, - 0x70, 0x3a, 0x3a, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x3a, 0x3a, 0x49, 0x6e, 0x74, 0x65, 0x72, - 0x6e, 0x61, 0x6c, 0x3a, 0x3a, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, - 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x54, 0x4c, 0x53, + 0x53, 0x44, 0x53, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x03, 0x53, 0x44, 0x53, 0x12, 0x24, + 0x0a, 0x0d, 0x54, 0x4c, 0x53, 0x4d, 0x69, 0x6e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x54, 0x4c, 0x53, 0x4d, 0x69, 0x6e, 0x56, 0x65, 0x72, + 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x24, 0x0a, 0x0d, 0x54, 0x4c, 0x53, 0x4d, 0x61, 0x78, 0x56, 0x65, + 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x54, 0x4c, 0x53, + 0x4d, 0x61, 0x78, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x22, 0x0a, 0x0c, 0x43, 0x69, + 0x70, 0x68, 0x65, 0x72, 0x53, 0x75, 0x69, 0x74, 0x65, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, + 0x52, 0x0c, 0x43, 0x69, 0x70, 0x68, 0x65, 0x72, 0x53, 0x75, 0x69, 0x74, 0x65, 0x73, 0x22, 0x5b, + 0x0a, 0x13, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x54, 0x4c, 0x53, 0x53, 0x44, 0x53, 0x43, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x20, 0x0a, 0x0b, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, + 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x43, 0x6c, 0x75, 0x73, + 0x74, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x22, 0x0a, 0x0c, 0x43, 0x65, 0x72, 0x74, 0x52, + 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x43, + 0x65, 0x72, 0x74, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x22, 0xdf, 0x01, 0x0a, 0x0f, + 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x12, + 0x12, 0x0a, 0x04, 0x50, 0x6f, 0x72, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x50, + 0x6f, 0x72, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, + 0x51, 0x0a, 0x08, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x35, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, + 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, + 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x08, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, + 0x65, 0x73, 0x12, 0x49, 0x0a, 0x03, 0x54, 0x4c, 0x53, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x37, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, + 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x54, + 0x4c, 0x53, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x03, 0x54, 0x4c, 0x53, 0x22, 0xbe, 0x04, + 0x0a, 0x0e, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x12, 0x12, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, + 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x48, 0x6f, 0x73, 0x74, 0x73, 0x18, 0x02, 0x20, + 0x03, 0x28, 0x09, 0x52, 0x05, 0x48, 0x6f, 0x73, 0x74, 0x73, 0x12, 0x50, 0x0a, 0x03, 0x54, 0x4c, + 0x53, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3e, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, + 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, + 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, + 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x54, 0x4c, + 0x53, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x03, 0x54, 0x4c, 0x53, 0x12, 0x62, 0x0a, 0x0e, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, + 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, + 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, + 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x4d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x72, 0x73, + 0x52, 0x0e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, + 0x12, 0x64, 0x0a, 0x0f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x65, 0x61, 0x64, + 0x65, 0x72, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x68, 0x61, 0x73, 0x68, + 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, + 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, + 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x4d, 0x6f, 0x64, 0x69, + 0x66, 0x69, 0x65, 0x72, 0x73, 0x52, 0x0f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, + 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x12, 0x53, 0x0a, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x18, 0x06, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3f, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, + 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, + 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x49, 0x6e, 0x67, + 0x72, 0x65, 0x73, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x4d, 0x65, 0x74, 0x61, + 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x58, 0x0a, 0x0e, 0x45, + 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x18, 0x07, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, + 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, + 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, + 0x65, 0x4d, 0x65, 0x74, 0x61, 0x52, 0x0e, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, + 0x65, 0x4d, 0x65, 0x74, 0x61, 0x1a, 0x37, 0x0a, 0x09, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, + 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x67, + 0x0a, 0x17, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x54, 0x4c, 0x53, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x4c, 0x0a, 0x03, 0x53, 0x44, 0x53, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, + 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, + 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x47, + 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x54, 0x4c, 0x53, 0x53, 0x44, 0x53, 0x43, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x52, 0x03, 0x53, 0x44, 0x53, 0x22, 0xcb, 0x02, 0x0a, 0x13, 0x48, 0x54, 0x54, 0x50, + 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x4d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x72, 0x73, 0x12, + 0x55, 0x0a, 0x03, 0x41, 0x64, 0x64, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x43, 0x2e, 0x68, + 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, + 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, + 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x4d, + 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x72, 0x73, 0x2e, 0x41, 0x64, 0x64, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x52, 0x03, 0x41, 0x64, 0x64, 0x12, 0x55, 0x0a, 0x03, 0x53, 0x65, 0x74, 0x18, 0x02, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x43, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, + 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, + 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x54, 0x54, 0x50, + 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x4d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x72, 0x73, 0x2e, + 0x53, 0x65, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x03, 0x53, 0x65, 0x74, 0x12, 0x16, 0x0a, + 0x06, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, 0x52, + 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x1a, 0x36, 0x0a, 0x08, 0x41, 0x64, 0x64, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, + 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x36, 0x0a, + 0x08, 0x53, 0x65, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xf6, 0x01, 0x0a, 0x11, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, + 0x65, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x50, 0x0a, 0x07, 0x53, + 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x36, 0x2e, 0x68, + 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, + 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, + 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x6e, + 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x07, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x12, 0x56, 0x0a, + 0x04, 0x4d, 0x65, 0x74, 0x61, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x42, 0x2e, 0x68, 0x61, + 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, + 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, + 0x74, 0x72, 0x79, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x6e, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, + 0x04, 0x4d, 0x65, 0x74, 0x61, 0x1a, 0x37, 0x0a, 0x09, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, + 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xa6, + 0x06, 0x0a, 0x0f, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, + 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x4e, 0x0a, 0x06, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x36, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, + 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, + 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x49, + 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x06, + 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x5c, 0x0a, 0x0b, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, + 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x68, 0x61, + 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, + 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, + 0x74, 0x72, 0x79, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x65, 0x72, + 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x0b, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, + 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x50, 0x72, 0x65, 0x63, 0x65, 0x64, 0x65, 0x6e, + 0x63, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x50, 0x72, 0x65, 0x63, 0x65, 0x64, + 0x65, 0x6e, 0x63, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x4c, 0x65, 0x67, 0x61, 0x63, 0x79, 0x49, 0x44, + 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x4c, 0x65, 0x67, 0x61, 0x63, 0x79, 0x49, 0x44, + 0x12, 0x4e, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x3a, + 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, + 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, + 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x54, 0x79, 0x70, 0x65, + 0x12, 0x20, 0x0a, 0x0b, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, + 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, + 0x6f, 0x6e, 0x12, 0x66, 0x0a, 0x0a, 0x4c, 0x65, 0x67, 0x61, 0x63, 0x79, 0x4d, 0x65, 0x74, 0x61, + 0x18, 0x08, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x46, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, + 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, + 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x53, + 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x4c, + 0x65, 0x67, 0x61, 0x63, 0x79, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, + 0x4c, 0x65, 0x67, 0x61, 0x63, 0x79, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x46, 0x0a, 0x10, 0x4c, 0x65, + 0x67, 0x61, 0x63, 0x79, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x18, 0x09, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, + 0x52, 0x10, 0x4c, 0x65, 0x67, 0x61, 0x63, 0x79, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x69, + 0x6d, 0x65, 0x12, 0x46, 0x0a, 0x10, 0x4c, 0x65, 0x67, 0x61, 0x63, 0x79, 0x55, 0x70, 0x64, 0x61, + 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, + 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x10, 0x4c, 0x65, 0x67, 0x61, 0x63, 0x79, + 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x58, 0x0a, 0x0e, 0x45, 0x6e, + 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x18, 0x0b, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, + 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, + 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, + 0x4d, 0x65, 0x74, 0x61, 0x52, 0x0e, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, + 0x4d, 0x65, 0x74, 0x61, 0x12, 0x12, 0x0a, 0x04, 0x50, 0x65, 0x65, 0x72, 0x18, 0x0c, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x04, 0x50, 0x65, 0x65, 0x72, 0x1a, 0x3d, 0x0a, 0x0f, 0x4c, 0x65, 0x67, 0x61, + 0x63, 0x79, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, + 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xb9, 0x01, 0x0a, 0x13, 0x49, 0x6e, 0x74, 0x65, + 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, + 0x4e, 0x0a, 0x06, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, + 0x36, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, + 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, + 0x6e, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x06, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, + 0x52, 0x0a, 0x04, 0x48, 0x54, 0x54, 0x50, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3e, 0x2e, + 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, + 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x48, + 0x54, 0x54, 0x50, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x04, 0x48, + 0x54, 0x54, 0x50, 0x22, 0xed, 0x01, 0x0a, 0x17, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, + 0x6e, 0x48, 0x54, 0x54, 0x50, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, + 0x1c, 0x0a, 0x09, 0x50, 0x61, 0x74, 0x68, 0x45, 0x78, 0x61, 0x63, 0x74, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x09, 0x50, 0x61, 0x74, 0x68, 0x45, 0x78, 0x61, 0x63, 0x74, 0x12, 0x1e, 0x0a, + 0x0a, 0x50, 0x61, 0x74, 0x68, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x0a, 0x50, 0x61, 0x74, 0x68, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x12, 0x1c, 0x0a, + 0x09, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x67, 0x65, 0x78, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x09, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x67, 0x65, 0x78, 0x12, 0x5c, 0x0a, 0x06, 0x48, + 0x65, 0x61, 0x64, 0x65, 0x72, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x44, 0x2e, 0x68, 0x61, + 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, + 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, + 0x74, 0x72, 0x79, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x48, 0x54, 0x54, + 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, + 0x6e, 0x52, 0x06, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x12, 0x18, 0x0a, 0x07, 0x4d, 0x65, 0x74, + 0x68, 0x6f, 0x64, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x4d, 0x65, 0x74, 0x68, + 0x6f, 0x64, 0x73, 0x22, 0xc1, 0x01, 0x0a, 0x1d, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, + 0x6e, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x50, 0x65, 0x72, 0x6d, 0x69, + 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x50, 0x72, 0x65, + 0x73, 0x65, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x50, 0x72, 0x65, 0x73, + 0x65, 0x6e, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x45, 0x78, 0x61, 0x63, 0x74, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x05, 0x45, 0x78, 0x61, 0x63, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x50, 0x72, 0x65, + 0x66, 0x69, 0x78, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x50, 0x72, 0x65, 0x66, 0x69, + 0x78, 0x12, 0x16, 0x0a, 0x06, 0x53, 0x75, 0x66, 0x66, 0x69, 0x78, 0x18, 0x05, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x06, 0x53, 0x75, 0x66, 0x66, 0x69, 0x78, 0x12, 0x14, 0x0a, 0x05, 0x52, 0x65, 0x67, + 0x65, 0x78, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x52, 0x65, 0x67, 0x65, 0x78, 0x12, + 0x16, 0x0a, 0x06, 0x49, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, + 0x06, 0x49, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x2a, 0x77, 0x0a, 0x04, 0x4b, 0x69, 0x6e, 0x64, 0x12, + 0x0f, 0x0a, 0x0b, 0x4b, 0x69, 0x6e, 0x64, 0x55, 0x6e, 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x10, 0x00, + 0x12, 0x12, 0x0a, 0x0e, 0x4b, 0x69, 0x6e, 0x64, 0x4d, 0x65, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x10, 0x01, 0x12, 0x17, 0x0a, 0x13, 0x4b, 0x69, 0x6e, 0x64, 0x53, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x72, 0x10, 0x02, 0x12, 0x16, 0x0a, + 0x12, 0x4b, 0x69, 0x6e, 0x64, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x47, 0x61, 0x74, 0x65, + 0x77, 0x61, 0x79, 0x10, 0x03, 0x12, 0x19, 0x0a, 0x15, 0x4b, 0x69, 0x6e, 0x64, 0x53, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x10, 0x04, + 0x2a, 0x26, 0x0a, 0x0f, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x12, 0x08, 0x0a, 0x04, 0x44, 0x65, 0x6e, 0x79, 0x10, 0x00, 0x12, 0x09, 0x0a, + 0x05, 0x41, 0x6c, 0x6c, 0x6f, 0x77, 0x10, 0x01, 0x2a, 0x21, 0x0a, 0x13, 0x49, 0x6e, 0x74, 0x65, + 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, + 0x0a, 0x0a, 0x06, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x10, 0x00, 0x42, 0xa6, 0x02, 0x0a, 0x29, + 0x63, 0x6f, 0x6d, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, + 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x42, 0x10, 0x43, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x2f, 0x67, + 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, + 0x6f, 0x72, 0x70, 0x2f, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x2f, 0x70, 0x62, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0xa2, 0x02, + 0x04, 0x48, 0x43, 0x49, 0x43, 0xaa, 0x02, 0x25, 0x48, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, + 0x70, 0x2e, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, + 0x6c, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0xca, 0x02, 0x25, + 0x48, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x5c, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6c, + 0x5c, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5c, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x65, 0x6e, 0x74, 0x72, 0x79, 0xe2, 0x02, 0x31, 0x48, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, + 0x70, 0x5c, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x5c, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, + 0x6c, 0x5c, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x5c, 0x47, 0x50, + 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x28, 0x48, 0x61, 0x73, 0x68, + 0x69, 0x63, 0x6f, 0x72, 0x70, 0x3a, 0x3a, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x3a, 0x3a, 0x49, + 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x3a, 0x3a, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x65, + 0x6e, 0x74, 0x72, 0x79, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/proto/pbconfigentry/config_entry.proto b/proto/pbconfigentry/config_entry.proto index 37d497a68b..f92efc18b1 100644 --- a/proto/pbconfigentry/config_entry.proto +++ b/proto/pbconfigentry/config_entry.proto @@ -122,6 +122,7 @@ message ServiceResolverRedirect { string Namespace = 3; string Partition = 4; string Datacenter = 5; + string Peer = 6; } // mog annotation: From 93271f649c443f90db1631d012fdea9a3b98bf9d Mon Sep 17 00:00:00 2001 From: "Chris S. Kim" Date: Mon, 29 Aug 2022 10:19:46 -0400 Subject: [PATCH 21/31] Fix test --- agent/grpc-external/services/peerstream/stream_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agent/grpc-external/services/peerstream/stream_test.go b/agent/grpc-external/services/peerstream/stream_test.go index fcdd07422b..bee02592a3 100644 --- a/agent/grpc-external/services/peerstream/stream_test.go +++ b/agent/grpc-external/services/peerstream/stream_test.go @@ -655,7 +655,7 @@ func TestStreamResources_Server_StreamTracker(t *testing.T) { }, }, } - lastRecvResourceSuccess = it.FutureNow(2) + lastRecvResourceSuccess = it.FutureNow(1) err := client.Send(resp) require.NoError(t, err) From def529edd3be9c42ab63cc0b92552d78bda1a041 Mon Sep 17 00:00:00 2001 From: "Chris S. Kim" Date: Mon, 29 Aug 2022 10:34:50 -0400 Subject: [PATCH 22/31] Rename test --- agent/grpc-external/services/peerstream/stream_tracker_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agent/grpc-external/services/peerstream/stream_tracker_test.go b/agent/grpc-external/services/peerstream/stream_tracker_test.go index 7500ccd4b8..555e76258b 100644 --- a/agent/grpc-external/services/peerstream/stream_tracker_test.go +++ b/agent/grpc-external/services/peerstream/stream_tracker_test.go @@ -45,7 +45,7 @@ func TestStatus_IsHealthy(t *testing.T) { }, }, { - name: "receive error before receive success", + name: "nack before ack", expectedVal: false, modifierFunc: func(status *MutableStatus) { now := time.Now() From d8cb7731dde161e395b07ccba3566b18c6fe18a5 Mon Sep 17 00:00:00 2001 From: DanStough Date: Tue, 16 Aug 2022 16:47:39 -0400 Subject: [PATCH 23/31] chore: add multi-arch docker build for testing --- GNUmakefile | 30 ++++++++++++++++--- .../docker/Consul-Dev-Multiarch.dockerfile | 5 ++++ 2 files changed, 31 insertions(+), 4 deletions(-) create mode 100644 build-support/docker/Consul-Dev-Multiarch.dockerfile diff --git a/GNUmakefile b/GNUmakefile index 6327ea579b..77d6a7ec21 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -16,6 +16,7 @@ PROTOC_GO_INJECT_TAG_VERSION='v1.3.0' GOTAGS ?= GOPATH=$(shell go env GOPATH) +GOARCH?=$(shell go env GOARCH) MAIN_GOPATH=$(shell go env GOPATH | cut -d: -f1) export PATH := $(PWD)/bin:$(GOPATH)/bin:$(PATH) @@ -152,7 +153,28 @@ dev-docker: linux @docker pull consul:$(CONSUL_IMAGE_VERSION) >/dev/null @echo "Building Consul Development container - $(CONSUL_DEV_IMAGE)" # 'consul:local' tag is needed to run the integration tests - @DOCKER_DEFAULT_PLATFORM=linux/amd64 docker build $(NOCACHE) $(QUIET) -t '$(CONSUL_DEV_IMAGE)' -t 'consul:local' --build-arg CONSUL_IMAGE_VERSION=$(CONSUL_IMAGE_VERSION) $(CURDIR)/pkg/bin/linux_amd64 -f $(CURDIR)/build-support/docker/Consul-Dev.dockerfile + @docker buildx use default && docker buildx build -t 'consul:local' \ + --platform linux/$(GOARCH) \ + --build-arg CONSUL_IMAGE_VERSION=$(CONSUL_IMAGE_VERSION) \ + --load \ + -f $(CURDIR)/build-support/docker/Consul-Dev-Multiarch.dockerfile $(CURDIR)/pkg/bin/ + +check-remote-dev-image-env: +ifndef REMOTE_DEV_IMAGE + $(error REMOTE_DEV_IMAGE is undefined: set this image to /:, e.g. hashicorp/consul-k8s-dev:latest) +endif + +remote-docker: check-remote-dev-image-env + $(MAKE) GOARCH=amd64 linux + $(MAKE) GOARCH=arm64 linux + @echo "Pulling consul container image - $(CONSUL_IMAGE_VERSION)" + @docker pull consul:$(CONSUL_IMAGE_VERSION) >/dev/null + @echo "Building and Pushing Consul Development container - $(REMOTE_DEV_IMAGE)" + @docker buildx use default && docker buildx build -t '$(REMOTE_DEV_IMAGE)' \ + --platform linux/amd64,linux/arm64 \ + --build-arg CONSUL_IMAGE_VERSION=$(CONSUL_IMAGE_VERSION) \ + --push \ + -f $(CURDIR)/build-support/docker/Consul-Dev-Multiarch.dockerfile $(CURDIR)/pkg/bin/ # In CircleCI, the linux binary will be attached from a previous step at bin/. This make target # should only run in CI and not locally. @@ -174,10 +196,10 @@ ifeq ($(CIRCLE_BRANCH), main) @docker push $(CI_DEV_DOCKER_NAMESPACE)/$(CI_DEV_DOCKER_IMAGE_NAME):latest endif -# linux builds a linux binary independent of the source platform +# linux builds a linux binary compatible with the source platform linux: - @mkdir -p ./pkg/bin/linux_amd64 - CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o ./pkg/bin/linux_amd64 -ldflags "$(GOLDFLAGS)" -tags "$(GOTAGS)" + @mkdir -p ./pkg/bin/linux_$(GOARCH) + CGO_ENABLED=0 GOOS=linux GOARCH=$(GOARCH) go build -o ./pkg/bin/linux_$(GOARCH) -ldflags "$(GOLDFLAGS)" -tags "$(GOTAGS)" # dist builds binaries for all platforms and packages them for distribution dist: diff --git a/build-support/docker/Consul-Dev-Multiarch.dockerfile b/build-support/docker/Consul-Dev-Multiarch.dockerfile new file mode 100644 index 0000000000..a3069bd99c --- /dev/null +++ b/build-support/docker/Consul-Dev-Multiarch.dockerfile @@ -0,0 +1,5 @@ +ARG CONSUL_IMAGE_VERSION=latest +FROM consul:${CONSUL_IMAGE_VERSION} +RUN apk update && apk add iptables +ARG TARGETARCH +COPY linux_${TARGETARCH}/consul /bin/consul From 72f90754ae8c6dceeac08da4c1154b64e71dd184 Mon Sep 17 00:00:00 2001 From: Eric Haberkorn Date: Mon, 29 Aug 2022 13:46:41 -0400 Subject: [PATCH 24/31] Update max_ejection_percent on outlier detection for peered clusters to 100% (#14373) We can't trust health checks on peered services when service resolvers, splitters and routers are used. --- .changelog/14373.txt | 3 +++ agent/xds/clusters.go | 9 ++++++++- .../connect-proxy-with-peered-upstreams.latest.golden | 4 ++-- ...transparent-proxy-with-peered-upstreams.latest.golden | 6 +++--- 4 files changed, 16 insertions(+), 6 deletions(-) create mode 100644 .changelog/14373.txt diff --git a/.changelog/14373.txt b/.changelog/14373.txt new file mode 100644 index 0000000000..d9531b09ec --- /dev/null +++ b/.changelog/14373.txt @@ -0,0 +1,3 @@ +```release-note:improvement +xds: Set `max_ejection_percent` on Envoy's outlier detection to 100% for peered services. +``` diff --git a/agent/xds/clusters.go b/agent/xds/clusters.go index adde810d37..c3ac718472 100644 --- a/agent/xds/clusters.go +++ b/agent/xds/clusters.go @@ -772,6 +772,13 @@ func (s *ResourceGenerator) makeUpstreamClusterForPeerService( clusterName := generatePeeredClusterName(uid, tbs) + outlierDetection := ToOutlierDetection(cfg.PassiveHealthCheck) + // We can't rely on health checks for services on cluster peers because they + // don't take into account service resolvers, splitters and routers. Setting + // MaxEjectionPercent too 100% gives outlier detection the power to eject the + // entire cluster. + outlierDetection.MaxEjectionPercent = &wrappers.UInt32Value{Value: 100} + s.Logger.Trace("generating cluster for", "cluster", clusterName) if c == nil { c = &envoy_cluster_v3.Cluster{ @@ -785,7 +792,7 @@ func (s *ResourceGenerator) makeUpstreamClusterForPeerService( CircuitBreakers: &envoy_cluster_v3.CircuitBreakers{ Thresholds: makeThresholdsIfNeeded(cfg.Limits), }, - OutlierDetection: ToOutlierDetection(cfg.PassiveHealthCheck), + OutlierDetection: outlierDetection, } if cfg.Protocol == "http2" || cfg.Protocol == "grpc" { if err := s.setHttp2ProtocolOptions(c); err != nil { diff --git a/agent/xds/testdata/clusters/connect-proxy-with-peered-upstreams.latest.golden b/agent/xds/testdata/clusters/connect-proxy-with-peered-upstreams.latest.golden index d7c23515fc..29059b1435 100644 --- a/agent/xds/testdata/clusters/connect-proxy-with-peered-upstreams.latest.golden +++ b/agent/xds/testdata/clusters/connect-proxy-with-peered-upstreams.latest.golden @@ -58,7 +58,7 @@ "dnsRefreshRate": "10s", "dnsLookupFamily": "V4_ONLY", "outlierDetection": { - + "maxEjectionPercent": 100 }, "commonLbConfig": { "healthyPanicThreshold": { @@ -115,7 +115,7 @@ }, "outlierDetection": { - + "maxEjectionPercent": 100 }, "commonLbConfig": { "healthyPanicThreshold": { diff --git a/agent/xds/testdata/clusters/transparent-proxy-with-peered-upstreams.latest.golden b/agent/xds/testdata/clusters/transparent-proxy-with-peered-upstreams.latest.golden index 0dbbf4277d..d1f6d0bb0e 100644 --- a/agent/xds/testdata/clusters/transparent-proxy-with-peered-upstreams.latest.golden +++ b/agent/xds/testdata/clusters/transparent-proxy-with-peered-upstreams.latest.golden @@ -18,7 +18,7 @@ }, "outlierDetection": { - + "maxEjectionPercent": 100 }, "commonLbConfig": { "healthyPanicThreshold": { @@ -75,7 +75,7 @@ }, "outlierDetection": { - + "maxEjectionPercent": 100 }, "commonLbConfig": { "healthyPanicThreshold": { @@ -157,7 +157,7 @@ }, "outlierDetection": { - + "maxEjectionPercent": 100 }, "commonLbConfig": { "healthyPanicThreshold": { From 310608fb191557a508bb3f7d2c8f0e657f87ea90 Mon Sep 17 00:00:00 2001 From: freddygv Date: Mon, 29 Aug 2022 12:00:30 -0600 Subject: [PATCH 25/31] Add validation to prevent switching dialing mode This prevents unexpected changes to the output of ShouldDial, which should never change unless a peering is deleted and recreated. --- agent/consul/leader_peering_test.go | 12 +++-- agent/consul/state/peering.go | 10 ++++ agent/consul/state/peering_test.go | 76 ++++++++++++++++++++--------- 3 files changed, 72 insertions(+), 26 deletions(-) diff --git a/agent/consul/leader_peering_test.go b/agent/consul/leader_peering_test.go index b8b5166d8f..206608ed4e 100644 --- a/agent/consul/leader_peering_test.go +++ b/agent/consul/leader_peering_test.go @@ -40,6 +40,7 @@ func TestLeader_PeeringSync_Lifecycle_ClientDeletion(t *testing.T) { testLeader_PeeringSync_Lifecycle_ClientDeletion(t, true) }) } + func testLeader_PeeringSync_Lifecycle_ClientDeletion(t *testing.T, enableTLS bool) { if testing.Short() { t.Skip("too slow for testing.Short") @@ -137,9 +138,11 @@ func testLeader_PeeringSync_Lifecycle_ClientDeletion(t *testing.T, enableTLS boo // Delete the peering to trigger the termination sequence. deleted := &pbpeering.Peering{ - ID: p.Peering.ID, - Name: "my-peer-acceptor", - DeletedAt: structs.TimeToProto(time.Now()), + ID: p.Peering.ID, + Name: "my-peer-acceptor", + State: pbpeering.PeeringState_DELETING, + PeerServerAddresses: p.Peering.PeerServerAddresses, + DeletedAt: structs.TimeToProto(time.Now()), } require.NoError(t, dialer.fsm.State().PeeringWrite(2000, &pbpeering.PeeringWriteRequest{Peering: deleted})) dialer.logger.Trace("deleted peering for my-peer-acceptor") @@ -262,6 +265,7 @@ func testLeader_PeeringSync_Lifecycle_AcceptorDeletion(t *testing.T, enableTLS b deleted := &pbpeering.Peering{ ID: p.Peering.PeerID, Name: "my-peer-dialer", + State: pbpeering.PeeringState_DELETING, DeletedAt: structs.TimeToProto(time.Now()), } @@ -431,6 +435,7 @@ func TestLeader_Peering_DeferredDeletion(t *testing.T) { Peering: &pbpeering.Peering{ ID: peerID, Name: peerName, + State: pbpeering.PeeringState_DELETING, DeletedAt: structs.TimeToProto(time.Now()), }, })) @@ -1165,6 +1170,7 @@ func TestLeader_Peering_NoDeletionWhenPeeringDisabled(t *testing.T) { Peering: &pbpeering.Peering{ ID: peerID, Name: peerName, + State: pbpeering.PeeringState_DELETING, DeletedAt: structs.TimeToProto(time.Now()), }, })) diff --git a/agent/consul/state/peering.go b/agent/consul/state/peering.go index 9457dd811a..eef76aa726 100644 --- a/agent/consul/state/peering.go +++ b/agent/consul/state/peering.go @@ -535,6 +535,12 @@ func (s *Store) PeeringWrite(idx uint64, req *pbpeering.PeeringWriteRequest) err if req.Peering.Name == "" { return errors.New("Missing Peering Name") } + if req.Peering.State == pbpeering.PeeringState_DELETING && (req.Peering.DeletedAt == nil || structs.IsZeroProtoTime(req.Peering.DeletedAt)) { + return errors.New("Missing deletion time for peering in deleting state") + } + if req.Peering.DeletedAt != nil && !structs.IsZeroProtoTime(req.Peering.DeletedAt) && req.Peering.State != pbpeering.PeeringState_DELETING { + return fmt.Errorf("Unexpected state for peering with deletion time: %s", pbpeering.PeeringStateToAPI(req.Peering.State)) + } // Ensure the name is unique (cannot conflict with another peering with a different ID). _, existing, err := peeringReadTxn(tx, nil, Query{ @@ -546,6 +552,10 @@ func (s *Store) PeeringWrite(idx uint64, req *pbpeering.PeeringWriteRequest) err } if existing != nil { + if req.Peering.ShouldDial() != existing.ShouldDial() { + return fmt.Errorf("Cannot switch peering dialing mode from %t to %t", existing.ShouldDial(), req.Peering.ShouldDial()) + } + if req.Peering.ID != existing.ID { return fmt.Errorf("A peering already exists with the name %q and a different ID %q", req.Peering.Name, existing.ID) } diff --git a/agent/consul/state/peering_test.go b/agent/consul/state/peering_test.go index 1dc2446fe1..a90727f0eb 100644 --- a/agent/consul/state/peering_test.go +++ b/agent/consul/state/peering_test.go @@ -950,6 +950,7 @@ func TestStore_Peering_Watch(t *testing.T) { Peering: &pbpeering.Peering{ ID: testFooPeerID, Name: "foo", + State: pbpeering.PeeringState_DELETING, DeletedAt: structs.TimeToProto(time.Now()), }, }) @@ -976,6 +977,7 @@ func TestStore_Peering_Watch(t *testing.T) { err := s.PeeringWrite(lastIdx, &pbpeering.PeeringWriteRequest{Peering: &pbpeering.Peering{ ID: testBarPeerID, Name: "bar", + State: pbpeering.PeeringState_DELETING, DeletedAt: structs.TimeToProto(time.Now()), }, }) @@ -1077,6 +1079,7 @@ func TestStore_PeeringList_Watch(t *testing.T) { Peering: &pbpeering.Peering{ ID: testFooPeerID, Name: "foo", + State: pbpeering.PeeringState_DELETING, DeletedAt: structs.TimeToProto(time.Now()), Partition: structs.NodeEnterpriseMetaInDefaultPartition().PartitionOrEmpty(), }, @@ -1199,6 +1202,22 @@ func TestStore_PeeringWrite(t *testing.T) { err: `A peering already exists with the name "baz" and a different ID`, }, }, + { + name: "cannot change dialer status for baz", + input: &pbpeering.PeeringWriteRequest{ + Peering: &pbpeering.Peering{ + ID: "123", + Name: "baz", + State: pbpeering.PeeringState_FAILING, + // Excluding the peer server addresses leads to baz not being considered a dialer. + // PeerServerAddresses: []string{"localhost:8502"}, + Partition: structs.NodeEnterpriseMetaInDefaultPartition().PartitionOrEmpty(), + }, + }, + expect: expectations{ + err: "Cannot switch peering dialing mode from true to false", + }, + }, { name: "update baz", input: &pbpeering.PeeringWriteRequest{ @@ -1273,15 +1292,17 @@ func TestStore_PeeringWrite(t *testing.T) { }, }, { - name: "cannot edit data during no-op termination", + name: "cannot modify peering during no-op termination", input: &pbpeering.PeeringWriteRequest{ Peering: &pbpeering.Peering{ - ID: testBazPeerID, - Name: "baz", - State: pbpeering.PeeringState_TERMINATED, - // Attempt to modify the addresses - Meta: map[string]string{"foo": "bar"}, - Partition: structs.NodeEnterpriseMetaInDefaultPartition().PartitionOrEmpty(), + ID: testBazPeerID, + Name: "baz", + State: pbpeering.PeeringState_TERMINATED, + Partition: structs.NodeEnterpriseMetaInDefaultPartition().PartitionOrEmpty(), + PeerServerAddresses: []string{"localhost:8502"}, + + // Attempt to add metadata + Meta: map[string]string{"foo": "bar"}, }, }, expect: expectations{ @@ -1320,11 +1341,12 @@ func TestStore_PeeringWrite(t *testing.T) { name: "deleting a deleted peering is a no-op", input: &pbpeering.PeeringWriteRequest{ Peering: &pbpeering.Peering{ - ID: testBazPeerID, - Name: "baz", - State: pbpeering.PeeringState_DELETING, - DeletedAt: structs.TimeToProto(time.Now()), - Partition: structs.NodeEnterpriseMetaInDefaultPartition().PartitionOrEmpty(), + ID: testBazPeerID, + Name: "baz", + State: pbpeering.PeeringState_DELETING, + PeerServerAddresses: []string{"localhost:8502"}, + DeletedAt: structs.TimeToProto(time.Now()), + Partition: structs.NodeEnterpriseMetaInDefaultPartition().PartitionOrEmpty(), }, }, expect: expectations{ @@ -1343,10 +1365,11 @@ func TestStore_PeeringWrite(t *testing.T) { name: "terminating a peering marked for deletion is a no-op", input: &pbpeering.PeeringWriteRequest{ Peering: &pbpeering.Peering{ - ID: testBazPeerID, - Name: "baz", - State: pbpeering.PeeringState_TERMINATED, - Partition: structs.NodeEnterpriseMetaInDefaultPartition().PartitionOrEmpty(), + ID: testBazPeerID, + Name: "baz", + State: pbpeering.PeeringState_TERMINATED, + PeerServerAddresses: []string{"localhost:8502"}, + Partition: structs.NodeEnterpriseMetaInDefaultPartition().PartitionOrEmpty(), }, }, expect: expectations{ @@ -1364,13 +1387,15 @@ func TestStore_PeeringWrite(t *testing.T) { name: "cannot update peering marked for deletion", input: &pbpeering.PeeringWriteRequest{ Peering: &pbpeering.Peering{ - ID: testBazPeerID, - Name: "baz", + ID: testBazPeerID, + Name: "baz", + PeerServerAddresses: []string{"localhost:8502"}, + Partition: structs.NodeEnterpriseMetaInDefaultPartition().PartitionOrEmpty(), + // Attempt to add metadata Meta: map[string]string{ "source": "kubernetes", }, - Partition: structs.NodeEnterpriseMetaInDefaultPartition().PartitionOrEmpty(), }, }, expect: expectations{ @@ -1381,10 +1406,12 @@ func TestStore_PeeringWrite(t *testing.T) { name: "cannot create peering marked for deletion", input: &pbpeering.PeeringWriteRequest{ Peering: &pbpeering.Peering{ - ID: testFooPeerID, - Name: "foo", - DeletedAt: structs.TimeToProto(time.Now()), - Partition: structs.NodeEnterpriseMetaInDefaultPartition().PartitionOrEmpty(), + ID: testFooPeerID, + Name: "foo", + PeerServerAddresses: []string{"localhost:8502"}, + State: pbpeering.PeeringState_DELETING, + DeletedAt: structs.TimeToProto(time.Now()), + Partition: structs.NodeEnterpriseMetaInDefaultPartition().PartitionOrEmpty(), }, }, expect: expectations{ @@ -1414,6 +1441,7 @@ func TestStore_PeeringDelete(t *testing.T) { Peering: &pbpeering.Peering{ ID: testFooPeerID, Name: "foo", + State: pbpeering.PeeringState_DELETING, DeletedAt: structs.TimeToProto(time.Now()), }, })) @@ -1927,6 +1955,7 @@ func TestStateStore_PeeringsForService(t *testing.T) { copied := pbpeering.Peering{ ID: tp.peering.ID, Name: tp.peering.Name, + State: pbpeering.PeeringState_DELETING, DeletedAt: structs.TimeToProto(time.Now()), } require.NoError(t, s.PeeringWrite(lastIdx, &pbpeering.PeeringWriteRequest{Peering: &copied})) @@ -2369,6 +2398,7 @@ func TestStore_TrustBundleListByService(t *testing.T) { Peering: &pbpeering.Peering{ ID: peerID1, Name: "peer1", + State: pbpeering.PeeringState_DELETING, DeletedAt: structs.TimeToProto(time.Now()), }, })) From d3955bd84cf7fcb7f3cdc3feb21360c242dc9557 Mon Sep 17 00:00:00 2001 From: freddygv Date: Mon, 29 Aug 2022 12:07:18 -0600 Subject: [PATCH 26/31] Add changelog entry --- .changelog/14364.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .changelog/14364.txt diff --git a/.changelog/14364.txt b/.changelog/14364.txt new file mode 100644 index 0000000000..d2f777af49 --- /dev/null +++ b/.changelog/14364.txt @@ -0,0 +1,3 @@ +```release-note:bugfix +peering: Fix issue preventing deletion and recreation of peerings in TERMINATED state. +``` \ No newline at end of file From daa30a03c37cc5d71038069d1b06d4a22d70f35a Mon Sep 17 00:00:00 2001 From: David Yu Date: Mon, 29 Aug 2022 11:34:39 -0700 Subject: [PATCH 27/31] docs: Update Consul K8s release notes (#14379) --- website/content/docs/release-notes/consul-k8s/v0_47_x.mdx | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/website/content/docs/release-notes/consul-k8s/v0_47_x.mdx b/website/content/docs/release-notes/consul-k8s/v0_47_x.mdx index b040787c5c..a9228e9984 100644 --- a/website/content/docs/release-notes/consul-k8s/v0_47_x.mdx +++ b/website/content/docs/release-notes/consul-k8s/v0_47_x.mdx @@ -20,8 +20,7 @@ description: >- ## Supported Software - Consul 1.11.x, Consul 1.12.x and Consul 1.13.1+ -- Kubernetes 1.19+ - - Kubernetes 1.24 is not supported at this time. +- Kubernetes 1.19-1.23 - Kubectl 1.21+ - Envoy proxy support is determined by the Consul version deployed. Refer to [Envoy Integration](/docs/connect/proxies/envoy) for details. From e6b63221eb403290ece5770ce1f68e44da68cb0f Mon Sep 17 00:00:00 2001 From: David Yu Date: Mon, 29 Aug 2022 13:07:08 -0700 Subject: [PATCH 28/31] docs: Cluster peering with Transparent Proxy updates (#14369) * Update Cluster Peering docs to show example with Transparent Proxy Co-authored-by: trujillo-adam <47586768+trujillo-adam@users.noreply.github.com> --- .../docs/connect/cluster-peering/k8s.mdx | 271 ++++++++++++++---- 1 file changed, 215 insertions(+), 56 deletions(-) diff --git a/website/content/docs/connect/cluster-peering/k8s.mdx b/website/content/docs/connect/cluster-peering/k8s.mdx index 35f17959cb..b18633f091 100644 --- a/website/content/docs/connect/cluster-peering/k8s.mdx +++ b/website/content/docs/connect/cluster-peering/k8s.mdx @@ -25,50 +25,82 @@ You must implement the following requirements to create and use cluster peering - At least two Kubernetes clusters - The installation must be running on Consul on Kubernetes version 0.47.1 or later -### Helm chart configuration +### Prepare for install -To establish cluster peering through Kubernetes, deploy clusters with the following Helm values. +1. After provisioning a Kubernetes cluster and setting up your kubeconfig file to manage access to multiple Kubernetes clusters, export the Kubernetes context names for future use with `kubectl`. For more information on how to use kubeconfig and contexts, refer to [Configure access to multiple clusters](https://kubernetes.io/docs/tasks/access-application-cluster/configure-access-multiple-clusters/) on the Kubernetes documentation website. - + You can use the following methods to get the context names for your clusters: + + * Issue the `kubectl config current-context` command to get the context for the cluster you are currently in. + * Issue the `kubectl config get-contexts` command to get all configured contexts in your kubeconfig file. + + ```shell-session + $ export CLUSTER1_CONTEXT= + $ export CLUSTER2_CONTEXT= + ``` - ```yaml - global: - image: "hashicorp/consul:1.13.1" - peering: +1. To establish cluster peering through Kubernetes, create a `values.yaml` file with the following Helm values. + + With these values, + the servers in each cluster will be exposed over a Kubernetes Load balancer service. This service can be customized + using [`server.exposeService`](/docs/k8s/helm#v-server-exposeservice). + + When generating a peering token from one of the clusters, Consul uses the address(es) of the load balancer in the peering token so that the peering stream goes through the load balancer in front of the servers. For customizing the addresses used in the peering token, refer to [`global.peering.tokenGeneration`](/docs/k8s/helm#v-global-peering-tokengeneration). + + + + ```yaml + global: + image: "hashicorp/consul:1.13.1" + peering: + enabled: true + connectInject: enabled: true - connectInject: - enabled: true - controller: - enabled: true - meshGateway: - enabled: true - replicas: 1 - ``` + dns: + enabled: true + enableRedirection: true + server: + exposeService: + enabeld: true + controller: + enabled: true + meshGateway: + enabled: true + replicas: 1 + ``` - + + +### Install Consul on Kubernetes -Install Consul on Kubernetes on each Kubernetes cluster by applying `values.yaml` using the Helm CLI. With these values, -the servers in each cluster will be exposed over a Kubernetes Load balancer service. This service can be customized -using [`server.exposeService`](/docs/k8s/helm#v-server-exposeservice). When generating a peering token from one of the -clusters, the address(es) of the load balancer will be used in the peering token, so the peering stream will go through -the load balancer in front of the servers. For customizing the addresses used in the peering token, see -[`global.peering.tokenGeneration`](/docs/k8s/helm#v-global-peering-tokengeneration). +1. Install Consul on Kubernetes on each Kubernetes cluster by applying `values.yaml` using the Helm CLI. + + 1. Install Consul on Kubernetes on `cluster-01` + + ```shell-session + $ export HELM_RELEASE_NAME=cluster-01 + ``` -```shell-session -$ export HELM_RELEASE_NAME=cluster-name -``` + ```shell-session + $ helm install ${HELM_RELEASE_NAME} hashicorp/consul --create-namespace --namespace consul --version "0.47.1" --values values.yaml --kube-context $CLUSTER1_CONTEXT + ``` + 1. Install Consul on Kubernetes on `cluster-02` + + ```shell-session + $ export HELM_RELEASE_NAME=cluster-02 + ``` -```shell-session -$ helm install ${HELM_RELEASE_NAME} hashicorp/consul --version "0.47.1" --values values.yaml -``` + ```shell-session + $ helm install ${HELM_RELEASE_NAME} hashicorp/consul --create-namespace --namespace consul --version "0.47.1" --values values.yaml --kube-context $CLUSTER2_CONTEXT + ``` ## Create a peering token -To peer Kubernetes clusters running Consul, you need to create a peering token and share it with the other cluster. +To peer Kubernetes clusters running Consul, you need to create a peering token and share it with the other cluster. As part of the peering process, the peer names for each respective cluster within the peering are established by using the `metadata.name` values for the `PeeringAcceptor` and `PeeringDialer` CRDs. 1. In `cluster-01`, create the `PeeringAcceptor` custom resource. - + ```yaml apiVersion: consul.hashicorp.com/v1alpha1 @@ -88,13 +120,13 @@ To peer Kubernetes clusters running Consul, you need to create a peering token a 1. Apply the `PeeringAcceptor` resource to the first cluster. ```shell-session - $ kubectl apply --filename acceptor.yml + $ kubectl --context $CLUSTER1_CONTEXT apply --filename acceptor.yaml ```` 1. Save your peering token so that you can export it to the other cluster. ```shell-session - $ kubectl get secret peering-token --output yaml > peering-token.yml + $ kubectl --context $CLUSTER1_CONTEXT get secret peering-token --output yaml > peering-token.yaml ``` ## Establish a peering connection between clusters @@ -102,12 +134,12 @@ To peer Kubernetes clusters running Consul, you need to create a peering token a 1. Apply the peering token to the second cluster. ```shell-session - $ kubectl apply --filename peering-token.yml + $ kubectl --context $CLUSTER2_CONTEXT apply --filename peering-token.yaml ``` -1. In `cluster-02`, create the `PeeringDialer` custom resource. +1. In `cluster-02`, create the `PeeringDialer` custom resource. - + ```yaml apiVersion: consul.hashicorp.com/v1alpha1 @@ -127,27 +159,74 @@ To peer Kubernetes clusters running Consul, you need to create a peering token a 1. Apply the `PeeringDialer` resource to the second cluster. ```shell-session - $ kubectl apply --filename dialer.yml + $ kubectl --context $CLUSTER2_CONTEXT apply --filename dialer.yaml ``` ## Export services between clusters 1. For the service in "cluster-02" that you want to export, add the following [annotation](/docs/k8s/annotations-and-labels) to your service's pods. - + ```yaml - ##… - annotations: - "consul.hashicorp.com/connect-inject": "true" - ##… + # Service to expose backend + apiVersion: v1 + kind: Service + metadata: + name: backend-service + spec: + selector: + app: backend + ports: + - name: http + protocol: TCP + port: 80 + targetPort: 9090 + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: backend + --- + # deployment for backend + apiVersion: apps/v1 + kind: Deployment + metadata: + name: backend + labels: + app: backend + spec: + replicas: 1 + selector: + matchLabels: + app: backend + template: + metadata: + labels: + app: backend + annotations: + "consul.hashicorp.com/connect-inject": "true" + spec: + serviceAccountName: backend + containers: + - name: backend + image: nicholasjackson/fake-service:v0.22.4 + ports: + - containerPort: 9090 + env: + - name: "LISTEN_ADDR" + value: "0.0.0.0:9090" + - name: "NAME" + value: "backend" + - name: "MESSAGE" + value: "Response from backend" ``` 1. In `cluster-02`, create an `ExportedServices` custom resource. - + ```yaml apiVersion: consul.hashicorp.com/v1alpha1 @@ -166,7 +245,7 @@ To peer Kubernetes clusters running Consul, you need to create a peering token a 1. Apply the service file and the `ExportedServices` resource for the second cluster. ```shell-session - $ kubectl apply --filename backend-service.yml --filename exportedsvc.yml + $ kubectl apply --context $CLUSTER2_CONTEXT --filename backend-service.yaml --filename exportedsvc.yaml ``` ## Authorize services for peers @@ -195,18 +274,71 @@ To peer Kubernetes clusters running Consul, you need to create a peering token a 1. Apply the intentions to the second cluster. ```shell-session - $ kubectl apply --filename intention.yml + $ kubectl --context $CLUSTER2_CONTEXT apply --filename intention.yml ``` -1. For the services in `cluster-01` that you want to access the "backend-service," add the following annotations to the service file. +1. For the services in `cluster-01` that you want to access the "backend-service," add the following annotations to the service file. To dial the upstream service from an application, ensure that the requests are sent to the correct DNS name as specified in [Service Virtual IP Lookups](/docs/discovery/dns#service-virtual-ip-lookups). - + ```yaml - ##… - annotations: - "consul.hashicorp.com/connect-inject": "true" - ##… + # Service to expose frontend + apiVersion: v1 + kind: Service + metadata: + name: frontend-service + spec: + selector: + app: frontend + ports: + - name: http + protocol: TCP + port: 9090 + targetPort: 9090 + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: frontend + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: frontend + labels: + app: frontend + spec: + replicas: 1 + selector: + matchLabels: + app: frontend + template: + metadata: + labels: + app: frontend + annotations: + "consul.hashicorp.com/connect-inject": "true" + spec: + serviceAccountName: frontend + containers: + - name: frontend + image: nicholasjackson/fake-service:v0.22.4 + securityContext: + capabilities: + add: ["NET_ADMIN"] + ports: + - containerPort: 9090 + env: + - name: "LISTEN_ADDR" + value: "0.0.0.0:9090" + - name: "UPSTREAM_URIS" + value: "http://backend-service.virtual.cluster-02.consul" + - name: "NAME" + value: "frontend" + - name: "MESSAGE" + value: "Hello World" + - name: "HTTP_CLIENT_KEEP_ALIVES" + value: "false" ``` @@ -214,18 +346,45 @@ To peer Kubernetes clusters running Consul, you need to create a peering token a 1. Apply the service file to the first cluster. ```shell-session - $ kubectl apply --filename frontend-service.yml + $ kubectl --context $CLUSTER1_CONTEXT apply --filename frontend-service.yaml ``` 1. Run the following command in `frontend-service` and check the output to confirm that you peered your clusters successfully. ```shell-session - $ kubectl exec -it $(kubectl get pod -l app=frontend -o name) -- curl localhost:1234 + $ kubectl --context $CLUSTER1_CONTEXT exec -it $(kubectl --context $CLUSTER1_CONTEXT get pod -l app=frontend -o name) -- curl localhost:9090 { - "name": "backend-service", - ##… - "body": "Response from backend", - "code": 200 + "name": "frontend", + "uri": "/", + "type": "HTTP", + "ip_addresses": [ + "10.16.2.11" + ], + "start_time": "2022-08-26T23:40:01.167199", + "end_time": "2022-08-26T23:40:01.226951", + "duration": "59.752279ms", + "body": "Hello World", + "upstream_calls": { + "http://backend-service.virtual.cluster-02.consul": { + "name": "backend", + "uri": "http://backend-service.virtual.cluster-02.consul", + "type": "HTTP", + "ip_addresses": [ + "10.32.2.10" + ], + "start_time": "2022-08-26T23:40:01.223503", + "end_time": "2022-08-26T23:40:01.224653", + "duration": "1.149666ms", + "headers": { + "Content-Length": "266", + "Content-Type": "text/plain; charset=utf-8", + "Date": "Fri, 26 Aug 2022 23:40:01 GMT" + }, + "body": "Response from backend", + "code": 200 + } + }, + "code": 200 } ``` From 74ddf040dd1776aabcf5b4e81ced2dea89a39b29 Mon Sep 17 00:00:00 2001 From: "Chris S. Kim" Date: Mon, 29 Aug 2022 16:32:26 -0400 Subject: [PATCH 29/31] Add heartbeat timeout grace period when accounting for peering health --- agent/consul/leader_peering.go | 4 +- agent/consul/leader_peering_test.go | 2 +- agent/consul/server.go | 8 +- .../services/peerstream/server.go | 7 +- .../services/peerstream/stream_test.go | 17 ++-- .../services/peerstream/stream_tracker.go | 81 +++++++++-------- .../peerstream/stream_tracker_test.go | 87 +++++++++++++------ 7 files changed, 122 insertions(+), 84 deletions(-) diff --git a/agent/consul/leader_peering.go b/agent/consul/leader_peering.go index 556f1b5bfc..d80038397b 100644 --- a/agent/consul/leader_peering.go +++ b/agent/consul/leader_peering.go @@ -112,7 +112,7 @@ func (s *Server) emitPeeringMetricsOnce(logger hclog.Logger, metricsImpl *metric if status.NeverConnected { metricsImpl.SetGaugeWithLabels(leaderHealthyPeeringKey, float32(math.NaN()), labels) } else { - healthy := status.IsHealthy() + healthy := s.peerStreamServer.Tracker.IsHealthy(status) healthyInt := 0 if healthy { healthyInt = 1 @@ -305,7 +305,7 @@ func (s *Server) establishStream(ctx context.Context, logger hclog.Logger, ws me logger.Trace("establishing stream to peer") - streamStatus, err := s.peerStreamTracker.Register(peer.ID) + streamStatus, err := s.peerStreamServer.Tracker.Register(peer.ID) if err != nil { return fmt.Errorf("failed to register stream: %v", err) } diff --git a/agent/consul/leader_peering_test.go b/agent/consul/leader_peering_test.go index b8b5166d8f..61f060802a 100644 --- a/agent/consul/leader_peering_test.go +++ b/agent/consul/leader_peering_test.go @@ -1216,7 +1216,7 @@ func TestLeader_Peering_NoEstablishmentWhenPeeringDisabled(t *testing.T) { })) require.Never(t, func() bool { - _, found := s1.peerStreamTracker.StreamStatus(peerID) + _, found := s1.peerStreamServer.StreamStatus(peerID) return found }, 7*time.Second, 1*time.Second, "peering should not have been established") } diff --git a/agent/consul/server.go b/agent/consul/server.go index f92a03ccd2..94048d06f2 100644 --- a/agent/consul/server.go +++ b/agent/consul/server.go @@ -370,9 +370,9 @@ type Server struct { // peerStreamServer is a server used to handle peering streams from external clusters. peerStreamServer *peerstream.Server + // peeringServer handles peering RPC requests internal to this cluster, like generating peering tokens. - peeringServer *peering.Server - peerStreamTracker *peerstream.Tracker + peeringServer *peering.Server // embedded struct to hold all the enterprise specific data EnterpriseServer @@ -724,11 +724,9 @@ func NewServer(config *Config, flat Deps, externalGRPCServer *grpc.Server) (*Ser Logger: logger.Named("grpc-api.server-discovery"), }).Register(s.externalGRPCServer) - s.peerStreamTracker = peerstream.NewTracker() s.peeringBackend = NewPeeringBackend(s) s.peerStreamServer = peerstream.NewServer(peerstream.Config{ Backend: s.peeringBackend, - Tracker: s.peerStreamTracker, GetStore: func() peerstream.StateStore { return s.FSM().State() }, Logger: logger.Named("grpc-api.peerstream"), ACLResolver: s.ACLResolver, @@ -790,7 +788,7 @@ func newGRPCHandlerFromConfig(deps Deps, config *Config, s *Server) connHandler p := peering.NewServer(peering.Config{ Backend: s.peeringBackend, - Tracker: s.peerStreamTracker, + Tracker: s.peerStreamServer.Tracker, Logger: deps.Logger.Named("grpc-api.peering"), ForwardRPC: func(info structs.RPCInfo, fn func(*grpc.ClientConn) error) (bool, error) { // Only forward the request if the dc in the request matches the server's datacenter. diff --git a/agent/grpc-external/services/peerstream/server.go b/agent/grpc-external/services/peerstream/server.go index 7254c60c7c..17388f4a25 100644 --- a/agent/grpc-external/services/peerstream/server.go +++ b/agent/grpc-external/services/peerstream/server.go @@ -26,11 +26,12 @@ const ( type Server struct { Config + + Tracker *Tracker } type Config struct { Backend Backend - Tracker *Tracker GetStore func() StateStore Logger hclog.Logger ForwardRPC func(structs.RPCInfo, func(*grpc.ClientConn) error) (bool, error) @@ -53,7 +54,6 @@ type ACLResolver interface { func NewServer(cfg Config) *Server { requireNotNil(cfg.Backend, "Backend") - requireNotNil(cfg.Tracker, "Tracker") requireNotNil(cfg.GetStore, "GetStore") requireNotNil(cfg.Logger, "Logger") // requireNotNil(cfg.ACLResolver, "ACLResolver") // TODO(peering): reenable check when ACLs are required @@ -67,7 +67,8 @@ func NewServer(cfg Config) *Server { cfg.incomingHeartbeatTimeout = defaultIncomingHeartbeatTimeout } return &Server{ - Config: cfg, + Config: cfg, + Tracker: NewTracker(cfg.incomingHeartbeatTimeout), } } diff --git a/agent/grpc-external/services/peerstream/stream_test.go b/agent/grpc-external/services/peerstream/stream_test.go index bee02592a3..9116d7a313 100644 --- a/agent/grpc-external/services/peerstream/stream_test.go +++ b/agent/grpc-external/services/peerstream/stream_test.go @@ -499,9 +499,8 @@ func TestStreamResources_Server_Terminate(t *testing.T) { base: time.Date(2000, time.January, 1, 0, 0, 0, 0, time.UTC), } - srv, store := newTestServer(t, func(c *Config) { - c.Tracker.SetClock(it.Now) - }) + srv, store := newTestServer(t, nil) + srv.Tracker.setClock(it.Now) p := writePeeringToBeDialed(t, store, 1, "my-peer") require.Empty(t, p.PeerID, "should be empty if being dialed") @@ -552,9 +551,8 @@ func TestStreamResources_Server_StreamTracker(t *testing.T) { base: time.Date(2000, time.January, 1, 0, 0, 0, 0, time.UTC), } - srv, store := newTestServer(t, func(c *Config) { - c.Tracker.SetClock(it.Now) - }) + srv, store := newTestServer(t, nil) + srv.Tracker.setClock(it.Now) // Set the initial roots and CA configuration. _, rootA := writeInitialRootsAndCA(t, store) @@ -1128,9 +1126,9 @@ func TestStreamResources_Server_DisconnectsOnHeartbeatTimeout(t *testing.T) { } srv, store := newTestServer(t, func(c *Config) { - c.Tracker.SetClock(it.Now) c.incomingHeartbeatTimeout = 5 * time.Millisecond }) + srv.Tracker.setClock(it.Now) p := writePeeringToBeDialed(t, store, 1, "my-peer") require.Empty(t, p.PeerID, "should be empty if being dialed") @@ -1176,9 +1174,9 @@ func TestStreamResources_Server_SendsHeartbeats(t *testing.T) { outgoingHeartbeatInterval := 5 * time.Millisecond srv, store := newTestServer(t, func(c *Config) { - c.Tracker.SetClock(it.Now) c.outgoingHeartbeatInterval = outgoingHeartbeatInterval }) + srv.Tracker.setClock(it.Now) p := writePeeringToBeDialed(t, store, 1, "my-peer") require.Empty(t, p.PeerID, "should be empty if being dialed") @@ -1235,9 +1233,9 @@ func TestStreamResources_Server_KeepsConnectionOpenWithHeartbeat(t *testing.T) { incomingHeartbeatTimeout := 10 * time.Millisecond srv, store := newTestServer(t, func(c *Config) { - c.Tracker.SetClock(it.Now) c.incomingHeartbeatTimeout = incomingHeartbeatTimeout }) + srv.Tracker.setClock(it.Now) p := writePeeringToBeDialed(t, store, 1, "my-peer") require.Empty(t, p.PeerID, "should be empty if being dialed") @@ -2746,7 +2744,6 @@ func newTestServer(t *testing.T, configFn func(c *Config)) (*testServer, *state. store: store, pub: publisher, }, - Tracker: NewTracker(), GetStore: func() StateStore { return store }, Logger: testutil.Logger(t), Datacenter: "dc1", diff --git a/agent/grpc-external/services/peerstream/stream_tracker.go b/agent/grpc-external/services/peerstream/stream_tracker.go index d0dcd39770..c3108e71e7 100644 --- a/agent/grpc-external/services/peerstream/stream_tracker.go +++ b/agent/grpc-external/services/peerstream/stream_tracker.go @@ -14,18 +14,27 @@ type Tracker struct { mu sync.RWMutex streams map[string]*MutableStatus + // heartbeatTimeout is the max duration a connection is allowed to be + // disconnected before the stream health is reported as non-healthy + heartbeatTimeout time.Duration + // timeNow is a shim for testing. timeNow func() time.Time } -func NewTracker() *Tracker { +func NewTracker(heartbeatTimeout time.Duration) *Tracker { + if heartbeatTimeout == 0 { + heartbeatTimeout = defaultIncomingHeartbeatTimeout + } return &Tracker{ - streams: make(map[string]*MutableStatus), - timeNow: time.Now, + streams: make(map[string]*MutableStatus), + timeNow: time.Now, + heartbeatTimeout: heartbeatTimeout, } } -func (t *Tracker) SetClock(clock func() time.Time) { +// setClock is used for debugging purposes only. +func (t *Tracker) setClock(clock func() time.Time) { if clock == nil { t.timeNow = time.Now } else { @@ -128,6 +137,39 @@ func (t *Tracker) DeleteStatus(id string) { delete(t.streams, id) } +// IsHealthy is a calculates the health of a peering status. +// We define a peering as unhealthy if its status has been in the following +// states for longer than the configured incomingHeartbeatTimeout. +// - If it is disconnected +// - If the last received Nack is newer than last received Ack +// - If the last received error is newer than last received success +// +// If none of these conditions apply, we call the peering healthy. +func (t *Tracker) IsHealthy(s Status) bool { + // If stream is in a disconnected state for longer than the configured + // heartbeat timeout, report as unhealthy. + if !s.DisconnectTime.IsZero() && + t.timeNow().Sub(s.DisconnectTime) > t.heartbeatTimeout { + return false + } + + // If last Nack is after last Ack, it means the peer is unable to + // handle our replication message. + if s.LastNack.After(s.LastAck) && + t.timeNow().Sub(s.LastAck) > t.heartbeatTimeout { + return false + } + + // If last recv error is newer than last recv success, we were unable + // to handle the peer's replication message. + if s.LastRecvError.After(s.LastRecvResourceSuccess) && + t.timeNow().Sub(s.LastRecvError) > t.heartbeatTimeout { + return false + } + + return true +} + type MutableStatus struct { mu sync.RWMutex @@ -201,37 +243,6 @@ func (s *Status) GetExportedServicesCount() uint64 { return uint64(len(s.ExportedServices)) } -// IsHealthy is a convenience func that returns true/ false for a peering status. -// We define a peering as unhealthy if its status satisfies one of the following: -// - If it is disconnected -// - If the last received Nack is newer than last received Ack -// - If the last received error is newer than last received success -// If none of these conditions apply, we call the peering healthy. -func (s *Status) IsHealthy() bool { - if !s.Connected { - return false - } - - // If stream is in a disconnected state, report unhealthy. - // This should be logically equivalent to s.Connected above. - if !s.DisconnectTime.IsZero() { - return false - } - - // If last Nack is after last Ack, it means the peer is unable to - // handle our replication messages. - if s.LastNack.After(s.LastAck) { - return false - } - - // If last recv error is newer than last recv success - report unhealthy - if s.LastRecvError.After(s.LastRecvResourceSuccess) { - return false - } - - return true -} - func newMutableStatus(now func() time.Time, connected bool) *MutableStatus { return &MutableStatus{ Status: Status{ diff --git a/agent/grpc-external/services/peerstream/stream_tracker_test.go b/agent/grpc-external/services/peerstream/stream_tracker_test.go index 555e76258b..bb018b4b46 100644 --- a/agent/grpc-external/services/peerstream/stream_tracker_test.go +++ b/agent/grpc-external/services/peerstream/stream_tracker_test.go @@ -5,6 +5,7 @@ import ( "testing" "time" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/hashicorp/consul/sdk/testutil" @@ -14,30 +15,35 @@ const ( aPeerID = "63b60245-c475-426b-b314-4588d210859d" ) -func TestStatus_IsHealthy(t *testing.T) { +func TestTracker_IsHealthy(t *testing.T) { type testcase struct { name string - dontConnect bool + tracker *Tracker modifierFunc func(status *MutableStatus) expectedVal bool } tcs := []testcase{ { - name: "never connected, unhealthy", - expectedVal: false, - dontConnect: true, - }, - { - name: "disconnect time not zero", - expectedVal: false, + name: "disconnect time within timeout", + tracker: NewTracker(defaultIncomingHeartbeatTimeout), + expectedVal: true, modifierFunc: func(status *MutableStatus) { status.DisconnectTime = time.Now() }, }, { - name: "receive error before receive success", + name: "disconnect time past timeout", + tracker: NewTracker(1 * time.Millisecond), expectedVal: false, + modifierFunc: func(status *MutableStatus) { + status.DisconnectTime = time.Now().Add(-1 * time.Minute) + }, + }, + { + name: "receive error before receive success within timeout", + tracker: NewTracker(defaultIncomingHeartbeatTimeout), + expectedVal: true, modifierFunc: func(status *MutableStatus) { now := time.Now() status.LastRecvResourceSuccess = now @@ -45,46 +51,71 @@ func TestStatus_IsHealthy(t *testing.T) { }, }, { - name: "nack before ack", + name: "receive error before receive success within timeout", + tracker: NewTracker(defaultIncomingHeartbeatTimeout), + expectedVal: true, + modifierFunc: func(status *MutableStatus) { + now := time.Now() + status.LastRecvResourceSuccess = now + status.LastRecvError = now.Add(1 * time.Second) + }, + }, + { + name: "receive error before receive success past timeout", + tracker: NewTracker(1 * time.Millisecond), expectedVal: false, + modifierFunc: func(status *MutableStatus) { + now := time.Now().Add(-2 * time.Second) + status.LastRecvResourceSuccess = now + status.LastRecvError = now.Add(1 * time.Second) + }, + }, + { + name: "nack before ack within timeout", + tracker: NewTracker(defaultIncomingHeartbeatTimeout), + expectedVal: true, modifierFunc: func(status *MutableStatus) { now := time.Now() status.LastAck = now status.LastNack = now.Add(1 * time.Second) }, }, + { + name: "nack before ack past timeout", + tracker: NewTracker(1 * time.Millisecond), + expectedVal: false, + modifierFunc: func(status *MutableStatus) { + now := time.Now().Add(-2 * time.Second) + status.LastAck = now + status.LastNack = now.Add(1 * time.Second) + }, + }, { name: "healthy", + tracker: NewTracker(defaultIncomingHeartbeatTimeout), expectedVal: true, }, } for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { - tracker := NewTracker() + tracker := tc.tracker - if !tc.dontConnect { - st, err := tracker.Connected(aPeerID) - require.NoError(t, err) - require.True(t, st.Connected) + st, err := tracker.Connected(aPeerID) + require.NoError(t, err) + require.True(t, st.Connected) - if tc.modifierFunc != nil { - tc.modifierFunc(st) - } - - require.Equal(t, tc.expectedVal, st.IsHealthy()) - - } else { - st, found := tracker.StreamStatus(aPeerID) - require.False(t, found) - require.Equal(t, tc.expectedVal, st.IsHealthy()) + if tc.modifierFunc != nil { + tc.modifierFunc(st) } + + assert.Equal(t, tc.expectedVal, tracker.IsHealthy(st.GetStatus())) }) } } func TestTracker_EnsureConnectedDisconnected(t *testing.T) { - tracker := NewTracker() + tracker := NewTracker(defaultIncomingHeartbeatTimeout) peerID := "63b60245-c475-426b-b314-4588d210859d" it := incrementalTime{ @@ -181,7 +212,7 @@ func TestTracker_connectedStreams(t *testing.T) { } run := func(t *testing.T, tc testCase) { - tracker := NewTracker() + tracker := NewTracker(defaultIncomingHeartbeatTimeout) if tc.setup != nil { tc.setup(t, tracker) } From 77918d9dea0830aae2261925a1bb727a4446f36c Mon Sep 17 00:00:00 2001 From: Eric Haberkorn Date: Mon, 29 Aug 2022 16:59:27 -0400 Subject: [PATCH 30/31] Fix a breaking change to the API package introduced in #13835 (#14378) `QueryDatacenterOptions` was renamed to `QueryFailoverOptions` without creating an alias. This adds `QueryDatacenterOptions` back as an alias to `QueryFailoverOptions` and marks it is deprecated. --- .changelog/14378.txt | 5 +++++ api/prepared_query.go | 3 +++ 2 files changed, 8 insertions(+) create mode 100644 .changelog/14378.txt diff --git a/.changelog/14378.txt b/.changelog/14378.txt new file mode 100644 index 0000000000..2ab1b8f138 --- /dev/null +++ b/.changelog/14378.txt @@ -0,0 +1,5 @@ +```release-note:bug +api: Fix a breaking change caused by renaming `QueryDatacenterOptions` to +`QueryFailoverOptions`. This adds `QueryDatacenterOptions` back as an alias to +`QueryFailoverOptions` and marks it as deprecated. +``` diff --git a/api/prepared_query.go b/api/prepared_query.go index 60cd437cb7..7e0518f580 100644 --- a/api/prepared_query.go +++ b/api/prepared_query.go @@ -17,6 +17,9 @@ type QueryFailoverOptions struct { Targets []QueryFailoverTarget } +// Deprecated: use QueryFailoverOptions instead. +type QueryDatacenterOptions = QueryFailoverOptions + type QueryFailoverTarget struct { // PeerName specifies a peer to try during failover. PeerName string From 70bb6a2abdbc5ed4a6e728e8da243c5394a631d1 Mon Sep 17 00:00:00 2001 From: Luke Kysow <1034429+lkysow@users.noreply.github.com> Date: Mon, 29 Aug 2022 16:13:49 -0700 Subject: [PATCH 31/31] Run integration tests locally using amd64 (#14365) Locally, always run integration tests using amd64, even if running on an arm mac. This ensures the architecture locally always matches the CI/CD environment. In addition: * Use consul:local for envoy integration and upgrade tests. Previously, consul:local was used for upgrade tests and consul-dev for integration tests. I didn't see a reason to use separate images as it's more confusing. * By default, disable the requirement that aws credentials are set. These are only needed for the lambda tests and make it so you can't run any tests locally, even if you're not running the lambda tests. Now they'll only run if the LAMBDA_TESTS_ENABLED env var is set. * Split out the building of the Docker image for integration tests into its own target from `dev-docker`. This allows us to always use an amd64 image without messing up the `dev-docker` target. * Add support for passing GO_TEST_FLAGs to `test-envoy-integ` target. * Add a wait_for_leader function because tests were failing locally without it. --- .circleci/config.yml | 7 +++-- GNUmakefile | 18 +++++++++-- .../connect/envoy/Dockerfile-consul-envoy | 2 +- .../envoy/case-wanfed-gw/global-setup.sh | 2 +- test/integration/connect/envoy/helpers.bash | 13 ++++++-- test/integration/connect/envoy/run-tests.sh | 30 +++++++++++-------- 6 files changed, 49 insertions(+), 23 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 105666c661..053d50d256 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -816,7 +816,7 @@ jobs: # Get go binary from workspace - attach_workspace: at: . - # Build the consul-dev image from the already built binary + # Build the consul:local image from the already built binary - run: command: | sudo rm -rf /usr/local/go @@ -887,8 +887,8 @@ jobs: - attach_workspace: at: . - run: *install-gotestsum - # Build the consul-dev image from the already built binary - - run: docker build -t consul-dev -f ./build-support/docker/Consul-Dev.dockerfile . + # Build the consul:local image from the already built binary + - run: docker build -t consul:local -f ./build-support/docker/Consul-Dev.dockerfile . - run: name: Envoy Integration Tests command: | @@ -902,6 +902,7 @@ jobs: GOTESTSUM_JUNITFILE: /tmp/test-results/results.xml GOTESTSUM_FORMAT: standard-verbose COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" # tput complains if this isn't set to something. TERM: ansi - store_artifacts: diff --git a/GNUmakefile b/GNUmakefile index 77d6a7ec21..f9dd160811 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -130,7 +130,7 @@ export GOLDFLAGS # Allow skipping docker build during integration tests in CI since we already # have a built binary -ENVOY_INTEG_DEPS?=dev-docker +ENVOY_INTEG_DEPS?=docker-envoy-integ ifdef SKIP_DOCKER_BUILD ENVOY_INTEG_DEPS=noop endif @@ -346,8 +346,22 @@ consul-docker: go-build-image ui-docker: ui-build-image @$(SHELL) $(CURDIR)/build-support/scripts/build-docker.sh ui +# Build image used to run integration tests locally. +docker-envoy-integ: + $(MAKE) GOARCH=amd64 linux + docker build \ + --platform linux/amd64 $(NOCACHE) $(QUIET) \ + -t 'consul:local' \ + --build-arg CONSUL_IMAGE_VERSION=$(CONSUL_IMAGE_VERSION) \ + $(CURDIR)/pkg/bin/linux_amd64 \ + -f $(CURDIR)/build-support/docker/Consul-Dev.dockerfile + +# Run integration tests. +# Use GO_TEST_FLAGS to run specific tests: +# make test-envoy-integ GO_TEST_FLAGS="-run TestEnvoy/case-basic" +# NOTE: Always uses amd64 images, even when running on M1 macs, to match CI/CD environment. test-envoy-integ: $(ENVOY_INTEG_DEPS) - @go test -v -timeout=30m -tags integration ./test/integration/connect/envoy + @go test -v -timeout=30m -tags integration $(GO_TEST_FLAGS) ./test/integration/connect/envoy .PHONY: test-compat-integ test-compat-integ: dev-docker diff --git a/test/integration/connect/envoy/Dockerfile-consul-envoy b/test/integration/connect/envoy/Dockerfile-consul-envoy index b6d5b3e8e2..41941a3367 100644 --- a/test/integration/connect/envoy/Dockerfile-consul-envoy +++ b/test/integration/connect/envoy/Dockerfile-consul-envoy @@ -1,7 +1,7 @@ # Note this arg has to be before the first FROM ARG ENVOY_VERSION -FROM consul-dev as consul +FROM consul:local as consul FROM docker.mirror.hashicorp.services/envoyproxy/envoy:v${ENVOY_VERSION} COPY --from=consul /bin/consul /bin/consul diff --git a/test/integration/connect/envoy/case-wanfed-gw/global-setup.sh b/test/integration/connect/envoy/case-wanfed-gw/global-setup.sh index fee985addd..3e61204459 100755 --- a/test/integration/connect/envoy/case-wanfed-gw/global-setup.sh +++ b/test/integration/connect/envoy/case-wanfed-gw/global-setup.sh @@ -17,7 +17,7 @@ consul tls cert create -dc=secondary -server -node=sec " docker rm -f "$container" &>/dev/null || true -docker run -i --net=none --name="$container" consul-dev:latest sh -c "${scriptlet}" +docker run -i --net=none --name="$container" consul:local sh -c "${scriptlet}" # primary for f in \ diff --git a/test/integration/connect/envoy/helpers.bash b/test/integration/connect/envoy/helpers.bash index 2fd9be7e38..d7fe0ae024 100755 --- a/test/integration/connect/envoy/helpers.bash +++ b/test/integration/connect/envoy/helpers.bash @@ -562,14 +562,14 @@ function assert_intention_denied { function docker_consul { local DC=$1 shift 1 - docker run -i --rm --network container:envoy_consul-${DC}_1 consul-dev "$@" + docker run -i --rm --network container:envoy_consul-${DC}_1 consul:local "$@" } function docker_consul_for_proxy_bootstrap { local DC=$1 shift 1 - docker run -i --rm --network container:envoy_consul-${DC}_1 consul-dev "$@" + docker run -i --rm --network container:envoy_consul-${DC}_1 consul:local "$@" 2> /dev/null } function docker_wget { @@ -581,7 +581,7 @@ function docker_wget { function docker_curl { local DC=$1 shift 1 - docker run --rm --network container:envoy_consul-${DC}_1 --entrypoint curl consul-dev "$@" + docker run --rm --network container:envoy_consul-${DC}_1 --entrypoint curl consul:local "$@" } function docker_exec { @@ -806,9 +806,16 @@ function delete_config_entry { function register_services { local DC=${1:-primary} + wait_for_leader "$DC" docker_consul_exec ${DC} sh -c "consul services register /workdir/${DC}/register/service_*.hcl" } +# wait_for_leader waits until a leader is elected. +# Its first argument must be the datacenter name. +function wait_for_leader { + retry_default docker_consul_exec "$1" sh -c '[[ $(curl --fail -sS http://127.0.0.1:8500/v1/status/leader) ]]' +} + function setup_upsert_l4_intention { local SOURCE=$1 local DESTINATION=$2 diff --git a/test/integration/connect/envoy/run-tests.sh b/test/integration/connect/envoy/run-tests.sh index c8c392c34f..f0e6b165cb 100755 --- a/test/integration/connect/envoy/run-tests.sh +++ b/test/integration/connect/envoy/run-tests.sh @@ -16,6 +16,8 @@ ENVOY_VERSION=${ENVOY_VERSION:-"1.23.0"} export ENVOY_VERSION export DOCKER_BUILDKIT=1 +# Always run tests on amd64 because that's what the CI environment uses. +export DOCKER_DEFAULT_PLATFORM="linux/amd64" if [ ! -z "$DEBUG" ] ; then set -x @@ -44,17 +46,19 @@ function network_snippet { } function aws_snippet { - local snippet="" + if [[ ! -z "$LAMBDA_TESTS_ENABLED" ]]; then + local snippet="" - # The Lambda integration cases assume that a Lambda function exists in $AWS_REGION with an ARN of $AWS_LAMBDA_ARN. - # The AWS credentials must have permission to invoke the Lambda function. - [ -n "$(set | grep '^AWS_ACCESS_KEY_ID=')" ] && snippet="${snippet} -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID" - [ -n "$(set | grep '^AWS_SECRET_ACCESS_KEY=')" ] && snippet="${snippet} -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY" - [ -n "$(set | grep '^AWS_SESSION_TOKEN=')" ] && snippet="${snippet} -e AWS_SESSION_TOKEN=$AWS_SESSION_TOKEN" - [ -n "$(set | grep '^AWS_LAMBDA_REGION=')" ] && snippet="${snippet} -e AWS_LAMBDA_REGION=$AWS_LAMBDA_REGION" - [ -n "$(set | grep '^AWS_LAMBDA_ARN=')" ] && snippet="${snippet} -e AWS_LAMBDA_ARN=$AWS_LAMBDA_ARN" + # The Lambda integration cases assume that a Lambda function exists in $AWS_REGION with an ARN of $AWS_LAMBDA_ARN. + # The AWS credentials must have permission to invoke the Lambda function. + [ -n "$(set | grep '^AWS_ACCESS_KEY_ID=')" ] && snippet="${snippet} -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID" + [ -n "$(set | grep '^AWS_SECRET_ACCESS_KEY=')" ] && snippet="${snippet} -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY" + [ -n "$(set | grep '^AWS_SESSION_TOKEN=')" ] && snippet="${snippet} -e AWS_SESSION_TOKEN=$AWS_SESSION_TOKEN" + [ -n "$(set | grep '^AWS_LAMBDA_REGION=')" ] && snippet="${snippet} -e AWS_LAMBDA_REGION=$AWS_LAMBDA_REGION" + [ -n "$(set | grep '^AWS_LAMBDA_ARN=')" ] && snippet="${snippet} -e AWS_LAMBDA_ARN=$AWS_LAMBDA_ARN" - echo "$snippet" + echo "$snippet" + fi } function init_workdir { @@ -222,7 +226,7 @@ function start_consul { --hostname "consul-${DC}-server" \ --network-alias "consul-${DC}-server" \ -e "CONSUL_LICENSE=$license" \ - consul-dev \ + consul:local \ agent -dev -datacenter "${DC}" \ -config-dir "/workdir/${DC}/consul" \ -config-dir "/workdir/${DC}/consul-server" \ @@ -237,7 +241,7 @@ function start_consul { --network-alias "consul-${DC}-client" \ -e "CONSUL_LICENSE=$license" \ ${ports[@]} \ - consul-dev \ + consul:local \ agent -datacenter "${DC}" \ -config-dir "/workdir/${DC}/consul" \ -data-dir "/tmp/consul" \ @@ -256,7 +260,7 @@ function start_consul { --network-alias "consul-${DC}-server" \ -e "CONSUL_LICENSE=$license" \ ${ports[@]} \ - consul-dev \ + consul:local \ agent -dev -datacenter "${DC}" \ -config-dir "/workdir/${DC}/consul" \ -config-dir "/workdir/${DC}/consul-server" \ @@ -290,7 +294,7 @@ function start_partitioned_client { --hostname "consul-${PARTITION}-client" \ --network-alias "consul-${PARTITION}-client" \ -e "CONSUL_LICENSE=$license" \ - consul-dev agent \ + consul:local agent \ -datacenter "primary" \ -retry-join "consul-primary-server" \ -grpc-port 8502 \