From e85e63c18cdea1263bea9ab1c9b627eef85e1ac6 Mon Sep 17 00:00:00 2001
From: Guido Iaquinti <giaquinti@slack-corp.com>
Date: Wed, 13 Jun 2018 12:35:45 +0100
Subject: [PATCH 1/3] Client: add metric for failed RPC calls to server

---
 agent/consul/client.go                      | 1 +
 website/source/docs/agent/telemetry.html.md | 6 ++++++
 2 files changed, 7 insertions(+)

diff --git a/agent/consul/client.go b/agent/consul/client.go
index a69e76160e..e90daec371 100644
--- a/agent/consul/client.go
+++ b/agent/consul/client.go
@@ -276,6 +276,7 @@ TRY:
 
 	// Move off to another server, and see if we can retry.
 	c.logger.Printf("[ERR] consul: %q RPC failed to server %s: %v", method, server.Addr, rpcErr)
+	metrics.IncrCounter([]string{"client", "rpc", "failed"}, 1)
 	c.routers.NotifyFailedServer(server)
 	if retry := canRetry(args, rpcErr); !retry {
 		return rpcErr
diff --git a/website/source/docs/agent/telemetry.html.md b/website/source/docs/agent/telemetry.html.md
index d0888d29e4..1f8e49020f 100644
--- a/website/source/docs/agent/telemetry.html.md
+++ b/website/source/docs/agent/telemetry.html.md
@@ -75,6 +75,12 @@ These metrics are used to monitor the health of specific Consul agents.
     <td>rejected requests</td>
     <td>counter</td>
   </tr>
+  <tr>
+    <td>`consul.client.rpc.failed`</td>
+    <td>This increments whenever a Consul agent in client mode makes an RPC request to a Consul server and fails.</td>
+    <td>requests</td>
+    <td>counter</td>
+  </tr>
   <tr>
     <td>`consul.client.api.catalog_register.<node>`</td>
     <td>This increments whenever a Consul agent receives a catalog register request.</td>

From 3d230dee8017cc9b168fda51cb5c96f045052933 Mon Sep 17 00:00:00 2001
From: Guido Iaquinti <giaquinti@slack-corp.com>
Date: Wed, 13 Jun 2018 14:53:44 +0100
Subject: [PATCH 2/3] Attach server.ID label to client.rpc.failed

---
 agent/consul/client.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/agent/consul/client.go b/agent/consul/client.go
index e90daec371..f5aa434ee0 100644
--- a/agent/consul/client.go
+++ b/agent/consul/client.go
@@ -276,7 +276,7 @@ TRY:
 
 	// Move off to another server, and see if we can retry.
 	c.logger.Printf("[ERR] consul: %q RPC failed to server %s: %v", method, server.Addr, rpcErr)
-	metrics.IncrCounter([]string{"client", "rpc", "failed"}, 1)
+	metrics.IncrCounterWithLabels([]string{"client", "rpc", "failed"}, 1, []metrics.Label{{Name: "server", Value: server.ID}})
 	c.routers.NotifyFailedServer(server)
 	if retry := canRetry(args, rpcErr); !retry {
 		return rpcErr

From f7fe6c2a87a4497c5860ab2348585778a3a7439e Mon Sep 17 00:00:00 2001
From: Guido Iaquinti <giaquinti@slack-corp.com>
Date: Wed, 13 Jun 2018 14:56:14 +0100
Subject: [PATCH 3/3] Attach server.Name label to client.rpc.failed

---
 agent/consul/client.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/agent/consul/client.go b/agent/consul/client.go
index f5aa434ee0..4dc1f33655 100644
--- a/agent/consul/client.go
+++ b/agent/consul/client.go
@@ -276,7 +276,7 @@ TRY:
 
 	// Move off to another server, and see if we can retry.
 	c.logger.Printf("[ERR] consul: %q RPC failed to server %s: %v", method, server.Addr, rpcErr)
-	metrics.IncrCounterWithLabels([]string{"client", "rpc", "failed"}, 1, []metrics.Label{{Name: "server", Value: server.ID}})
+	metrics.IncrCounterWithLabels([]string{"client", "rpc", "failed"}, 1, []metrics.Label{{Name: "server", Value: server.Name}})
 	c.routers.NotifyFailedServer(server)
 	if retry := canRetry(args, rpcErr); !retry {
 		return rpcErr