Client: add metric for failed RPC calls to server

agent/consul/client.go

@@ -276,6 +276,7 @@ TRY:
 
 	// Move off to another server, and see if we can retry.
 	c.logger.Printf("[ERR] consul: %q RPC failed to server %s: %v", method, server.Addr, rpcErr)
+	metrics.IncrCounter([]string{"client", "rpc", "failed"}, 1)
 	c.routers.NotifyFailedServer(server)
 	if retry := canRetry(args, rpcErr); !retry {
 		return rpcErr

website/source/docs/agent/telemetry.html.md

@@ -75,6 +75,12 @@ These metrics are used to monitor the health of specific Consul agents.
     <td>rejected requests</td>
     <td>counter</td>
   </tr>
+  <tr>
+    <td>`consul.client.rpc.failed`</td>
+    <td>This increments whenever a Consul agent in client mode makes an RPC request to a Consul server and fails.</td>
+    <td>requests</td>
+    <td>counter</td>
+  </tr>
   <tr>
     <td>`consul.client.api.catalog_register.<node>`</td>
     <td>This increments whenever a Consul agent receives a catalog register request.</td>