From d3e658b0e714ed763c6af19ea3953258d6dce248 Mon Sep 17 00:00:00 2001
From: Jared Kirschner <85913323+jkirschner-hashicorp@users.noreply.github.com>
Date: Wed, 6 Dec 2023 13:21:08 -0500
Subject: [PATCH] improve client RPC metrics consistency (#19721)

The client.rpc metric now excludes internal retries for consistency
with client.rpc.exceeded and client.rpc.failed. All of these metrics
now increment at most once per RPC method call, allowing for
accurate calculation of failure / rate limit application occurrence.

Additionally, if an RPC fails because no servers are present,
client.rpc.failed is now incremented.
---
 .changelog/19721.txt   | 6 ++++++
 agent/consul/client.go | 4 +++-
 2 files changed, 9 insertions(+), 1 deletion(-)
 create mode 100644 .changelog/19721.txt

diff --git a/.changelog/19721.txt b/.changelog/19721.txt
new file mode 100644
index 0000000000..5a6b4bbd37
--- /dev/null
+++ b/.changelog/19721.txt
@@ -0,0 +1,6 @@
+```release-note:improvement
+metrics: modify consul.client.rpc metric to exclude internal retries for consistency with consul.client.rpc.exceeded and consul.client.rpc.failed
+```
+```release-note:improvement
+metrics: increment consul.client.rpc.failed if RPC fails because no servers are accessible
+```
diff --git a/agent/consul/client.go b/agent/consul/client.go
index fa5f1239e1..5216ffe2d5 100644
--- a/agent/consul/client.go
+++ b/agent/consul/client.go
@@ -288,15 +288,17 @@ func (c *Client) RPC(ctx context.Context, method string, args interface{}, reply
 	firstCheck := time.Now()
 	retryCount := 0
 	previousJitter := time.Duration(0)
+
+	metrics.IncrCounter([]string{"client", "rpc"}, 1)
 TRY:
 	retryCount++
 	manager, server := c.router.FindLANRoute()
 	if server == nil {
+		metrics.IncrCounter([]string{"client", "rpc", "failed"}, 1)
 		return structs.ErrNoServers
 	}
 
 	// Enforce the RPC limit.
-	metrics.IncrCounter([]string{"client", "rpc"}, 1)
 	if !c.rpcLimiter.Load().(*rate.Limiter).Allow() {
 		metrics.IncrCounter([]string{"client", "rpc", "exceeded"}, 1)
 		return structs.ErrRPCRateExceeded