diff --git a/.changelog/14233.txt b/.changelog/14233.txt new file mode 100644 index 0000000000..5a2c6dee18 --- /dev/null +++ b/.changelog/14233.txt @@ -0,0 +1,3 @@ +```release-note:bugfix +rpc: Adds max jitter to client deadlines to prevent i/o deadline errors on blocking queries +``` diff --git a/agent/consul/client_test.go b/agent/consul/client_test.go index 84135ee184..32199d8aba 100644 --- a/agent/consul/client_test.go +++ b/agent/consul/client_test.go @@ -893,8 +893,8 @@ func TestClient_RPC_Timeout(t *testing.T) { } }) - // waiter will sleep for 50ms - require.NoError(t, s1.RegisterEndpoint("Wait", &waiter{duration: 50 * time.Millisecond})) + // waiter will sleep for 101ms which is 1ms more than the DefaultQueryTime + require.NoError(t, s1.RegisterEndpoint("Wait", &waiter{duration: 101 * time.Millisecond})) // Requests with QueryOptions have a default timeout of RPCHoldTimeout (10ms) // so we expect the RPC call to timeout. @@ -903,7 +903,8 @@ func TestClient_RPC_Timeout(t *testing.T) { require.Error(t, err) require.Contains(t, err.Error(), "rpc error making call: i/o deadline reached") - // Blocking requests have a longer timeout (100ms) so this should pass + // Blocking requests have a longer timeout (100ms) so this should pass since we + // add the maximum jitter which should be 16ms out = struct{}{} err = c1.RPC("Wait.Wait", &structs.NodeSpecificRequest{ QueryOptions: structs.QueryOptions{ diff --git a/agent/structs/structs.go b/agent/structs/structs.go index 22fb47ca9d..8301688886 100644 --- a/agent/structs/structs.go +++ b/agent/structs/structs.go @@ -353,7 +353,7 @@ func (q QueryOptions) Timeout(rpcHoldTimeout, maxQueryTime, defaultQueryTime tim q.MaxQueryTime = defaultQueryTime } // Timeout after maximum jitter has elapsed. - q.MaxQueryTime += lib.RandomStagger(q.MaxQueryTime / JitterFraction) + q.MaxQueryTime += q.MaxQueryTime / JitterFraction return q.MaxQueryTime + rpcHoldTimeout }