From 799b34f1a9746dc3789cfd18306c3fbe505aa776 Mon Sep 17 00:00:00 2001 From: Semir Patel Date: Mon, 19 Dec 2022 16:04:52 -0600 Subject: [PATCH] Map net/rpc endpoints to a read/write/exempt op for rate-limiting (#15825) Also fixed TestRequestRecorder flaky tests due to loss of precision in elapsed time in the test. --- agent/consul/rate/handler.go | 3 + agent/rpc/middleware/interceptors_test.go | 2 +- agent/rpc/middleware/rate_limit_mappings.go | 144 ++++++++++++++++++++ 3 files changed, 148 insertions(+), 1 deletion(-) create mode 100644 agent/rpc/middleware/rate_limit_mappings.go diff --git a/agent/consul/rate/handler.go b/agent/consul/rate/handler.go index de20ab668b..8bfa7b247f 100644 --- a/agent/consul/rate/handler.go +++ b/agent/consul/rate/handler.go @@ -84,6 +84,9 @@ const ( // OperationTypeWrite represents a write operation. OperationTypeWrite + + // OperationTypeExempt represents an operation that is exempt from rate-limiting. + OperationTypeExempt ) // Operation the client is attempting to perform. diff --git a/agent/rpc/middleware/interceptors_test.go b/agent/rpc/middleware/interceptors_test.go index d9676846bc..c47cf17f4b 100644 --- a/agent/rpc/middleware/interceptors_test.go +++ b/agent/rpc/middleware/interceptors_test.go @@ -260,7 +260,7 @@ func TestRequestRecorder(t *testing.T) { o := store.get(key) require.Equal(t, o.key, metricRPCRequest) - require.LessOrEqual(t, o.elapsed, float32(start.Sub(time.Now()).Milliseconds())) + require.LessOrEqual(t, o.elapsed, float32(time.Now().Sub(start).Microseconds())/1000) require.Equal(t, o.labels, tc.expectedLabels) }) diff --git a/agent/rpc/middleware/rate_limit_mappings.go b/agent/rpc/middleware/rate_limit_mappings.go new file mode 100644 index 0000000000..a0ca13c4c1 --- /dev/null +++ b/agent/rpc/middleware/rate_limit_mappings.go @@ -0,0 +1,144 @@ +package middleware + +import "github.com/hashicorp/consul/agent/consul/rate" + +// Maps each net/rpc endpoint to a read or write operation type +// for rate limiting purposes. Please be sure to update this list +// if a net/rpc endpoint is removed. +var rpcRateLimitSpecs = map[string]rate.OperationType{ + "ACL.AuthMethodDelete": rate.OperationTypeWrite, + "ACL.AuthMethodList": rate.OperationTypeRead, + "ACL.AuthMethodRead": rate.OperationTypeRead, + "ACL.AuthMethodSet": rate.OperationTypeWrite, + "ACL.Authorize": rate.OperationTypeRead, + "ACL.BindingRuleDelete": rate.OperationTypeWrite, + "ACL.BindingRuleList": rate.OperationTypeRead, + "ACL.BindingRuleRead": rate.OperationTypeRead, + "ACL.BindingRuleSet": rate.OperationTypeWrite, + "ACL.BootstrapTokens": rate.OperationTypeRead, + "ACL.Login": rate.OperationTypeWrite, + "ACL.Logout": rate.OperationTypeWrite, + "ACL.PolicyBatchRead": rate.OperationTypeRead, + "ACL.PolicyDelete": rate.OperationTypeWrite, + "ACL.PolicyList": rate.OperationTypeRead, + "ACL.PolicyRead": rate.OperationTypeRead, + "ACL.PolicyResolve": rate.OperationTypeRead, + "ACL.PolicySet": rate.OperationTypeWrite, + "ACL.ReplicationStatus": rate.OperationTypeRead, + "ACL.RoleBatchRead": rate.OperationTypeRead, + "ACL.RoleDelete": rate.OperationTypeWrite, + "ACL.RoleList": rate.OperationTypeRead, + "ACL.RoleRead": rate.OperationTypeRead, + "ACL.RoleResolve": rate.OperationTypeRead, + "ACL.RoleSet": rate.OperationTypeWrite, + "ACL.TokenBatchRead": rate.OperationTypeRead, + "ACL.TokenClone": rate.OperationTypeRead, + "ACL.TokenDelete": rate.OperationTypeWrite, + "ACL.TokenList": rate.OperationTypeRead, + "ACL.TokenRead": rate.OperationTypeRead, + "ACL.TokenSet": rate.OperationTypeWrite, + + "AutoConfig.InitialConfiguration": rate.OperationTypeRead, + + "AutoEncrypt.Sign": rate.OperationTypeWrite, + + "Catalog.Deregister": rate.OperationTypeWrite, + "Catalog.GatewayServices": rate.OperationTypeRead, + "Catalog.ListDatacenters": rate.OperationTypeRead, + "Catalog.ListNodes": rate.OperationTypeRead, + "Catalog.ListServices": rate.OperationTypeRead, + "Catalog.NodeServiceList": rate.OperationTypeRead, + "Catalog.NodeServices": rate.OperationTypeRead, + "Catalog.Register": rate.OperationTypeWrite, + "Catalog.ServiceList": rate.OperationTypeRead, + "Catalog.ServiceNodes": rate.OperationTypeRead, + "Catalog.VirtualIPForService": rate.OperationTypeRead, + + "ConfigEntry.Apply": rate.OperationTypeWrite, + "ConfigEntry.Delete": rate.OperationTypeWrite, + "ConfigEntry.Get": rate.OperationTypeRead, + "ConfigEntry.List": rate.OperationTypeRead, + "ConfigEntry.ListAll": rate.OperationTypeRead, + "ConfigEntry.ResolveServiceConfig": rate.OperationTypeRead, + + "ConnectCA.ConfigurationGet": rate.OperationTypeRead, + "ConnectCA.ConfigurationSet": rate.OperationTypeWrite, + "ConnectCA.Roots": rate.OperationTypeRead, + "ConnectCA.Sign": rate.OperationTypeWrite, + "ConnectCA.SignIntermediate": rate.OperationTypeWrite, + + "Coordinate.ListDatacenters": rate.OperationTypeRead, + "Coordinate.ListNodes": rate.OperationTypeRead, + "Coordinate.Node": rate.OperationTypeRead, + "Coordinate.Update": rate.OperationTypeWrite, + + "DiscoveryChain.Get": rate.OperationTypeRead, + + "FederationState.Apply": rate.OperationTypeWrite, + "FederationState.Delete": rate.OperationTypeWrite, + "FederationState.Get": rate.OperationTypeRead, + "FederationState.List": rate.OperationTypeRead, + "FederationState.ListMeshGateways": rate.OperationTypeRead, + + "Health.ChecksInState": rate.OperationTypeRead, + "Health.NodeChecks": rate.OperationTypeRead, + "Health.ServiceChecks": rate.OperationTypeRead, + "Health.ServiceNodes": rate.OperationTypeRead, + + "Intention.Apply": rate.OperationTypeWrite, + "Intention.Check": rate.OperationTypeRead, + "Intention.Get": rate.OperationTypeRead, + "Intention.List": rate.OperationTypeRead, + "Intention.Match": rate.OperationTypeRead, + + "Internal.CatalogOverview": rate.OperationTypeRead, + "Internal.EventFire": rate.OperationTypeWrite, + "Internal.ExportedPeeredServices": rate.OperationTypeRead, + "Internal.ExportedServicesForPeer": rate.OperationTypeRead, + "Internal.GatewayIntentions": rate.OperationTypeRead, + "Internal.GatewayServiceDump": rate.OperationTypeRead, + "Internal.IntentionUpstreams": rate.OperationTypeRead, + "Internal.IntentionUpstreamsDestination": rate.OperationTypeRead, + "Internal.KeyringOperation": rate.OperationTypeRead, + "Internal.NodeDump": rate.OperationTypeRead, + "Internal.NodeInfo": rate.OperationTypeRead, + "Internal.PeeredUpstreams": rate.OperationTypeRead, + "Internal.ServiceDump": rate.OperationTypeRead, + "Internal.ServiceGateways": rate.OperationTypeRead, + "Internal.ServiceTopology": rate.OperationTypeRead, + + "KVS.Apply": rate.OperationTypeWrite, + "KVS.Get": rate.OperationTypeRead, + "KVS.List": rate.OperationTypeRead, + "KVS.ListKeys": rate.OperationTypeRead, + + "Operator.AutopilotGetConfiguration": rate.OperationTypeExempt, + "Operator.AutopilotSetConfiguration": rate.OperationTypeExempt, + "Operator.AutopilotState": rate.OperationTypeExempt, + "Operator.RaftGetConfiguration": rate.OperationTypeExempt, + "Operator.RaftRemovePeerByAddress": rate.OperationTypeExempt, + "Operator.RaftRemovePeerByID": rate.OperationTypeExempt, + "Operator.ServerHealth": rate.OperationTypeExempt, + + "PreparedQuery.Apply": rate.OperationTypeWrite, + "PreparedQuery.Execute": rate.OperationTypeRead, + "PreparedQuery.ExecuteRemote": rate.OperationTypeRead, + "PreparedQuery.Explain": rate.OperationTypeRead, + "PreparedQuery.Get": rate.OperationTypeRead, + "PreparedQuery.List": rate.OperationTypeRead, + + "Session.Apply": rate.OperationTypeWrite, + "Session.Check": rate.OperationTypeRead, + "Session.Get": rate.OperationTypeRead, + "Session.List": rate.OperationTypeRead, + "Session.NodeSessions": rate.OperationTypeRead, + "Session.Renew": rate.OperationTypeWrite, + + "Status.Leader": rate.OperationTypeExempt, + "Status.Peers": rate.OperationTypeExempt, + "Status.Ping": rate.OperationTypeExempt, + "Status.RaftStats": rate.OperationTypeExempt, + + "Txn.Apply": rate.OperationTypeWrite, + "Txn.Read": rate.OperationTypeRead, +}