Merge pull request #3765 from diptanu/f-agent-telemetry

Added telemetry around Catalog APIs

agent/catalog_endpoint.go

@@ -5,12 +5,15 @@ import (
 	"net/http"
 	"strings"
 
+	metrics "github.com/armon/go-metrics"
 	"github.com/hashicorp/consul/agent/structs"
 )
 
 var durations = NewDurationFixer("interval", "timeout", "deregistercriticalserviceafter")
 
 func (s *HTTPServer) CatalogRegister(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
+	metrics.IncrCounterWithLabels([]string{"client", "api", "catalog_register"}, 1,
+		[]metrics.Label{{Name: "node", Value: s.nodeName()}})
 	if req.Method != "PUT" {
 		return nil, MethodNotAllowedError{req.Method, []string{"PUT"}}
 	}
@@ -31,12 +34,18 @@ func (s *HTTPServer) CatalogRegister(resp http.ResponseWriter, req *http.Request
 	// Forward to the servers
 	var out struct{}
 	if err := s.agent.RPC("Catalog.Register", &args, &out); err != nil {
+		metrics.IncrCounterWithLabels([]string{"client", "rpc", "error", "catalog_register"}, 1,
+			[]metrics.Label{{Name: "node", Value: s.nodeName()}})
 		return nil, err
 	}
+	metrics.IncrCounterWithLabels([]string{"client", "api", "success", "catalog_register"}, 1,
+		[]metrics.Label{{Name: "node", Value: s.nodeName()}})
 	return true, nil
 }
 
 func (s *HTTPServer) CatalogDeregister(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
+	metrics.IncrCounterWithLabels([]string{"client", "api", "catalog_deregister"}, 1,
+		[]metrics.Label{{Name: "node", Value: s.nodeName()}})
 	if req.Method != "PUT" {
 		return nil, MethodNotAllowedError{req.Method, []string{"PUT"}}
 	}
@@ -57,24 +66,36 @@ func (s *HTTPServer) CatalogDeregister(resp http.ResponseWriter, req *http.Reque
 	// Forward to the servers
 	var out struct{}
 	if err := s.agent.RPC("Catalog.Deregister", &args, &out); err != nil {
+		metrics.IncrCounterWithLabels([]string{"client", "rpc", "error", "catalog_deregister"}, 1,
+			[]metrics.Label{{Name: "node", Value: s.nodeName()}})
 		return nil, err
 	}
+	metrics.IncrCounterWithLabels([]string{"client", "api", "success", "catalog_deregister"}, 1,
+		[]metrics.Label{{Name: "node", Value: s.nodeName()}})
 	return true, nil
 }
 
 func (s *HTTPServer) CatalogDatacenters(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
+	metrics.IncrCounterWithLabels([]string{"client", "api", "catalog_datacenters"}, 1,
+		[]metrics.Label{{Name: "node", Value: s.nodeName()}})
 	if req.Method != "GET" {
 		return nil, MethodNotAllowedError{req.Method, []string{"GET"}}
 	}
 
 	var out []string
 	if err := s.agent.RPC("Catalog.ListDatacenters", struct{}{}, &out); err != nil {
+		metrics.IncrCounterWithLabels([]string{"client", "rpc", "error", "catalog_datacenters"}, 1,
+			[]metrics.Label{{Name: "node", Value: s.nodeName()}})
 		return nil, err
 	}
+	metrics.IncrCounterWithLabels([]string{"client", "api", "success", "catalog_datacenters"}, 1,
+		[]metrics.Label{{Name: "node", Value: s.nodeName()}})
 	return out, nil
 }
 
 func (s *HTTPServer) CatalogNodes(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
+	metrics.IncrCounterWithLabels([]string{"client", "api", "catalog_nodes"}, 1,
+		[]metrics.Label{{Name: "node", Value: s.nodeName()}})
 	if req.Method != "GET" {
 		return nil, MethodNotAllowedError{req.Method, []string{"GET"}}
 	}
@@ -84,6 +105,8 @@ func (s *HTTPServer) CatalogNodes(resp http.ResponseWriter, req *http.Request) (
 	s.parseSource(req, &args.Source)
 	args.NodeMetaFilters = s.parseMetaFilter(req)
 	if done := s.parse(resp, req, &args.Datacenter, &args.QueryOptions); done {
+		metrics.IncrCounterWithLabels([]string{"client", "rpc", "error", "catalog_nodes"}, 1,
+			[]metrics.Label{{Name: "node", Value: s.nodeName()}})
 		return nil, nil
 	}
 
@@ -98,10 +121,14 @@ func (s *HTTPServer) CatalogNodes(resp http.ResponseWriter, req *http.Request) (
 	if out.Nodes == nil {
 		out.Nodes = make(structs.Nodes, 0)
 	}
+	metrics.IncrCounterWithLabels([]string{"client", "api", "success", "catalog_nodes"}, 1,
+		[]metrics.Label{{Name: "node", Value: s.nodeName()}})
 	return out.Nodes, nil
 }
 
 func (s *HTTPServer) CatalogServices(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
+	metrics.IncrCounterWithLabels([]string{"client", "api", "catalog_services"}, 1,
+		[]metrics.Label{{Name: "node", Value: s.nodeName()}})
 	if req.Method != "GET" {
 		return nil, MethodNotAllowedError{req.Method, []string{"GET"}}
 	}
@@ -116,6 +143,8 @@ func (s *HTTPServer) CatalogServices(resp http.ResponseWriter, req *http.Request
 	var out structs.IndexedServices
 	defer setMeta(resp, &out.QueryMeta)
 	if err := s.agent.RPC("Catalog.ListServices", &args, &out); err != nil {
+		metrics.IncrCounterWithLabels([]string{"client", "rpc", "error", "catalog_services"}, 1,
+			[]metrics.Label{{Name: "node", Value: s.nodeName()}})
 		return nil, err
 	}
 
@@ -123,10 +152,14 @@ func (s *HTTPServer) CatalogServices(resp http.ResponseWriter, req *http.Request
 	if out.Services == nil {
 		out.Services = make(structs.Services, 0)
 	}
+	metrics.IncrCounterWithLabels([]string{"client", "api", "success", "catalog_services"}, 1,
+		[]metrics.Label{{Name: "node", Value: s.nodeName()}})
 	return out.Services, nil
 }
 
 func (s *HTTPServer) CatalogServiceNodes(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
+	metrics.IncrCounterWithLabels([]string{"client", "api", "catalog_service_nodes"}, 1,
+		[]metrics.Label{{Name: "node", Value: s.nodeName()}})
 	if req.Method != "GET" {
 		return nil, MethodNotAllowedError{req.Method, []string{"GET"}}
 	}
@@ -158,6 +191,8 @@ func (s *HTTPServer) CatalogServiceNodes(resp http.ResponseWriter, req *http.Req
 	var out structs.IndexedServiceNodes
 	defer setMeta(resp, &out.QueryMeta)
 	if err := s.agent.RPC("Catalog.ServiceNodes", &args, &out); err != nil {
+		metrics.IncrCounterWithLabels([]string{"client", "rpc", "error", "catalog_service_nodes"}, 1,
+			[]metrics.Label{{Name: "node", Value: s.nodeName()}})
 		return nil, err
 	}
 	s.agent.TranslateAddresses(args.Datacenter, out.ServiceNodes)
@@ -171,10 +206,14 @@ func (s *HTTPServer) CatalogServiceNodes(resp http.ResponseWriter, req *http.Req
 			s.ServiceTags = make([]string, 0)
 		}
 	}
+	metrics.IncrCounterWithLabels([]string{"client", "api", "success", "catalog_service_nodes"}, 1,
+		[]metrics.Label{{Name: "node", Value: s.nodeName()}})
 	return out.ServiceNodes, nil
 }
 
 func (s *HTTPServer) CatalogNodeServices(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
+	metrics.IncrCounterWithLabels([]string{"client", "api", "catalog_node_services"}, 1,
+		[]metrics.Label{{Name: "node", Value: s.nodeName()}})
 	if req.Method != "GET" {
 		return nil, MethodNotAllowedError{req.Method, []string{"GET"}}
 	}
@@ -197,6 +236,8 @@ func (s *HTTPServer) CatalogNodeServices(resp http.ResponseWriter, req *http.Req
 	var out structs.IndexedNodeServices
 	defer setMeta(resp, &out.QueryMeta)
 	if err := s.agent.RPC("Catalog.NodeServices", &args, &out); err != nil {
+		metrics.IncrCounterWithLabels([]string{"client", "rpc", "error", "catalog_node_services"}, 1,
+			[]metrics.Label{{Name: "node", Value: s.nodeName()}})
 		return nil, err
 	}
 	if out.NodeServices != nil && out.NodeServices.Node != nil {
@@ -211,5 +252,7 @@ func (s *HTTPServer) CatalogNodeServices(resp http.ResponseWriter, req *http.Req
 			}
 		}
 	}
+	metrics.IncrCounterWithLabels([]string{"client", "api", "success", "catalog_node_services"}, 1,
+		[]metrics.Label{{Name: "node", Value: s.nodeName()}})
 	return out.NodeServices, nil
 }

agent/http.go

@@ -139,6 +139,11 @@ func (s *HTTPServer) handler(enableDebug bool) http.Handler {
 	}
 }
 
+// nodeName returns the node name of the agent
+func (s *HTTPServer) nodeName() string {
+	return s.agent.config.NodeName
+}
+
 // aclEndpointRE is used to find old ACL endpoints that take tokens in the URL
 // so that we can redact them. The ACL endpoints that take the token in the URL
 // are all of the form /v1/acl/<verb>/<token>, and can optionally include query

website/source/docs/agent/telemetry.html.md

@@ -74,6 +74,90 @@ These metrics are used to monitor the health of specific Consul agents.
     <td>rejected requests</td>
     <td>counter</td>
   </tr>
+  <tr>
+    <td>`consul.client.api.catalog_register.<node>`</td>
+    <td>This increments whenever a Consul agent receives a catalog register request.</td>
+    <td>requests</td>
+    <td>counter</td>
+  </tr>
+  <tr>
+    <td>`consul.client.api.success.catalog_register.<node>`</td>
+    <td>This increments whenever a Consul agent successfully responds to a catalog register request.</td>
+    <td>requests</td>
+    <td>counter</td>
+  </tr>
+  <tr>
+    <td>`consul.client.api.catalog_deregister.<node>`</td>
+    <td>This increments whenever a Consul agent receives a catalog de-register request.</td>
+    <td>requests</td>
+    <td>counter</td>
+  </tr>
+  <tr>
+    <td>`consul.client.api.success.catalog_deregister.<node>`</td>
+    <td>This increments whenever a Consul agent successfully responds to a catalog de-register request.</td>
+    <td>requests</td>
+    <td>counter</td>
+  </tr>
+  <tr>
+    <td>`consul.client.api.catalog_datacenters.<node>`</td>
+    <td>This increments whenever a Consul agent receives a request to list datacenters in the catalog.</td>
+    <td>requests</td>
+    <td>counter</td>
+  </tr>
+  <tr>
+    <td>`consul.client.api.success.catalog_datacenters.<node>`</td>
+    <td>This increments whenever a Consul agent successfully responds to a request to list datacenters.</td>
+    <td>requests</td>
+    <td>counter</td>
+  </tr>
+  <tr>
+    <td>`consul.client.api.catalog_nodes.<node>`</td>
+    <td>This increments whenever a Consul agent receives a request to list nodes from the catalog.</td>
+    <td>requests</td>
+    <td>counter</td>
+  </tr>
+  <tr>
+    <td>`consul.client.api.success.catalog_nodes.<node>`</td>
+    <td>This increments whenever a Consul agent successfully responds to a request to list nodes.</td>
+    <td>requests</td>
+    <td>counter</td>
+  </tr>
+  <tr>
+    <td>`consul.client.api.catalog_services.<node>`</td>
+    <td>This increments whenever a Consul agent receives a request to list services from the catalog.</td>
+    <td>requests</td>
+    <td>counter</td>
+  </tr>
+  <tr>
+    <td>`consul.client.api.success.catalog_services.<node>`</td>
+    <td>This increments whenever a Consul agent successfully responds to a request to list services.</td>
+    <td>requests</td>
+    <td>counter</td>
+  </tr>
+  <tr>
+    <td>`consul.client.api.catalog_service_nodes.<node>`</td>
+    <td>This increments whenever a Consul agent receives a request to list nodes offering a service.</td>
+    <td>requests</td>
+    <td>counter</td>
+  </tr>
+  <tr>
+    <td>`consul.client.api.success.catalog_service_nodes.<node>`</td>
+    <td>This increments whenever a Consul agent successfully responds to a request to list nodes offering a service.</td>
+    <td>requests</td>
+    <td>counter</td>
+  </tr>
+  <tr>
+    <td>`consul.client.api.catalog_node_services.<node>`</td>
+    <td>This increments whenever a Consul agent receives a request to list services registered in a node.</td>
+    <td>requests</td>
+    <td>counter</td>
+  </tr>
+  <tr>
+    <td>`consul.client.api.success.catalog_node_services.<node>`</td>
+    <td>This increments whenever a Consul agent successfully responds to a request to list services in a service.</td>
+    <td>requests</td>
+    <td>counter</td>
+  </tr>
   <tr>
     <td>`consul.runtime.num_goroutines`</td>
     <td>This tracks the number of running goroutines and is a general load pressure indicator. This may burst from time to time but should return to a steady state value.</td>