diff --git a/apps/sonda/sonda.py b/apps/sonda/sonda.py index 8b74bd029..e5fe6c657 100644 --- a/apps/sonda/sonda.py +++ b/apps/sonda/sonda.py @@ -23,6 +23,22 @@ store_query_latency = Gauge('store_query_latency', 'Latency of the last store qu consecutive_successful_responses = Gauge('consecutive_successful_responses', 'Consecutive successful store responses', ['node']) node_health = Gauge('node_health', "Binary indicator of a node's health. 1 is healthy, 0 is not", ['node']) +# Bounded set of values for the 'error' label of failed_store_queries. A label must +# never hold a raw cause string: that is what blows up cardinality. +ERR_LOCAL_NODE_UNREACHABLE = 'local_node_unreachable' +ERR_PEER_UNREACHABLE = 'peer_unreachable' +ERR_QUERY_TIMEOUT = 'query_timeout' +ERR_STORE_BACKEND_ERROR = 'store_backend_error' +ERR_UNKNOWN = 'unknown' + +# Store protocol status code -> error label. 300 (BAD_RESPONSE) is absent on purpose: +# it is the only status whose cause string has to be inspected to be classified. +STORE_STATUS_ERRORS = { + 504: ERR_PEER_UNREACHABLE, # PEER_DIAL_FAILURE + 503: ERR_STORE_BACKEND_ERROR, # SERVICE_UNAVAILABLE + 429: ERR_STORE_BACKEND_ERROR, # TOO_MANY_REQUESTS: load shedding is store-side +} + # Argparser configuration parser = argparse.ArgumentParser(description='') @@ -77,18 +93,49 @@ def send_sonda_msg(rest_address, pubsub_topic, content_topic, timestamp): return False +# Maps the cause of a store BAD_RESPONSE (300) onto the bounded error set. +# This is the only place where a cause string is inspected. +def classify_bad_response(cause): + cause = cause.lower() + if 'archive error' in cause or 'driver_error' in cause: + return ERR_STORE_BACKEND_ERROR + if 'cancel' in cause or 'timed out' in cause or 'timeout' in cause: + return ERR_QUERY_TIMEOUT + if 'closed' in cause or 'reset' in cause or 'eof' in cause: + return ERR_PEER_UNREACHABLE + return ERR_UNKNOWN + + +# Increments failed_store_queries with a bounded 'error' label, keeping the raw cause in +# the log only. blame_peer=False leaves 'node' empty for failures where sonda never got +# an answer about the store peer, so we don't blame a peer that was never contacted. +def record_store_failure(store_node, error, cause, blame_peer=True): + log_with_utc(f'Failed store query node={store_node} error={error} cause={cause}') + failed_store_queries.labels(node=store_node if blame_peer else '', error=error).inc() + consecutive_successful_responses.labels(node=store_node).set(0) + + # We return true if both our node and the queried Store node returned a 200 # If our message isn't found but we did get a store 200 response, this function still returns true def check_store_response(json_response, store_node, timestamp): + status_code = json_response.get('statusCode') + status_desc = json_response.get('statusDesc') or '' + # Check for the store node status code - if json_response.get('statusCode') != 200: - error = f"{json_response.get('statusCode')} {json_response.get('statusDesc')}" - log_with_utc(f'Failed performing store query {error}') - failed_store_queries.labels(node=store_node, error=error).inc() - consecutive_successful_responses.labels(node=store_node).set(0) - + if status_code != 200: + blame_peer = True + if status_code == 300: + error = classify_bad_response(status_desc) + elif status_code in STORE_STATUS_ERRORS: + error = STORE_STATUS_ERRORS[status_code] + else: + # A 400 means sonda built a bad query. That is a sonda bug, not the peer's fault. + error = ERR_UNKNOWN + blame_peer = False + + record_store_failure(store_node, error, f'{status_code} {status_desc}', blame_peer) return False - + messages = json_response.get('messages') # If there's no message in the response, increase counters and return if not messages: @@ -132,27 +179,34 @@ def send_store_query(rest_address, store_node, encoded_pubsub_topic, encoded_con try: log_with_utc(f'Sending store request to {store_node}') response = requests.get(url, params=params) + except requests.RequestException as e: + # No HTTP response at all, so sonda learned nothing about the store peer. + # Checked first: attributing this to a peer we never reached would be a lie. + record_store_failure(store_node, ERR_LOCAL_NODE_UNREACHABLE, str(e), blame_peer=False) + return False except Exception as e: - log_with_utc(f'Error sending request: {e}') - failed_store_queries.labels(node=store_node, error=str(e)).inc() - consecutive_successful_responses.labels(node=store_node).set(0) - return False + # An exception inside sonda itself. + record_store_failure(store_node, ERR_UNKNOWN, str(e), blame_peer=False) + return False elapsed_seconds = time.time() - s_time log_with_utc(f'Response from {rest_address}: status:{response.status_code} [{elapsed_seconds:.4f} s.]') + # Our node turns a store TOO_MANY_REQUESTS into an HTTP 429 instead of a status code. + if response.status_code == 429: + record_store_failure(store_node, ERR_STORE_BACKEND_ERROR, response.text) + return False + if response.status_code != 200: - failed_store_queries.labels(node=store_node, error=f'{response.status_code} {response.content}').inc() - consecutive_successful_responses.labels(node=store_node).set(0) + record_store_failure(store_node, ERR_UNKNOWN, + f'HTTP {response.status_code} {response.text}', blame_peer=False) return False # Parse REST response into JSON try: json_response = response.json() - except Exception as e: - log_with_utc(f'Error parsing response JSON: {e}') - failed_store_queries.labels(node=store_node, error="JSON parse error").inc() - consecutive_successful_responses.labels(node=store_node).set(0) + except ValueError as e: + record_store_failure(store_node, ERR_UNKNOWN, str(e), blame_peer=False) return False # Analyze Store response. Return false if response is incorrect or has an error status