status-im/infra-nimbus
2
0
Fork 0
mirror of https://github.com/status-im/infra-nimbus.git synced 2025-01-17 17:21:16 +00:00
Code Issues Projects Releases Wiki Activity

add logger, make -m option use append action

Browse Source 
Signed-off-by: Jakub Sokołowski <jakub@status.im>
This commit is contained in:
Jakub Sokołowski 2019-04-18 09:35:33 +02:00
parent d0f32c610e
commit b140075d12
No known key found for this signature in database
GPG Key ID: 4EF064D0E6D63020
1 changed files with 30 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
ansible/roles/nimbus-stats/files/collect.py
View File

@ -1,13 +1,28 @@
#!/usr/bin/env python3
import os
import sys
import json
import socket
import logging
import requests
from datetime import datetime
from optparse import OptionParser
from elasticsearch import Elasticsearch
HELP_DESCRIPTION='This script collects latest log entries for provided messages from all nodes in a Nimbus fleet'
HELP_EXAMPLE='Example: collect -i logstash-2019.03.01 output.json'
DEFAULT_MESSAGES = [
'Fork chosen',
'Attestation received',
'Slot start',
]
ENV = os.environ
LOG = logging.getLogger('root')
handler = logging.StreamHandler(sys.stderr)
formatter = logging.Formatter('%(asctime)s [%(levelname)s]: %(message)s')
handler.setFormatter(formatter)
LOG.addHandler(handler)
class ES:
def __init__(self, host, port, page_size):
@ -48,19 +63,13 @@ def get_first_for_node(logs):
prog_obj[log['message']] = json.loads(log['raw'])
return data
HELP_DESCRIPTION='This script collects latest log entries for provided messages from all nodes in a Nimbus fleet'
HELP_EXAMPLE='Example: collect -i logstash-2019.03.01 output.json'
DEFAULT_MESSAGES = [
'Fork chosen',
'Attestation received',
'Slot start',
]
def parse_opts():
parser = OptionParser(description=HELP_DESCRIPTION, epilog=HELP_EXAMPLE)
parser.add_option('-i', '--index', dest='es_index',
default='logstash-'+datetime.today().strftime('%Y.%m.%d'),
help='Patter for matching indices. (%default)')
parser.add_option('-m', '--messages', action="append", default=DEFAULT_MESSAGES,
help='Messages to query for. (%default)')
parser.add_option('-H', '--host', dest='es_host', default='localhost',
help='ElasticSearch host. (%default)')
parser.add_option('-P', '--port', dest='es_port', default=9200,
@ -71,28 +80,35 @@ def parse_opts():
help='Period for which to query logs. (%default)')
parser.add_option('-S', '--page-size', default=10000,
help='Size of results page. (%default)')
parser.add_option('-m', '--messages', default=DEFAULT_MESSAGES,
help='Messages to query for. (%default)')
parser.add_option('-f', '--fleet', default='nimbus.test',
help='Fleet to query for. (%default)')
parser.add_option('-l', '--log-level', default='INFO',
help='Logging level. (%default)')
parser.add_option('-o', '--output-file',
help='File to which write the resulting JSON.')
return parser.parse_args()
def debug_options(opts):
LOG.debug('Settings:')
for key, val in opts.__dict__.items():
LOG.debug('%s=%s', key, val)
def main():
(opts, args) = parse_opts()
LOG.setLevel(opts.log_level)
debug_options(opts)
es = ES(opts.es_host, opts.es_port, opts.page_size)
logs = []
#print('Querying: {}'.format(opts.fleet))
LOG.info('Querying fleet: %s', opts.fleet)
for msg in opts.messages:
#print(' - MSG: {}'.format(msg))
query = es.make_query(opts.fleet, opts.program, msg, opts.since)
rval = es.get_logs(query)
#print(' + Found: {}'.format(rval['hits']['total']))
LOG.info('Message: "%s" Found: %d', msg, rval['hits']['total'])
logs.extend(rval['hits']['hits'])
data = get_first_for_node(logs)