status-im/infra-nimbus
2
0
Fork 0
mirror of https://github.com/status-im/infra-nimbus.git synced 2025-01-18 01:31:51 +00:00
Code Issues Projects Releases Wiki Activity

add logger, make -m option use append action

Browse Source 
Signed-off-by: Jakub Sokołowski <jakub@status.im>
This commit is contained in:
Jakub Sokołowski 2019-04-18 09:35:33 +02:00
parent d0f32c610e
commit b140075d12
No known key found for this signature in database
GPG Key ID: 4EF064D0E6D63020
1 changed files with 30 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
ansible/roles/nimbus-stats/files/collect.py
View File

@ -1,13 +1,28 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
import os import os
import sys
import json import json
import socket import socket
import logging
import requests import requests
from datetime import datetime from datetime import datetime
from optparse import OptionParser from optparse import OptionParser
from elasticsearch import Elasticsearch from elasticsearch import Elasticsearch
HELP_DESCRIPTION='This script collects latest log entries for provided messages from all nodes in a Nimbus fleet'
HELP_EXAMPLE='Example: collect -i logstash-2019.03.01 output.json'
DEFAULT_MESSAGES = [
'Fork chosen',
'Attestation received',
'Slot start',
]
ENV = os.environ ENV = os.environ
LOG = logging.getLogger('root')
handler = logging.StreamHandler(sys.stderr)
formatter = logging.Formatter('%(asctime)s [%(levelname)s]: %(message)s')
handler.setFormatter(formatter)
LOG.addHandler(handler)
class ES: class ES:
def __init__(self, host, port, page_size): def __init__(self, host, port, page_size):
@ -48,19 +63,13 @@ def get_first_for_node(logs):
prog_obj[log['message']] = json.loads(log['raw']) prog_obj[log['message']] = json.loads(log['raw'])
return data return data
HELP_DESCRIPTION='This script collects latest log entries for provided messages from all nodes in a Nimbus fleet'
HELP_EXAMPLE='Example: collect -i logstash-2019.03.01 output.json'
DEFAULT_MESSAGES = [
'Fork chosen',
'Attestation received',
'Slot start',
]
def parse_opts(): def parse_opts():
parser = OptionParser(description=HELP_DESCRIPTION, epilog=HELP_EXAMPLE) parser = OptionParser(description=HELP_DESCRIPTION, epilog=HELP_EXAMPLE)
parser.add_option('-i', '--index', dest='es_index', parser.add_option('-i', '--index', dest='es_index',
default='logstash-'+datetime.today().strftime('%Y.%m.%d'), default='logstash-'+datetime.today().strftime('%Y.%m.%d'),
help='Patter for matching indices. (%default)') help='Patter for matching indices. (%default)')
parser.add_option('-m', '--messages', action="append", default=DEFAULT_MESSAGES,
help='Messages to query for. (%default)')
parser.add_option('-H', '--host', dest='es_host', default='localhost', parser.add_option('-H', '--host', dest='es_host', default='localhost',
help='ElasticSearch host. (%default)') help='ElasticSearch host. (%default)')
parser.add_option('-P', '--port', dest='es_port', default=9200, parser.add_option('-P', '--port', dest='es_port', default=9200,
@ -71,28 +80,35 @@ def parse_opts():
help='Period for which to query logs. (%default)') help='Period for which to query logs. (%default)')
parser.add_option('-S', '--page-size', default=10000, parser.add_option('-S', '--page-size', default=10000,
help='Size of results page. (%default)') help='Size of results page. (%default)')
parser.add_option('-m', '--messages', default=DEFAULT_MESSAGES,
help='Messages to query for. (%default)')
parser.add_option('-f', '--fleet', default='nimbus.test', parser.add_option('-f', '--fleet', default='nimbus.test',
help='Fleet to query for. (%default)') help='Fleet to query for. (%default)')
parser.add_option('-l', '--log-level', default='INFO',
help='Logging level. (%default)')
parser.add_option('-o', '--output-file', parser.add_option('-o', '--output-file',
help='File to which write the resulting JSON.') help='File to which write the resulting JSON.')
return parser.parse_args() return parser.parse_args()
def debug_options(opts):
LOG.debug('Settings:')
for key, val in opts.__dict__.items():
LOG.debug('%s=%s', key, val)
def main(): def main():
(opts, args) = parse_opts() (opts, args) = parse_opts()
LOG.setLevel(opts.log_level)
debug_options(opts)
es = ES(opts.es_host, opts.es_port, opts.page_size) es = ES(opts.es_host, opts.es_port, opts.page_size)
logs = [] logs = []
#print('Querying: {}'.format(opts.fleet)) LOG.info('Querying fleet: %s', opts.fleet)
for msg in opts.messages: for msg in opts.messages:
#print(' - MSG: {}'.format(msg))
query = es.make_query(opts.fleet, opts.program, msg, opts.since) query = es.make_query(opts.fleet, opts.program, msg, opts.since)
rval = es.get_logs(query) rval = es.get_logs(query)
#print(' + Found: {}'.format(rval['hits']['total'])) LOG.info('Message: "%s" Found: %d', msg, rval['hits']['total'])
logs.extend(rval['hits']['hits']) logs.extend(rval['hits']['hits'])
data = get_first_for_node(logs) data = get_first_for_node(logs)