status-im
/
infra-nimbus
mirror of https://github.com/status-im/infra-nimbus.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

use one query for all types of messages 

Signed-off-by: Jakub Sokołowski <jakub@status.im>
This commit is contained in:
Jakub Sokołowski 2019-04-19 12:50:55 +02:00
parent 0de6132789
commit 3035b457b1
No known key found for this signature in database
GPG Key ID: 4EF064D0E6D63020
1 changed files with 32 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
ansible/roles/nimbus-stats/files/collect.py
View File

@ -29,15 +29,18 @@ class ES:
self.page_size = page_size
self.es = Elasticsearch([host], port=port, timeout=30)
def make_query(self, fleet, program, message, after):
def make_query(self, fleet, program, messages, after):
return {
'query': { 'bool': {
'must': [
{ 'match': { 'fleet': fleet } },
{ 'match': { 'program': program } },
{ 'match_phrase': { 'message': message } },
{ 'range': { '@timestamp': { 'gt': after } } },
],
'should': [
{ 'match_phrase': { 'message': msg } } for msg in messages
],
'minimum_should_match': 1,
}, },
'sort': [
{ '@timestamp': { 'order': 'desc' } },
@ -59,10 +62,30 @@ def get_first_for_node(logs):
for log_obj in logs:
log = log_obj['_source']
host_obj = data.setdefault(log['logsource'], {})
prog_obj = host_obj.setdefault(log['program'], {})
# remove "docker/" prefix from program name
program = log['program'].replace('docker/', '')
prog_obj = host_obj.setdefault(program, {})
prog_obj[log['message']] = json.loads(log['raw'])
return data
def save_stats(data, output_file):
# add metadata for easier debugging
output = {
'meta': {
'hostname': socket.gethostname(),
'timestamp': datetime.utcnow().isoformat(),
},
'data': data,
}
if output_file:
LOG.info('Saving to file: %s', output_file)
with open(output_file, 'w') as f:
json.dump(data, f, indent=4)
else:
LOG.info('Printing results to STDOUT')
print(json.dumps(data, indent=4))
def parse_opts():
parser = OptionParser(description=HELP_DESCRIPTION, epilog=HELP_EXAMPLE)
parser.add_option('-i', '--index', dest='es_index',
@ -102,31 +125,16 @@ def main():
es = ES(opts.es_host, opts.es_port, opts.page_size)
logs = []
LOG.info('Querying fleet: %s', opts.fleet)
for msg in opts.messages:
query = es.make_query(opts.fleet, opts.program, msg, opts.since)
rval = es.get_logs(query)
LOG.info('Message: "%s" Found: %d', msg, rval['hits']['total'])
logs.extend(rval['hits']['hits'])
query = es.make_query(opts.fleet, opts.program, opts.messages, opts.since)
rval = es.get_logs(query)
LOG.info('Found matching logs: %d', rval['hits']['total'])
logs = rval['hits']['hits']
data = get_first_for_node(logs)
# add metadata for easier debugging
output = {
'meta': {
'hostname': socket.gethostname(),
'timestamp': datetime.utcnow().isoformat(),
},
'data': data,
}
if opts.output_file:
with open(opts.output_file, 'w') as f:
json.dump(data, f, indent=4)
else:
print(json.dumps(data, indent=4))
save_stats(data, opts.output_file)
if __name__ == '__main__':
main()