add logger, make -m option use append action

Signed-off-by: Jakub Sokołowski <jakub@status.im>

ansible/roles/nimbus-stats/files/collect.py

@@ -1,13 +1,28 @@
 #!/usr/bin/env python3
 import os
+import sys
 import json
 import socket
+import logging
 import requests
 from datetime import datetime
 from optparse import OptionParser
 from elasticsearch import Elasticsearch
 
+HELP_DESCRIPTION='This script collects latest log entries for provided messages from all nodes in a Nimbus fleet'
+HELP_EXAMPLE='Example: collect -i logstash-2019.03.01 output.json'
+DEFAULT_MESSAGES = [
+    'Fork chosen',
+    'Attestation received',
+    'Slot start',
+]
+
 ENV = os.environ
+LOG = logging.getLogger('root')
+handler = logging.StreamHandler(sys.stderr)
+formatter = logging.Formatter('%(asctime)s [%(levelname)s]: %(message)s')
+handler.setFormatter(formatter)
+LOG.addHandler(handler)
 
 class ES:
     def __init__(self, host, port, page_size):
@@ -48,19 +63,13 @@ def get_first_for_node(logs):
         prog_obj[log['message']] = json.loads(log['raw'])
     return data
 
-HELP_DESCRIPTION='This script collects latest log entries for provided messages from all nodes in a Nimbus fleet'
-HELP_EXAMPLE='Example: collect -i logstash-2019.03.01 output.json'
-DEFAULT_MESSAGES = [
-    'Fork chosen',
-    'Attestation received',
-    'Slot start',
-]
-
 def parse_opts():
     parser = OptionParser(description=HELP_DESCRIPTION, epilog=HELP_EXAMPLE)
     parser.add_option('-i', '--index', dest='es_index',
                       default='logstash-'+datetime.today().strftime('%Y.%m.%d'),
                       help='Patter for matching indices. (%default)')
+    parser.add_option('-m', '--messages', action="append", default=DEFAULT_MESSAGES,
+                      help='Messages to query for. (%default)')
     parser.add_option('-H', '--host', dest='es_host', default='localhost',
                       help='ElasticSearch host. (%default)')
     parser.add_option('-P', '--port', dest='es_port', default=9200,
@@ -71,28 +80,35 @@ def parse_opts():
                       help='Period for which to query logs. (%default)')
     parser.add_option('-S', '--page-size', default=10000,
                       help='Size of results page. (%default)')
-    parser.add_option('-m', '--messages', default=DEFAULT_MESSAGES,
-                      help='Messages to query for. (%default)')
     parser.add_option('-f', '--fleet', default='nimbus.test',
                       help='Fleet to query for. (%default)')
+    parser.add_option('-l', '--log-level', default='INFO',
+                      help='Logging level. (%default)')
     parser.add_option('-o', '--output-file',
                       help='File to which write the resulting JSON.')
     
     return parser.parse_args()
 
+def debug_options(opts):
+    LOG.debug('Settings:')
+    for key, val in opts.__dict__.items():
+       LOG.debug('%s=%s', key, val)
+
 def main():
     (opts, args) = parse_opts()
+    LOG.setLevel(opts.log_level)
+
+    debug_options(opts)
 
     es = ES(opts.es_host, opts.es_port, opts.page_size)
     
     logs = []
 
-    #print('Querying: {}'.format(opts.fleet))
+    LOG.info('Querying fleet: %s', opts.fleet)
     for msg in opts.messages:
-        #print(' - MSG: {}'.format(msg))
         query = es.make_query(opts.fleet, opts.program, msg, opts.since)
         rval = es.get_logs(query)
-        #print(' + Found: {}'.format(rval['hits']['total']))
+        LOG.info('Message: "%s" Found: %d', msg, rval['hits']['total'])
         logs.extend(rval['hits']['hits'])
     
     data = get_first_for_node(logs)