From c3492f2d49975887bacac0583e2bef7aa378b262 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Soko=C5=82owski?= Date: Tue, 14 Jul 2020 12:08:17 +0200 Subject: [PATCH] skip current day MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jakub SokoĊ‚owski --- main.py | 9 +++++++-- postgres.py | 6 ------ 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/main.py b/main.py index 586ee96..ffce116 100755 --- a/main.py +++ b/main.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 from os import path +from datetime import datetime from optparse import OptionParser from query import ESQueryPeers @@ -59,8 +60,12 @@ def main(): peers = [] for index in esq.get_indices(opts.index_pattern): + # skip already injected indices if index in present_indices: continue + # skip current day as it's incomplete + if index == datetime.now().strftime('logstash-%Y.%m.%d'): + continue print('Index: {}'.format(index)) peers.extend(esq.get_peers(index, opts.field, opts.max_size)) @@ -68,8 +73,8 @@ def main(): print('Nothing to insert into database.') exit(0) - rval = psg.inject_peers(peers) - print(rval) + print('Injecting peers data into database...') + psg.inject_peers(peers) if __name__ == '__main__': main() diff --git a/postgres.py b/postgres.py index b82ac0f..a1a294b 100644 --- a/postgres.py +++ b/postgres.py @@ -1,5 +1,4 @@ import psycopg2 -from datetime import datetime class PGDatabase: _SCHEMA = """ @@ -21,10 +20,6 @@ class PGDatabase: self.c.execute(self._SCHEMA) self.db.commit() - def get_last_day(self): - self.c.execute('SELECT date FROM peers ORDER BY date DESC LIMIT 1;') - return self.c.fetchone() - def get_present_days(self): self.c.execute('SELECT DISTINCT date FROM peers;') return [d[0].strftime('%Y-%m-%d') for d in self.c.fetchall()] @@ -38,4 +33,3 @@ class PGDatabase: 'INSERT INTO peers(date, peer, count) VALUES {}'.format(args) ) self.db.commit() - return rval