status-im/infra-utils
2
0
Fork 0
mirror of https://github.com/status-im/infra-utils.git synced 2025-02-23 09:28:08 +00:00
Code Issues Projects Releases Wiki Activity

elasticsearch: hash peer_id's before putting them into CSV

Browse Source 
Signed-off-by: Jakub Sokołowski <jakub@status.im>
This commit is contained in:
Jakub Sokołowski 2020-06-30 09:56:16 +02:00
parent 0a5c14a854
commit d1e0426cc2
No known key found for this signature in database
GPG Key ID: 4EF064D0E6D63020
1 changed files with 15 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
elasticsearch/unique_count.py
View File

@ -1,6 +1,7 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
import os import os
import csv import csv
import hashlib
from optparse import OptionParser from optparse import OptionParser
from elasticsearch import Elasticsearch from elasticsearch import Elasticsearch
@ -31,6 +32,9 @@ def parse_opts():
def remove_prefix(text, prefix): def remove_prefix(text, prefix):
return text[text.startswith(prefix) and len(prefix):] return text[text.startswith(prefix) and len(prefix):]
def hash_string(text):
return hashlib.sha256(text.encode('utf-8')).hexdigest()
def main(): def main():
(opts, args) = parse_opts() (opts, args) = parse_opts()
@ -46,17 +50,16 @@ def main():
indices = es.indices.get(index=opts.index_pattern).keys() indices = es.indices.get(index=opts.index_pattern).keys()
body = { body = {
"size": 0, 'size': 0,
"aggs": { "peers": { 'aggs': { 'peers': {
"terms": { 'terms': {
"field": opts.field, 'field': opts.field,
"size": 10000, 'size': 10000,
#"min_doc_count": 100,
}, },
}, }, }, },
} }
csv_field_names = ["date", "peer", "count"] csv_field_names = ['date', 'peer', 'count']
with open(opts.out_file, 'w') as f: with open(opts.out_file, 'w') as f:
writer = csv.DictWriter(f, fieldnames=csv_field_names) writer = csv.DictWriter(f, fieldnames=csv_field_names)
@ -65,13 +68,13 @@ def main():
for index in indices: for index in indices:
resp = es.search(index=index, body=body) resp = es.search(index=index, body=body)
aggs = resp.get('aggregations') aggs = resp.get('aggregations')
print('{:22} count: {:6}'.format(index, len(aggs["peers"]["buckets"]))) print('{:22} count: {:6}'.format(index, len(aggs['peers']['buckets'])))
for bucket in aggs["peers"]["buckets"]: for bucket in aggs['peers']['buckets']:
writer.writerow({ writer.writerow({
"date": remove_prefix(index, 'logstash-'), 'date': remove_prefix(index, 'logstash-'),
"peer": bucket["key"], 'peer': hash_string(bucket['key']),
"count": bucket["doc_count"], 'count': bucket['doc_count'],
}) })
if __name__ == '__main__': if __name__ == '__main__':