initial commit of prometheus whisper monitoring

prometheus.py

@@ -0,0 +1,141 @@
+#!/usr/bin/env python3
+
+
+# sum(shh_messages_total) by (chat)
+
+
+def openPrometheusURL(query, timestep):
+    from time import time
+    from urllib.request import urlopen
+
+    # TODO: could escape it, but could also just avoid queries which need escaping
+
+    cur_time = time()
+    return urlopen(
+        "http://master-01.do-ams3.metrics.hq.tinc:9090/api/v1/query_range?query=%s&end=%d&start=%d&step=%s"
+        % (query, time(), time() - 3600 * 24 * 60, timestep)
+    ).read()
+
+
+def parse_data(raw_json):
+    from json import loads
+
+    j = loads(raw_json)
+    assert j["status"] == "success"
+    assert j["data"]["resultType"] == "matrix"
+    return j["data"]["result"][0]["values"]
+
+
+def getData(pj):
+    from datetime import datetime
+
+    return list(
+        zip(
+            *[
+                (datetime.fromtimestamp(z[0]), float(z[1]))
+                for z in sorted(pj, key=lambda _: _[0])
+            ]
+        )
+    )
+
+
+def plotData(filename, ylabel, timespan, x, y):
+    # TODO: factor out as common routine
+    # allow this to run on a headless server
+    import matplotlib
+
+    matplotlib.use("Agg")
+
+    import matplotlib.pyplot as plt
+    import matplotlib.dates as mdates
+
+    assert len(x) == len(y)
+
+    # https://stackoverflow.com/questions/9627686/plotting-dates-on-the-x-axis-with-pythons-matplotlib#9627970
+    plt.clf()
+    plt.gca().xaxis.set_major_formatter(mdates.DateFormatter("%Y-%m-%d"))
+    plt.gca().xaxis.set_major_locator(mdates.AutoDateLocator())
+    plt.xlabel("Date")
+
+    plt.ylabel(ylabel)
+
+    plt.title("Status.im %s %s (Prometheus)" % (timespan, ylabel))
+
+    plt.plot(x, y)
+    plt.gcf().autofmt_xdate()
+    plt.savefig(filename, dpi=150)
+
+
+def distinct_users():
+    from collections import Counter
+    from datetime import datetime
+    from json import loads
+
+    j = loads(
+        openPrometheusURL(
+            "sum(rate(shh_messages_total[1d])*3600)%20by%20(source)", "1d"
+        )
+    )
+    assert j["status"] == "success"
+    assert j["data"]["resultType"] == "matrix"
+
+    distinct_user_count = Counter()
+    for metric_values in j["data"]["result"]:
+        source = metric_values["metric"].get("source", "")
+        values = metric_values["values"]
+        for epoch_time, hourly_rate in values:
+            if float(hourly_rate) > 0:
+                distinct_user_count[datetime.fromtimestamp(epoch_time)] += 1
+
+    # TODO: refactor, etc
+    return list(zip(*sorted(distinct_user_count.items())))
+
+
+def main():
+    from os.path import join
+    from sys import argv
+
+    getPath = lambda n: join(argv[1], n)
+
+    plotData(
+        getPath("whisper_distinct_users_daily.png"),
+        "Whisper Users (Public Channels)",
+        "Daily",
+        *distinct_users()
+    )
+
+    plotData(
+        getPath("whisper_messages_hourly.png"),
+        "Whisper Messages (Public Channels)",
+        "Hourly",
+        *getData(
+            parse_data(
+                openPrometheusURL("sum(rate(shh_messages_total[1h]))*3600", "1h")
+            )
+        )
+    )
+
+    plotData(
+        getPath("whisper_messages_daily.png"),
+        "Whisper Messages (Public Channels)",
+        "Daily",
+        *getData(
+            parse_data(
+                openPrometheusURL("sum(rate(shh_messages_total[1d]))*3600*24", "1d")
+            )
+        )
+    )
+
+    plotData(
+        getPath("whisper_messages_weekly.png"),
+        "Whisper Messages (Public Channels)",
+        "Weekly",
+        *getData(
+            parse_data(
+                openPrometheusURL("sum(rate(shh_messages_total[1w]))*3600*24*7", "1w")
+            )
+        )
+    )
+
+
+main()