diff --git a/Untitled.ipynb b/Untitled.ipynb index 7fec515..5437ecb 100644 --- a/Untitled.ipynb +++ b/Untitled.ipynb @@ -1,6 +1,163 @@ { - "cells": [], - "metadata": {}, + "cells": [ + { + "cell_type": "code", + "execution_count": 36, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(array([16, 11, 6, 10, 3, 7, 17, 13, 9, 8]), array([1.58281495e+18, 1.58288336e+18, 1.58295176e+18, 1.58302017e+18,\n", + " 1.58308857e+18, 1.58315698e+18, 1.58322538e+18, 1.58329379e+18,\n", + " 1.58336219e+18, 1.58343060e+18, 1.58349900e+18])) 10\n" + ] + }, + { + "ename": "NameError", + "evalue": "name 'ax' is not defined", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 14\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 15\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 16\u001b[0;31m \u001b[0mlabels\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0max\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_xticks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtolist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 17\u001b[0m \u001b[0mlabels\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_datetime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlabels\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 18\u001b[0m \u001b[0max\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_xticklabels\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlabels\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrotation\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m90\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mNameError\u001b[0m: name 'ax' is not defined" + ] + } + ], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "\n", + "import matplotlib.pyplot as plt\n", + "\n", + "\n", + "\n", + "serie = pd.Series([0.0,950.0,-70.0,812.0,0.0,-90.0,0.0,0.0,-90.0,0.0,-64.0,208.0,0.0,-90.0,0.0,-80.0,0.0,0.0,-80.0,-48.0,840.0,-100.0,190.0,130.0,-100.0,-100.0,0.0,-50.0,0.0,-100.0,-100.0,0.0,-90.0,0.0,-90.0,-90.0,63.0,-90.0,0.0,0.0,-90.0,-80.0,0.0,])\n", + "\n", + "df = [i for i in np.random.randint(1582800000000000000, 1583500000000000000, 100, dtype=np.int64)]\n", + "\n", + "print(np.histogram(np.array(df).astype(np.int64)),10)\n", + "\n", + "\n", + "\n", + "labels = ax.get_xticks().tolist()\n", + "labels = pd.to_datetime(labels)\n", + "ax.set_xticklabels(labels, rotation=90)\n", + "plt.show()\n", + "print(x)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[1.0, 3.0, 5.0, 7.0, 9.0]" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "a = [0,2,4,6,8,10]\n", + "[a[i]+(a[i+1]-a[i])/2 for i in range(len(a)-1)]" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "metadata": {}, + "outputs": [], + "source": [ + "import datetime \n", + "\n", + "dt = datetime.datetime.now()\n" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "numpy.datetime64('2020-02-28T20:41:46.702000674')" + ] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df[0].to_datetime64()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "dt.timestamp()" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "metadata": {}, + "outputs": [ + { + "ename": "TypeError", + "evalue": "int() argument must be a string, a bytes-like object or a number, not 'datetime.datetime'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdt\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;31mTypeError\u001b[0m: int() argument must be a string, a bytes-like object or a number, not 'datetime.datetime'" + ] + } + ], + "source": [ + "int(dt)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.3" + } + }, "nbformat": 4, "nbformat_minor": 4 } diff --git a/communicator/__init__.py b/communicator/__init__.py index 4c90fe8..a444d3c 100644 --- a/communicator/__init__.py +++ b/communicator/__init__.py @@ -20,6 +20,8 @@ from sentry_sdk.integrations.flask import FlaskIntegration from webassets import Bundle from flask_executor import Executor +import numpy as np +import matplotlib.pyplot as plt logging.basicConfig(level=logging.INFO) @@ -73,12 +75,13 @@ from communicator import models from communicator import api from communicator import forms import random -from communicator import scheduler +# from communicator import scheduler connexion_app.add_api('api.yml', base_path='/v1.0') # Convert list of allowed origins to list of regexes -origins_re = [r"^https?:\/\/%s(.*)" % o.replace('.', '\.') for o in app.config['CORS_ALLOW_ORIGINS']] +origins_re = [r"^https?:\/\/%s(.*)" % o.replace('.', '\.') + for o in app.config['CORS_ALLOW_ORIGINS']] cors = CORS(connexion_app.app, origins=origins_re) # Sentry error handling @@ -89,16 +92,18 @@ if app.config['ENABLE_SENTRY']: traces_sample_rate=1.0 ) -### HTML Pages +# HTML Pages BASE_HREF = app.config['APPLICATION_ROOT'].strip('/') + def superuser(f): @wraps(f) def decorated_function(*args, **kwargs): from communicator.services.user_service import UserService if not UserService().is_valid_user(): flash("You do not have permission to view that page", "warning") - logging.info("Permission Denied to user " + UserService().get_user_info()) + logging.info("Permission Denied to user " + + UserService().get_user_info()) abort(404) return f(*args, **kwargs) return decorated_function @@ -117,7 +122,7 @@ def index(): samples = db.session.query(Sample).order_by(Sample.date.desc()) if request.method == "POST" or request.args.get('cancel') == 'true': - session["index_filter"] = {} # Clear out the session if it is invalid. + session["index_filter"] = {} # Clear out the session if it is invalid. if form.validate(): session["index_filter"] = {} @@ -146,15 +151,19 @@ def index(): if "end_date" in filters: samples = samples.filter(Sample.date <= filters["end_date"]) if "student_id" in filters: - samples = samples.filter(Sample.student_id.in_(filters["student_id"].split())) + samples = samples.filter( + Sample.student_id.in_(filters["student_id"].split())) if "location" in filters: - samples = samples.filter(Sample.location.in_(filters["location"].split())) + samples = samples.filter( + Sample.location.in_(filters["location"].split())) if "email" in filters: - samples = samples.filter(Sample.email.ilike(filters["email"] + "%")) + samples = samples.filter( + Sample.email.ilike(filters["email"] + "%")) except Exception as e: - logging.error("Encountered an error building filters, so clearing. " + e) + logging.error( + "Encountered an error building filters, so clearing. " + e) session["index_filter"] = {} - + # display results if download: csv = __make_csv(samples) @@ -162,61 +171,76 @@ def index(): else: page = request.args.get(get_page_parameter(), type=int, default=1) - pagination = Pagination(page=page, total=samples.count(), search=False, record_name='samples') - - table = SampleTable(samples.paginate(page,10,error_out=False).items) + pagination = Pagination(page=page, total=samples.count( + ), search=False, record_name='samples', css_framework='bootstrap4') + + table = SampleTable(samples.paginate(page, 10, error_out=False).items) chart_data = {"datasets": []} - active_stations = ["10","20","30","40","50","60"] + # Get Active Locations Info + active_stations = ["10", "20", "30", "40", "50", "60"] + # https://stackoverflow.com/questions/19442224/getting-information-for-bins-in-matplotlib-histogram-function + + # Seperate Data location_data = dict() + sample_times = dict() + for entry in samples: loc_code = str(entry.location)[:2] if loc_code not in location_data: location_data[loc_code] = [entry] + sample_times[loc_code] = [entry.date.timestamp()] + logging.info(entry.date) else: location_data[loc_code].append(entry) + sample_times[loc_code].append(entry.date.timestamp()) # Analysis - i = 0 + i = 0 for loc_code in location_data.keys(): - chart_data["datasets"].append({ - "label":loc_code, - "borderColor": f'rgba(255,{i*50},{i*10},.7)', - "pointBorderColor":f'rgba(255,{i*50},{i*10},1)', - "borderWidth": 10, - "data": [{ - "x": location_data[loc_code][0].date, "y": i - }, { - "x": location_data[loc_code][-1].date, "y": i - },], - }) - i+=1 + data_dict = dict({ + "label": loc_code, + "borderColor": f'rgba(255,{i*50},{i*20},.7)', + "pointBorderColor": f'rgba(255,{i*50},{i*20},1)', + "borderWidth": 10, + "data": [], + }) + + hist, bin_edges = np.histogram(np.array(sample_times[loc_code]))#, dtype = np.int64)) + #bin_edges = [datetime.fromtimestamp(date) for date in bin_edges] + bins = [bin_edges[i]+(bin_edges[i+1]-bin_edges[i])/2 for i in range(len(bin_edges)-1)] + + for cnt, date in zip(hist,bins): + data_dict["data"].append({ + "x": datetime.utcfromtimestamp(date), "y": int(cnt) + }) + + chart_data["datasets"].append(data_dict) + i += 1 # Check for Unresponsive for loc_code in active_stations: if loc_code not in location_data: chart_data["datasets"].append({ - "label":loc_code, - "borderColor": f'rgba(128,128,128,.7)', - "pointBorderColor":f'rgba(128,128,128,1)', - "borderWidth": 10, - "data": [{ - "x": session["index_filter"]["start_date"], "y": i - }, { - "x": session["index_filter"]["start_date"], "y": i - },], - }) - i+=1 - return render_template('layouts/default.html', - base_href=BASE_HREF, - content = render_template( - 'pages/index.html', - form=form, - table=table, - action=action, - pagination=pagination, - description_map={}, - chart_data= chart_data - )) + "label": loc_code, + "borderColor": f'rgba(128,128,128,.7)', + "pointBorderColor": f'rgba(128,128,128,1)', + "borderWidth": 10, + "data": [{ + "x": session["index_filter"]["start_date"], "y": i + }, ], + }) + i += 1 + return render_template('layouts/default.html', + base_href=BASE_HREF, + content=render_template( + 'pages/index.html', + form=form, + table=table, + action=action, + pagination=pagination, + description_map={}, + chart_data=chart_data + )) def __make_csv(sample_query): @@ -260,7 +284,6 @@ def __make_csv(sample_query): return mem - @app.route('/invitation', methods=['GET', 'POST']) @superuser def send_invitation(): @@ -273,14 +296,17 @@ def send_invitation(): if request.method == 'POST' and form.validate(): from communicator.services.notification_service import NotificationService with NotificationService(app) as ns: - ns.send_invitations(form.date.data, form.location.data, form.emails.data) + ns.send_invitations( + form.date.data, form.location.data, form.emails.data) return redirect(url_for('send_invitation')) # display results page = request.args.get(get_page_parameter(), type=int, default=1) - invites = db.session.query(Invitation).order_by(Invitation.date_sent.desc()) - pagination = Pagination(page=page, total=invites.count(), search=False, record_name='samples') + invites = db.session.query(Invitation).order_by( + Invitation.date_sent.desc()) + pagination = Pagination(page=page, total=invites.count(), + search=False, record_name='samples') - table = InvitationTable(invites.paginate(page,10,error_out=False).items) + table = InvitationTable(invites.paginate(page, 10, error_out=False).items) return render_template( 'form.html', @@ -293,6 +319,7 @@ def send_invitation(): base_href=BASE_HREF ) + @app.route('/imported_files', methods=['GET']) @superuser def list_imported_files_from_ivy(): @@ -302,7 +329,8 @@ def list_imported_files_from_ivy(): # display results page = request.args.get(get_page_parameter(), type=int, default=1) files = db.session.query(IvyFile).order_by(IvyFile.date_added.desc()) - pagination = Pagination(page=page, total=files.count(), search=False, record_name='samples') + pagination = Pagination(page=page, total=files.count(), + search=False, record_name='samples') table = IvyFileTable(files.paginate(page, 10, error_out=False).items) return render_template( @@ -312,6 +340,7 @@ def list_imported_files_from_ivy(): base_href=BASE_HREF ) + @app.route('/sso') def sso(): from communicator.services.user_service import UserService @@ -320,6 +349,7 @@ def sso(): response += f"

Current User: {user.display_name} ({user.uid})

" return response + @app.route('/debug-sentry') def trigger_error(): division_by_zero = 1 / 0 @@ -340,6 +370,7 @@ def count_files_in_ivy(): count = ivy_service.get_file_count_from_globus() print(f"There are {count} files awaiting transfer") + @app.cli.command() def transfer(): from communicator.services.ivy_service import IvyService @@ -352,12 +383,3 @@ def delete(): from communicator.services.ivy_service import IvyService ivy_service = IvyService() ivy_service.delete_file() - -# {% for location in locations %} -#
  • -# -# {{location.name}} -# -#
    • -# {% endfor %} - diff --git a/communicator/templates/includes/footer.html b/communicator/templates/includes/footer.html index cf89f96..4da63be 100644 --- a/communicator/templates/includes/footer.html +++ b/communicator/templates/includes/footer.html @@ -24,7 +24,7 @@
  • Blog + class="nav-link">Company Site
    • diff --git a/communicator/templates/includes/header.html b/communicator/templates/includes/header.html index 678d8e3..8ca1111 100644 --- a/communicator/templates/includes/header.html +++ b/communicator/templates/includes/header.html @@ -2,7 +2,7 @@ - + diff --git a/communicator/templates/includes/scripts.html b/communicator/templates/includes/scripts.html index dbae468..13f7c74 100644 --- a/communicator/templates/includes/scripts.html +++ b/communicator/templates/includes/scripts.html @@ -1,10 +1,11 @@ - - + + + - + diff --git a/communicator/templates/pages/index.html b/communicator/templates/pages/index.html index 0df35b1..759294a 100644 --- a/communicator/templates/pages/index.html +++ b/communicator/templates/pages/index.html @@ -47,55 +47,60 @@ - - + +
    - {{ form.csrf_token() }} -
    -
    -
    -

    Search

    -
    -
    - + {{ form.csrf_token() }} +
    +
    +
    +

    Search

    +
    +
    + +
    -
    -
    - - - - - {% for field in form if field.name != "csrf_token" %} - +
    -
    -
    + {% endfor %} + + +
    -
    {{ field.label() }}:
    + + + + {% for field in form if field.name != "csrf_token" %} + + +
    +
    + - - - {% endfor %} - - -
    +
    {{ field.label() }}:
    +     		+ {% if field.type == "date" %} + + {%else%} +
    {{ field }}
    +
    {{ description_map[field.name] }}
    + {% for error in field.errors %} +
    {{ error }}
    + {% endfor %} + {% endif %} +     		- {% if field.type == "date" %} - - {%else%} -
    {{ field }}
    -
    {{ description_map[field.name] }}
    - {% for error in field.errors %} -
    {{ error }}
    - {% endfor %} - {% endif %} - -
    - +
    +
    @@ -114,14 +119,17 @@
    +
    - {{table}} + {{ table }} +
    +
    {{ pagination.links }}
    - + {% include 'includes/footer.html' %} @@ -132,7 +140,7 @@ var data = JSON.parse('{{ chart_data | tojson | safe}}'); var ctx = document.getElementById('chart-sales').getContext('2d'); var timeFormat = 'YYYY-MM-DD h:mm:ss.SSS'; - + var chart = new Chart(ctx, { // The type of chart we want to create type: 'line', @@ -146,16 +154,16 @@ display: false, }, legend: { - display: true, - position: "right", - labels: { - usePointStyle: true, - fontColor: '#FFFFFF', - fontSize: 15, - padding: 20, - fontStyle: 'bold' - } - }, + display: true, + position: "right", + labels: { + usePointStyle: true, + fontColor: '#FFFFFF', + fontSize: 15, + padding: 20, + fontStyle: 'bold' + } + }, scales: { xAxes: [{ type: "time",