Added Basic Histograms

This commit is contained in:
Nile Walker 2020-12-16 11:00:01 -05:00
parent bc5afcb754
commit 09e53a3abf
6 changed files with 313 additions and 125 deletions

View File

@ -1,6 +1,163 @@
{
"cells": [],
"metadata": {},
"cells": [
{
"cell_type": "code",
"execution_count": 36,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"(array([16, 11, 6, 10, 3, 7, 17, 13, 9, 8]), array([1.58281495e+18, 1.58288336e+18, 1.58295176e+18, 1.58302017e+18,\n",
" 1.58308857e+18, 1.58315698e+18, 1.58322538e+18, 1.58329379e+18,\n",
" 1.58336219e+18, 1.58343060e+18, 1.58349900e+18])) 10\n"
]
},
{
"ename": "NameError",
"evalue": "name 'ax' is not defined",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-36-f7ba04e4ddbf>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 14\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 15\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 16\u001b[0;31m \u001b[0mlabels\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0max\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_xticks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtolist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 17\u001b[0m \u001b[0mlabels\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_datetime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlabels\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 18\u001b[0m \u001b[0max\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_xticklabels\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlabels\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrotation\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m90\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mNameError\u001b[0m: name 'ax' is not defined"
]
}
],
"source": [
"import pandas as pd\n",
"import numpy as np\n",
"\n",
"import matplotlib.pyplot as plt\n",
"\n",
"\n",
"\n",
"serie = pd.Series([0.0,950.0,-70.0,812.0,0.0,-90.0,0.0,0.0,-90.0,0.0,-64.0,208.0,0.0,-90.0,0.0,-80.0,0.0,0.0,-80.0,-48.0,840.0,-100.0,190.0,130.0,-100.0,-100.0,0.0,-50.0,0.0,-100.0,-100.0,0.0,-90.0,0.0,-90.0,-90.0,63.0,-90.0,0.0,0.0,-90.0,-80.0,0.0,])\n",
"\n",
"df = [i for i in np.random.randint(1582800000000000000, 1583500000000000000, 100, dtype=np.int64)]\n",
"\n",
"print(np.histogram(np.array(df).astype(np.int64)),10)\n",
"\n",
"\n",
"\n",
"labels = ax.get_xticks().tolist()\n",
"labels = pd.to_datetime(labels)\n",
"ax.set_xticklabels(labels, rotation=90)\n",
"plt.show()\n",
"print(x)\n"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[1.0, 3.0, 5.0, 7.0, 9.0]"
]
},
"execution_count": 19,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"a = [0,2,4,6,8,10]\n",
"[a[i]+(a[i+1]-a[i])/2 for i in range(len(a)-1)]"
]
},
{
"cell_type": "code",
"execution_count": 37,
"metadata": {},
"outputs": [],
"source": [
"import datetime \n",
"\n",
"dt = datetime.datetime.now()\n"
]
},
{
"cell_type": "code",
"execution_count": 25,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"numpy.datetime64('2020-02-28T20:41:46.702000674')"
]
},
"execution_count": 25,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"df[0].to_datetime64()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"dt.timestamp()"
]
},
{
"cell_type": "code",
"execution_count": 39,
"metadata": {},
"outputs": [
{
"ename": "TypeError",
"evalue": "int() argument must be a string, a bytes-like object or a number, not 'datetime.datetime'",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-39-8f76617de29c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdt\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
"\u001b[0;31mTypeError\u001b[0m: int() argument must be a string, a bytes-like object or a number, not 'datetime.datetime'"
]
}
],
"source": [
"int(dt)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.3"
}
},
"nbformat": 4,
"nbformat_minor": 4
}

View File

@ -20,6 +20,8 @@ from sentry_sdk.integrations.flask import FlaskIntegration
from webassets import Bundle
from flask_executor import Executor
import numpy as np
import matplotlib.pyplot as plt
logging.basicConfig(level=logging.INFO)
@ -73,12 +75,13 @@ from communicator import models
from communicator import api
from communicator import forms
import random
from communicator import scheduler
# from communicator import scheduler
connexion_app.add_api('api.yml', base_path='/v1.0')
# Convert list of allowed origins to list of regexes
origins_re = [r"^https?:\/\/%s(.*)" % o.replace('.', '\.') for o in app.config['CORS_ALLOW_ORIGINS']]
origins_re = [r"^https?:\/\/%s(.*)" % o.replace('.', '\.')
for o in app.config['CORS_ALLOW_ORIGINS']]
cors = CORS(connexion_app.app, origins=origins_re)
# Sentry error handling
@ -89,16 +92,18 @@ if app.config['ENABLE_SENTRY']:
traces_sample_rate=1.0
)
### HTML Pages
# HTML Pages
BASE_HREF = app.config['APPLICATION_ROOT'].strip('/')
def superuser(f):
@wraps(f)
def decorated_function(*args, **kwargs):
from communicator.services.user_service import UserService
if not UserService().is_valid_user():
flash("You do not have permission to view that page", "warning")
logging.info("Permission Denied to user " + UserService().get_user_info())
logging.info("Permission Denied to user " +
UserService().get_user_info())
abort(404)
return f(*args, **kwargs)
return decorated_function
@ -117,7 +122,7 @@ def index():
samples = db.session.query(Sample).order_by(Sample.date.desc())
if request.method == "POST" or request.args.get('cancel') == 'true':
session["index_filter"] = {} # Clear out the session if it is invalid.
session["index_filter"] = {} # Clear out the session if it is invalid.
if form.validate():
session["index_filter"] = {}
@ -146,15 +151,19 @@ def index():
if "end_date" in filters:
samples = samples.filter(Sample.date <= filters["end_date"])
if "student_id" in filters:
samples = samples.filter(Sample.student_id.in_(filters["student_id"].split()))
samples = samples.filter(
Sample.student_id.in_(filters["student_id"].split()))
if "location" in filters:
samples = samples.filter(Sample.location.in_(filters["location"].split()))
samples = samples.filter(
Sample.location.in_(filters["location"].split()))
if "email" in filters:
samples = samples.filter(Sample.email.ilike(filters["email"] + "%"))
samples = samples.filter(
Sample.email.ilike(filters["email"] + "%"))
except Exception as e:
logging.error("Encountered an error building filters, so clearing. " + e)
logging.error(
"Encountered an error building filters, so clearing. " + e)
session["index_filter"] = {}
# display results
if download:
csv = __make_csv(samples)
@ -162,61 +171,76 @@ def index():
else:
page = request.args.get(get_page_parameter(), type=int, default=1)
pagination = Pagination(page=page, total=samples.count(), search=False, record_name='samples')
table = SampleTable(samples.paginate(page,10,error_out=False).items)
pagination = Pagination(page=page, total=samples.count(
), search=False, record_name='samples', css_framework='bootstrap4')
table = SampleTable(samples.paginate(page, 10, error_out=False).items)
chart_data = {"datasets": []}
active_stations = ["10","20","30","40","50","60"]
# Get Active Locations Info
active_stations = ["10", "20", "30", "40", "50", "60"]
# https://stackoverflow.com/questions/19442224/getting-information-for-bins-in-matplotlib-histogram-function
# Seperate Data
location_data = dict()
sample_times = dict()
for entry in samples:
loc_code = str(entry.location)[:2]
if loc_code not in location_data:
location_data[loc_code] = [entry]
sample_times[loc_code] = [entry.date.timestamp()]
logging.info(entry.date)
else:
location_data[loc_code].append(entry)
sample_times[loc_code].append(entry.date.timestamp())
# Analysis
i = 0
i = 0
for loc_code in location_data.keys():
chart_data["datasets"].append({
"label":loc_code,
"borderColor": f'rgba(255,{i*50},{i*10},.7)',
"pointBorderColor":f'rgba(255,{i*50},{i*10},1)',
"borderWidth": 10,
"data": [{
"x": location_data[loc_code][0].date, "y": i
}, {
"x": location_data[loc_code][-1].date, "y": i
},],
})
i+=1
data_dict = dict({
"label": loc_code,
"borderColor": f'rgba(255,{i*50},{i*20},.7)',
"pointBorderColor": f'rgba(255,{i*50},{i*20},1)',
"borderWidth": 10,
"data": [],
})
hist, bin_edges = np.histogram(np.array(sample_times[loc_code]))#, dtype = np.int64))
#bin_edges = [datetime.fromtimestamp(date) for date in bin_edges]
bins = [bin_edges[i]+(bin_edges[i+1]-bin_edges[i])/2 for i in range(len(bin_edges)-1)]
for cnt, date in zip(hist,bins):
data_dict["data"].append({
"x": datetime.utcfromtimestamp(date), "y": int(cnt)
})
chart_data["datasets"].append(data_dict)
i += 1
# Check for Unresponsive
for loc_code in active_stations:
if loc_code not in location_data:
chart_data["datasets"].append({
"label":loc_code,
"borderColor": f'rgba(128,128,128,.7)',
"pointBorderColor":f'rgba(128,128,128,1)',
"borderWidth": 10,
"data": [{
"x": session["index_filter"]["start_date"], "y": i
}, {
"x": session["index_filter"]["start_date"], "y": i
},],
})
i+=1
return render_template('layouts/default.html',
base_href=BASE_HREF,
content = render_template(
'pages/index.html',
form=form,
table=table,
action=action,
pagination=pagination,
description_map={},
chart_data= chart_data
))
"label": loc_code,
"borderColor": f'rgba(128,128,128,.7)',
"pointBorderColor": f'rgba(128,128,128,1)',
"borderWidth": 10,
"data": [{
"x": session["index_filter"]["start_date"], "y": i
}, ],
})
i += 1
return render_template('layouts/default.html',
base_href=BASE_HREF,
content=render_template(
'pages/index.html',
form=form,
table=table,
action=action,
pagination=pagination,
description_map={},
chart_data=chart_data
))
def __make_csv(sample_query):
@ -260,7 +284,6 @@ def __make_csv(sample_query):
return mem
@app.route('/invitation', methods=['GET', 'POST'])
@superuser
def send_invitation():
@ -273,14 +296,17 @@ def send_invitation():
if request.method == 'POST' and form.validate():
from communicator.services.notification_service import NotificationService
with NotificationService(app) as ns:
ns.send_invitations(form.date.data, form.location.data, form.emails.data)
ns.send_invitations(
form.date.data, form.location.data, form.emails.data)
return redirect(url_for('send_invitation'))
# display results
page = request.args.get(get_page_parameter(), type=int, default=1)
invites = db.session.query(Invitation).order_by(Invitation.date_sent.desc())
pagination = Pagination(page=page, total=invites.count(), search=False, record_name='samples')
invites = db.session.query(Invitation).order_by(
Invitation.date_sent.desc())
pagination = Pagination(page=page, total=invites.count(),
search=False, record_name='samples')
table = InvitationTable(invites.paginate(page,10,error_out=False).items)
table = InvitationTable(invites.paginate(page, 10, error_out=False).items)
return render_template(
'form.html',
@ -293,6 +319,7 @@ def send_invitation():
base_href=BASE_HREF
)
@app.route('/imported_files', methods=['GET'])
@superuser
def list_imported_files_from_ivy():
@ -302,7 +329,8 @@ def list_imported_files_from_ivy():
# display results
page = request.args.get(get_page_parameter(), type=int, default=1)
files = db.session.query(IvyFile).order_by(IvyFile.date_added.desc())
pagination = Pagination(page=page, total=files.count(), search=False, record_name='samples')
pagination = Pagination(page=page, total=files.count(),
search=False, record_name='samples')
table = IvyFileTable(files.paginate(page, 10, error_out=False).items)
return render_template(
@ -312,6 +340,7 @@ def list_imported_files_from_ivy():
base_href=BASE_HREF
)
@app.route('/sso')
def sso():
from communicator.services.user_service import UserService
@ -320,6 +349,7 @@ def sso():
response += f"<h1>Current User: {user.display_name} ({user.uid})</h1>"
return response
@app.route('/debug-sentry')
def trigger_error():
division_by_zero = 1 / 0
@ -340,6 +370,7 @@ def count_files_in_ivy():
count = ivy_service.get_file_count_from_globus()
print(f"There are {count} files awaiting transfer")
@app.cli.command()
def transfer():
from communicator.services.ivy_service import IvyService
@ -352,12 +383,3 @@ def delete():
from communicator.services.ivy_service import IvyService
ivy_service = IvyService()
ivy_service.delete_file()
# {% for location in locations %}
# <li class="nav-item " data-toggle="chart" data-target="#chart-sales" data-update='{{location.data}}' data-suffix="e2">
# <a href="#" class="nav-link py-2 px-3" name = "location" value="{{location.name}}" type="radio" data-toggle="tab">
# <span class="d-none d-md-block">{{location.name}}</span>
# </a>
# </li>
# {% endfor %}

View File

@ -24,7 +24,7 @@
</li>
<li class="nav-item">
<a href="https://blog.appseed.us/flask-argon-dashboard-zero-to-full-stack"
class="nav-link">Blog</a>
class="nav-link">Company Site</a>
</li>
</ul>
</div>

View File

@ -2,7 +2,7 @@
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
<meta name="author" content="AppSeed App Generator">
<!-- <meta name="author" content="AppSeed App Generator"> -->
<link href="/static/assets/img/brand/favicon.png" rel="icon" type="image/png">
<link href="https://fonts.googleapis.com/css?family=Open+Sans:300,400,600,700" rel="stylesheet">

View File

@ -1,10 +1,11 @@
<!-- Argon Scripts -->
<!-- Core --><script src="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/js/bootstrap.min.js" integrity="sha384-JZR6Spejh4U02d8jOt6vLEHfe/JQGiRRSQQxSfFWpi1MquVdAyjUar5+76PVCmYl" crossorigin="anonymous"></script>
<!-- Core -->
<script src="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/js/bootstrap.min.js" integrity="sha384-JZR6Spejh4U02d8jOt6vLEHfe/JQGiRRSQQxSfFWpi1MquVdAyjUar5+76PVCmYl" crossorigin="anonymous"></script>
<script src="http://cdnjs.cloudflare.com/ajax/libs/jquery/2.1.3/jquery.min.js"></script>
<!-- Optional JS -->
<script src="http://cdnjs.cloudflare.com/ajax/libs/moment.js/2.13.0/moment.min.js"></script>
<script src="http://cdnjs.cloudflare.com/ajax/libs/jquery/2.1.3/jquery.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.4.0/Chart.bundle.js"></script>

View File

@ -47,55 +47,60 @@
</div>
</div>
</div>
<div class="col-xl-4">
<div class="card shadow">
<form action="{{ action }}" method="post">
{{ form.csrf_token() }}
<div class="card-header border-0">
<div class="row align-items-center">
<div class="col">
<h3 class="mb-0">Search</h3>
</div>
<div class="col text-right">
<button type="submit" class="btn btn-sm btn-primary">Run Search</button>
{{ form.csrf_token() }}
<div class="card-header border-0">
<div class="row align-items-center">
<div class="col">
<h3 class="mb-0">Search</h3>
</div>
<div class="col text-right">
<button type="submit" class="btn btn-sm btn-primary">Run Search</button>
</div>
</div>
</div>
</div>
<div class="table-responsive">
<!-- Projects table -->
<table class="table align-items-center table-flush">
<tbody>
{% for field in form if field.name != "csrf_token" %}
<tr>
<div class="table-responsive">
<div class="form-field {{ field.widget.input_type }}">
<td>
<div class="form-field-label">{{ field.label() }}:</div>
<!-- Projects table -->
<table class="table align-items-center table-flush">
<tbody>
{% for field in form if field.name != "csrf_token" %}
<tr>
<div class="form-field {{ field.widget.input_type }}">
<td>
<div class="form-field-label">{{ field.label() }}:</div>
</td>
<td>
{% if field.type == "date" %}
<input name="{{field.name}}" />
{%else%}
<div class="form-field-input">{{ field }}</div>
<div class="form-field-help">{{ description_map[field.name] }}</div>
{% for error in field.errors %}
<div class="form-field-error">{{ error }}</div>
{% endfor %}
{% endif %}
</div>
</td>
<td>
{% if field.type == "date" %}
<input name="{{field.name}}" />
{%else%}
<div class="form-field-input">{{ field }}</div>
<div class="form-field-help">{{ description_map[field.name] }}</div>
{% for error in field.errors %}
<div class="form-field-error">{{ error }}</div>
{% endfor %}
{% endif %}
</div>
</td>
</tr>
{% endfor %}
<script>
$('.datepicker').datepicker({format: 'yyyy-mm-dd',});
</script>
</tbody>
</table>
</div>
</tr>
{% endfor %}
<script>
$('.datepicker').datepicker({
showButtonPanel: true,
localToday: new Date(), // This option determines the highlighted today date
format: 'yyyy-mm-dd',
todayHighlight: true,
});
</script>
</tbody>
</table>
</div>
</form>
</div>
</div>
@ -114,14 +119,17 @@
</div>
</div>
</div>
<div class="table-responsive">
<!-- Projects table -->
{{table}}
{{ table }}
</div>
<div class="row justify-content-center">
{{ pagination.links }}
</div>
</div>
</div>
</div>
</div>
{% include 'includes/footer.html' %}
@ -132,7 +140,7 @@
var data = JSON.parse('{{ chart_data | tojson | safe}}');
var ctx = document.getElementById('chart-sales').getContext('2d');
var timeFormat = 'YYYY-MM-DD h:mm:ss.SSS';
var chart = new Chart(ctx, {
// The type of chart we want to create
type: 'line',
@ -146,16 +154,16 @@
display: false,
},
legend: {
display: true,
position: "right",
labels: {
usePointStyle: true,
fontColor: '#FFFFFF',
fontSize: 15,
padding: 20,
fontStyle: 'bold'
}
},
display: true,
position: "right",
labels: {
usePointStyle: true,
fontColor: '#FFFFFF',
fontSize: 15,
padding: 20,
fontStyle: 'bold'
}
},
scales: {
xAxes: [{
type: "time",