Added Basic Histograms

2025-02-23 12:28:26 +00:00 · 2020-12-16 11:00:01 -05:00 · 2020-12-16 11:00:01 -05:00 · 09e53a3abf
commit 09e53a3abf
parent bc5afcb754
6 changed files with 313 additions and 125 deletions
--- a/Untitled.ipynb
+++ b/Untitled.ipynb
@ -1,6 +1,163 @@
 {
- "cells": [],
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 36,
   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "(array([16, 11,  6, 10,  3,  7, 17, 13,  9,  8]), array([1.58281495e+18, 1.58288336e+18, 1.58295176e+18, 1.58302017e+18,\n",
+      "       1.58308857e+18, 1.58315698e+18, 1.58322538e+18, 1.58329379e+18,\n",
+      "       1.58336219e+18, 1.58343060e+18, 1.58349900e+18])) 10\n"
+     ]
+    },
+    {
+     "ename": "NameError",
+     "evalue": "name 'ax' is not defined",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
+      "\u001b[0;32m<ipython-input-36-f7ba04e4ddbf>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     14\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     15\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 16\u001b[0;31m \u001b[0mlabels\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0max\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_xticks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtolist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     17\u001b[0m \u001b[0mlabels\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_datetime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlabels\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     18\u001b[0m \u001b[0max\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_xticklabels\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlabels\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrotation\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m90\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;31mNameError\u001b[0m: name 'ax' is not defined"
+     ]
+    }
+   ],
+   "source": [
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "\n",
+    "import matplotlib.pyplot as plt\n",
+    "\n",
+    "\n",
+    "\n",
+    "serie = pd.Series([0.0,950.0,-70.0,812.0,0.0,-90.0,0.0,0.0,-90.0,0.0,-64.0,208.0,0.0,-90.0,0.0,-80.0,0.0,0.0,-80.0,-48.0,840.0,-100.0,190.0,130.0,-100.0,-100.0,0.0,-50.0,0.0,-100.0,-100.0,0.0,-90.0,0.0,-90.0,-90.0,63.0,-90.0,0.0,0.0,-90.0,-80.0,0.0,])\n",
+    "\n",
+    "df =  [i for i in np.random.randint(1582800000000000000, 1583500000000000000, 100, dtype=np.int64)]\n",
+    "\n",
+    "print(np.histogram(np.array(df).astype(np.int64)),10)\n",
+    "\n",
+    "\n",
+    "\n",
+    "labels = ax.get_xticks().tolist()\n",
+    "labels = pd.to_datetime(labels)\n",
+    "ax.set_xticklabels(labels, rotation=90)\n",
+    "plt.show()\n",
+    "print(x)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[1.0, 3.0, 5.0, 7.0, 9.0]"
+      ]
+     },
+     "execution_count": 19,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "a = [0,2,4,6,8,10]\n",
+    "[a[i]+(a[i+1]-a[i])/2 for i in range(len(a)-1)]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 37,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import datetime \n",
+    "\n",
+    "dt = datetime.datetime.now()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "numpy.datetime64('2020-02-28T20:41:46.702000674')"
+      ]
+     },
+     "execution_count": 25,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df[0].to_datetime64()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "dt.timestamp()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 39,
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "TypeError",
+     "evalue": "int() argument must be a string, a bytes-like object or a number, not 'datetime.datetime'",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
+      "\u001b[0;32m<ipython-input-39-8f76617de29c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdt\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
+      "\u001b[0;31mTypeError\u001b[0m: int() argument must be a string, a bytes-like object or a number, not 'datetime.datetime'"
+     ]
+    }
+   ],
+   "source": [
+    "int(dt)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.3"
+  }
+ },
 "nbformat": 4,
 "nbformat_minor": 4
 }
--- a/communicator/init.py
+++ b/communicator/init.py
@ -20,6 +20,8 @@ from sentry_sdk.integrations.flask import FlaskIntegration
 from webassets import Bundle
 from flask_executor import Executor

+import numpy as np
+import matplotlib.pyplot as plt

 logging.basicConfig(level=logging.INFO)

@ -73,12 +75,13 @@ from communicator import models
 from communicator import api
 from communicator import forms
 import random 
-from communicator import scheduler
+# from communicator import scheduler

 connexion_app.add_api('api.yml', base_path='/v1.0')

 # Convert list of allowed origins to list of regexes
-origins_re = [r"^https?:\/\/%s(.*)" % o.replace('.', '\.') for o in app.config['CORS_ALLOW_ORIGINS']]
+origins_re = [r"^https?:\/\/%s(.*)" % o.replace('.', '\.')
+              for o in app.config['CORS_ALLOW_ORIGINS']]
 cors = CORS(connexion_app.app, origins=origins_re)

 # Sentry error handling
@ -89,16 +92,18 @@ if app.config['ENABLE_SENTRY']:
        traces_sample_rate=1.0
    )

-### HTML Pages
+# HTML Pages
 BASE_HREF = app.config['APPLICATION_ROOT'].strip('/')

+
 def superuser(f):
    @wraps(f)
    def decorated_function(*args, **kwargs):
        from communicator.services.user_service import UserService
        if not UserService().is_valid_user():
            flash("You do not have permission to view that page", "warning")
-            logging.info("Permission Denied to user " + UserService().get_user_info())
+            logging.info("Permission Denied to user " +
+                         UserService().get_user_info())
            abort(404)
        return f(*args, **kwargs)
    return decorated_function
@ -146,13 +151,17 @@ def index():
            if "end_date" in filters:
                samples = samples.filter(Sample.date <= filters["end_date"])
            if "student_id" in filters:
-                samples = samples.filter(Sample.student_id.in_(filters["student_id"].split()))
+                samples = samples.filter(
+                    Sample.student_id.in_(filters["student_id"].split()))
            if "location" in filters:
-                samples = samples.filter(Sample.location.in_(filters["location"].split()))
+                samples = samples.filter(
+                    Sample.location.in_(filters["location"].split()))
            if "email" in filters:
-                samples = samples.filter(Sample.email.ilike(filters["email"] + "%"))
+                samples = samples.filter(
+                    Sample.email.ilike(filters["email"] + "%"))
        except Exception as e:
-            logging.error("Encountered an error building filters, so clearing. " + e)
+            logging.error(
+                "Encountered an error building filters, so clearing. " + e)
            session["index_filter"] = {}

    # display results
@ -162,60 +171,75 @@ def index():

    else:
        page = request.args.get(get_page_parameter(), type=int, default=1)
-        pagination = Pagination(page=page, total=samples.count(), search=False, record_name='samples')
+        pagination = Pagination(page=page, total=samples.count(
+        ), search=False, record_name='samples', css_framework='bootstrap4')

-        table = SampleTable(samples.paginate(page,10,error_out=False).items)
+        table = SampleTable(samples.paginate(page, 10, error_out=False).items)

        chart_data = {"datasets": []}
-        active_stations = ["10","20","30","40","50","60"]
+        # Get Active Locations Info 
+        active_stations = ["10", "20", "30", "40", "50", "60"]
+        # https://stackoverflow.com/questions/19442224/getting-information-for-bins-in-matplotlib-histogram-function
+       
+
        # Seperate Data
        location_data = dict()
+        sample_times = dict()
+
        for entry in samples:
            loc_code = str(entry.location)[:2]
            if loc_code not in location_data:
                location_data[loc_code] = [entry]
+                sample_times[loc_code] = [entry.date.timestamp()]
+                logging.info(entry.date)
            else:
                location_data[loc_code].append(entry)
+                sample_times[loc_code].append(entry.date.timestamp())
        # Analysis
        i = 0
        for loc_code in location_data.keys():
-            chart_data["datasets"].append({
-                "label":loc_code,
-                    "borderColor": f'rgba(255,{i*50},{i*10},.7)',
-                    "pointBorderColor":f'rgba(255,{i*50},{i*10},1)',
+            data_dict = dict({
+                "label": loc_code,
+                "borderColor": f'rgba(255,{i*50},{i*20},.7)',
+                "pointBorderColor": f'rgba(255,{i*50},{i*20},1)',
                "borderWidth": 10,
-                    "data": [{
-                        "x": location_data[loc_code][0].date, "y": i
-                    }, {
-                        "x": location_data[loc_code][-1].date, "y": i
-                    },],
+                "data": [],
            })
-            i+=1
+
+            hist, bin_edges = np.histogram(np.array(sample_times[loc_code]))#, dtype = np.int64))
+            #bin_edges = [datetime.fromtimestamp(date) for date in bin_edges]
+            bins = [bin_edges[i]+(bin_edges[i+1]-bin_edges[i])/2 for i in range(len(bin_edges)-1)]
+        
+            for cnt, date in zip(hist,bins):
+                data_dict["data"].append({
+                    "x": datetime.utcfromtimestamp(date), "y": int(cnt)
+                })
+
+            chart_data["datasets"].append(data_dict)
+            i += 1
        # Check for Unresponsive
        for loc_code in active_stations:
            if loc_code not in location_data:
                chart_data["datasets"].append({
-                    "label":loc_code,
+                    "label": loc_code,
                    "borderColor": f'rgba(128,128,128,.7)',
-                        "pointBorderColor":f'rgba(128,128,128,1)',
+                    "pointBorderColor": f'rgba(128,128,128,1)',
                    "borderWidth": 10,
                    "data": [{
                        "x": session["index_filter"]["start_date"], "y": i
-                        }, {
-                            "x": session["index_filter"]["start_date"], "y": i
-                        },],
+                    }, ],
                })
-            i+=1
+            i += 1
        return render_template('layouts/default.html',
                               base_href=BASE_HREF,
-        content = render_template(
+                               content=render_template(
                                   'pages/index.html',
                                   form=form,
                                   table=table,
                                   action=action,
                                   pagination=pagination,
                                   description_map={},
-            chart_data= chart_data
+                                   chart_data=chart_data
                               ))


@ -260,7 +284,6 @@ def __make_csv(sample_query):
    return mem


-
@app.route('/invitation', methods=['GET', 'POST'])
@superuser
 def send_invitation():
@ -273,14 +296,17 @@ def send_invitation():
    if request.method == 'POST' and form.validate():
        from communicator.services.notification_service import NotificationService
        with NotificationService(app) as ns:
-            ns.send_invitations(form.date.data, form.location.data, form.emails.data)
+            ns.send_invitations(
+                form.date.data, form.location.data, form.emails.data)
            return redirect(url_for('send_invitation'))
    # display results
    page = request.args.get(get_page_parameter(), type=int, default=1)
-    invites = db.session.query(Invitation).order_by(Invitation.date_sent.desc())
-    pagination = Pagination(page=page, total=invites.count(), search=False, record_name='samples')
+    invites = db.session.query(Invitation).order_by(
+        Invitation.date_sent.desc())
+    pagination = Pagination(page=page, total=invites.count(),
+                            search=False, record_name='samples')

-    table = InvitationTable(invites.paginate(page,10,error_out=False).items)
+    table = InvitationTable(invites.paginate(page, 10, error_out=False).items)

    return render_template(
        'form.html',
@ -293,6 +319,7 @@ def send_invitation():
        base_href=BASE_HREF
    )

+
@app.route('/imported_files', methods=['GET'])
@superuser
 def list_imported_files_from_ivy():
@ -302,7 +329,8 @@ def list_imported_files_from_ivy():
    # display results
    page = request.args.get(get_page_parameter(), type=int, default=1)
    files = db.session.query(IvyFile).order_by(IvyFile.date_added.desc())
-    pagination = Pagination(page=page, total=files.count(), search=False, record_name='samples')
+    pagination = Pagination(page=page, total=files.count(),
+                            search=False, record_name='samples')

    table = IvyFileTable(files.paginate(page, 10, error_out=False).items)
    return render_template(
@ -312,6 +340,7 @@ def list_imported_files_from_ivy():
        base_href=BASE_HREF
    )

+
@app.route('/sso')
 def sso():
    from communicator.services.user_service import UserService
@ -320,6 +349,7 @@ def sso():
    response += f"<h1>Current User: {user.display_name} ({user.uid})</h1>"
    return response

+
@app.route('/debug-sentry')
 def trigger_error():
    division_by_zero = 1 / 0
@ -340,6 +370,7 @@ def count_files_in_ivy():
    count = ivy_service.get_file_count_from_globus()
    print(f"There are {count} files awaiting transfer")

+
@app.cli.command()
 def transfer():
    from communicator.services.ivy_service import IvyService
@ -352,12 +383,3 @@ def delete():
    from communicator.services.ivy_service import IvyService
    ivy_service = IvyService()
    ivy_service.delete_file()
-
-# {% for location in locations %}
-#                     <li class="nav-item " data-toggle="chart" data-target="#chart-sales" data-update='{{location.data}}' data-suffix="e2">
-#                       <a href="#" class="nav-link py-2 px-3" name = "location" value="{{location.name}}" type="radio" data-toggle="tab">
-#                         <span class="d-none d-md-block">{{location.name}}</span>
-#                       </a>
-#                     </li>
-#                     {% endfor %}
-
--- a/communicator/templates/includes/footer.html
+++ b/communicator/templates/includes/footer.html
@ -24,7 +24,7 @@
              </li>
              <li class="nav-item">
                <a href="https://blog.appseed.us/flask-argon-dashboard-zero-to-full-stack" 
-                   class="nav-link">Blog</a>
+                   class="nav-link">Company Site</a>
              </li>
            </ul>
          </div>
--- a/communicator/templates/includes/header.html
+++ b/communicator/templates/includes/header.html
@ -2,7 +2,7 @@
  <meta charset="utf-8">
  <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">

-  <meta name="author" content="AppSeed App Generator">
+  <!-- <meta name="author" content="AppSeed App Generator"> -->
  
  <link href="/static/assets/img/brand/favicon.png" rel="icon" type="image/png">
  <link href="https://fonts.googleapis.com/css?family=Open+Sans:300,400,600,700" rel="stylesheet">
--- a/communicator/templates/includes/scripts.html
+++ b/communicator/templates/includes/scripts.html
@ -1,10 +1,11 @@

-  <!-- Argon Scripts -->
-  <!-- Core --><script src="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/js/bootstrap.min.js" integrity="sha384-JZR6Spejh4U02d8jOt6vLEHfe/JQGiRRSQQxSfFWpi1MquVdAyjUar5+76PVCmYl" crossorigin="anonymous"></script>
+  <!-- Core -->
+  <script src="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/js/bootstrap.min.js" integrity="sha384-JZR6Spejh4U02d8jOt6vLEHfe/JQGiRRSQQxSfFWpi1MquVdAyjUar5+76PVCmYl" crossorigin="anonymous"></script>
+  <script src="http://cdnjs.cloudflare.com/ajax/libs/jquery/2.1.3/jquery.min.js"></script>
  <!-- Optional JS -->

  <script src="http://cdnjs.cloudflare.com/ajax/libs/moment.js/2.13.0/moment.min.js"></script>
-  <script src="http://cdnjs.cloudflare.com/ajax/libs/jquery/2.1.3/jquery.min.js"></script>
+

  <script src="https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.4.0/Chart.bundle.js"></script>

--- a/communicator/templates/pages/index.html
+++ b/communicator/templates/pages/index.html
@ -91,7 +91,12 @@
                  </tr>
                  {% endfor %}
                  <script>
-                  $('.datepicker').datepicker({format: 'yyyy-mm-dd',});
+                    $('.datepicker').datepicker({
+                      showButtonPanel: true,
+                      localToday: new Date(),    // This option determines the highlighted today date
+                      format: 'yyyy-mm-dd',
+                      todayHighlight: true,
+                    });
                  </script>
                </tbody>
              </table>
@ -114,15 +119,18 @@
              </div>
            </div>
          </div>
+
          <div class="table-responsive">
            <!-- Projects table -->
-            {{table}}
+            {{ table }}
+          </div>
+          <div class="row justify-content-center">
            {{ pagination.links }}
          </div>
        </div>
-      </div>

      </div>
+    </div>

    {% include 'includes/footer.html' %}