From 8647ebee422686b5eeb2f42da7546b71a3063d52 Mon Sep 17 00:00:00 2001 From: Dan Funk Date: Wed, 20 May 2020 10:57:20 -0400 Subject: [PATCH 01/89] Moving spiff back to master branch, with fixes for parallel, and performance improvements. --- Pipfile | 2 +- Pipfile.lock | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Pipfile b/Pipfile index 6f374722..6c9960d2 100644 --- a/Pipfile +++ b/Pipfile @@ -24,7 +24,7 @@ pyjwt = "*" requests = "*" xlsxwriter = "*" webtest = "*" -spiffworkflow = {editable = true,git = "https://github.com/sartography/SpiffWorkflow.git",ref = "bug/the_horror"} +spiffworkflow = {editable = true,git = "https://github.com/sartography/SpiffWorkflow.git",ref = "master"} alembic = "*" coverage = "*" sphinx = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 38ab9ef7..904d32bd 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "bd289126c41b0f5f2761f0415d85e1110a584256460374a9ce4cda07c0033ddd" + "sha256": "26d23456010d3e5a559386d412cef3beacd92d5a4e474f2afdb0737ea0f20f04" }, "pipfile-spec": 6, "requires": { @@ -783,7 +783,7 @@ "spiffworkflow": { "editable": true, "git": "https://github.com/sartography/SpiffWorkflow.git", - "ref": "7dc54f1205de7006bdda6d966dc957e558f3c7f3" + "ref": "2c9698894f7e526a91bf3ca8c4b9fc9b6b01e807" }, "sqlalchemy": { "hashes": [ From 57615edde3ea915e17017ac0e8da58f585f3b47f Mon Sep 17 00:00:00 2001 From: Dan Funk Date: Wed, 20 May 2020 14:01:09 -0400 Subject: [PATCH 02/89] adding wsgi configuration. --- crconnect.wsgi | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 crconnect.wsgi diff --git a/crconnect.wsgi b/crconnect.wsgi new file mode 100644 index 00000000..2b1e701e --- /dev/null +++ b/crconnect.wsgi @@ -0,0 +1,4 @@ +import os +import sys + +from crc import app as application From 2c407ae607baaf6daa85001f013999ded3ad2301 Mon Sep 17 00:00:00 2001 From: Dan Funk Date: Wed, 20 May 2020 14:21:39 -0400 Subject: [PATCH 03/89] adding requirements.txt and a way to regenerate it. --- crconnect.wsgi | 11 +++++ deploy/requirements.txt | 90 +++++++++++++++++++++++++++++++++++ deploy/update_requirements.sh | 4 ++ 3 files changed, 105 insertions(+) create mode 100644 deploy/requirements.txt create mode 100755 deploy/update_requirements.sh diff --git a/crconnect.wsgi b/crconnect.wsgi index 2b1e701e..6c7ce423 100644 --- a/crconnect.wsgi +++ b/crconnect.wsgi @@ -1,4 +1,15 @@ +opython_home = '/usr/local/envs/crcpython3' + import os import sys +# Calculate path to site-packages directory. + +python_version = '.'.join(map(str, sys.version_info[:2])) +site_packages = python_home + '/lib/python%s/site-packages' % python_version + +# Add the site-packages directory. + +site.addsitedir(site_packages) + from crc import app as application diff --git a/deploy/requirements.txt b/deploy/requirements.txt new file mode 100644 index 00000000..420a888f --- /dev/null +++ b/deploy/requirements.txt @@ -0,0 +1,90 @@ +alabaster==0.7.12 +alembic==1.4.2 +amqp==2.5.2 +aniso8601==8.0.0 +attrs==19.3.0 +babel==2.8.0 +bcrypt==3.1.7 +beautifulsoup4==4.9.1 +billiard==3.6.3.0 +blinker==1.4 +celery==4.4.2 +certifi==2020.4.5.1 +cffi==1.14.0 +chardet==3.0.4 +click==7.1.2 +clickclick==1.2.2 +commonmark==0.9.1 +configparser==5.0.0 +connexion==2.7.0 +coverage==5.1 +docutils==0.16 +docxtpl==0.9.2 +et-xmlfile==1.0.1 +flask==1.1.2 +flask-bcrypt==0.7.1 +flask-cors==3.0.8 +flask-marshmallow==0.12.0 +flask-migrate==2.5.3 +flask-restful==0.3.8 +flask-sqlalchemy==2.4.1 +flask-sso==0.4.0 +future==0.18.2 +httpretty==1.0.2 +idna==2.9 +imagesize==1.2.0 +importlib-metadata==1.6.0 +inflection==0.4.0 +itsdangerous==1.1.0 +jdcal==1.4.1 +jinja2==2.11.2 +jsonschema==3.2.0 +kombu==4.6.8 +ldap3==2.7 +lxml==4.5.1 +mako==1.1.2 +markupsafe==1.1.1 +marshmallow==3.6.0 +marshmallow-enum==1.5.1 +marshmallow-sqlalchemy==0.23.0 +numpy==1.18.4 +openapi-spec-validator==0.2.8 +openpyxl==3.0.3 +packaging==20.4 +pandas==1.0.3 +psycopg2-binary==2.8.5 +pyasn1==0.4.8 +pycparser==2.20 +pygments==2.6.1 +pyjwt==1.7.1 +pyparsing==2.4.7 +pyrsistent==0.16.0 +python-dateutil==2.8.1 +python-docx==0.8.10 +python-editor==1.0.4 +pytz==2020.1 +pyyaml==5.3.1 +recommonmark==0.6.0 +requests==2.23.0 +six==1.14.0 +snowballstemmer==2.0.0 +soupsieve==2.0.1 +sphinx==3.0.3 +sphinxcontrib-applehelp==1.0.2 +sphinxcontrib-devhelp==1.0.2 +sphinxcontrib-htmlhelp==1.0.3 +sphinxcontrib-jsmath==1.0.1 +sphinxcontrib-qthelp==1.0.3 +sphinxcontrib-serializinghtml==1.1.4 +spiffworkflow +sqlalchemy==1.3.17 +swagger-ui-bundle==0.0.6 +urllib3==1.25.9 +vine==1.3.0 +waitress==1.4.3 +webob==1.8.6 +webtest==2.0.35 +werkzeug==1.0.1 +xlrd==1.2.0 +xlsxwriter==1.2.8 +zipp==3.1.0 diff --git a/deploy/update_requirements.sh b/deploy/update_requirements.sh new file mode 100755 index 00000000..adbf1142 --- /dev/null +++ b/deploy/update_requirements.sh @@ -0,0 +1,4 @@ +jq -r '.default + | to_entries[] + | .key + .value.version' \ + ../Pipfile.lock > requirements.txt From 91a228d3f4500d1658eddef1f6707eb3a6635d1e Mon Sep 17 00:00:00 2001 From: Aaron Louie Date: Wed, 20 May 2020 14:33:39 -0400 Subject: [PATCH 04/89] Trying to slim down the size of the image --- Dockerfile | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8ff7af23..e0223875 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,25 +1,18 @@ -FROM python:3.7 +FROM python:3.7-slim -ENV PATH=/root/.local/bin:/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin +WORKDIR /app -# install node and yarn -RUN apt-get update -RUN apt-get -y install postgresql-client +COPY Pipfile Pipfile.lock ./ -# config project dir -RUN mkdir /crc-workflow -WORKDIR /crc-workflow +RUN pip install pipenv && \ + apt-get update && \ + apt-get -y install --no-install-recommends gcc python3-dev libssl-dev postgresql-client && \ + pipenv install --deploy --system && \ + apt-get remove -y gcc python3-dev libssl-dev && \ + apt-get autoremove -y -# install python requirements -RUN pip install pipenv -ADD Pipfile /crc-workflow/ -ADD Pipfile.lock /crc-workflow/ -RUN pipenv install --dev +COPY app ./ -# include rejoiner code (gets overriden by local changes) -COPY . /crc-workflow/ - -# run webserver by default ENV FLASK_APP=./crc/__init__.py CMD ["pipenv", "run", "python", "./run.py"] From 97130e504aa46b203afe768e083a359c24d37d41 Mon Sep 17 00:00:00 2001 From: Aaron Louie Date: Wed, 20 May 2020 14:57:03 -0400 Subject: [PATCH 05/89] Adds git --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index e0223875..5d3d2d8a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ COPY Pipfile Pipfile.lock ./ RUN pip install pipenv && \ apt-get update && \ - apt-get -y install --no-install-recommends gcc python3-dev libssl-dev postgresql-client && \ + apt-get -y install --no-install-recommends gcc python3-dev libssl-dev postgresql-client git && \ pipenv install --deploy --system && \ apt-get remove -y gcc python3-dev libssl-dev && \ apt-get autoremove -y From 865188795f163a94d58397bff6c24aebf89f6d3e Mon Sep 17 00:00:00 2001 From: Aaron Louie Date: Wed, 20 May 2020 15:04:41 -0400 Subject: [PATCH 06/89] Adds git. Purges apt cache after --- Dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5d3d2d8a..f17074a2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,10 +6,12 @@ COPY Pipfile Pipfile.lock ./ RUN pip install pipenv && \ apt-get update && \ - apt-get -y install --no-install-recommends gcc python3-dev libssl-dev postgresql-client git && \ + apt-get install -y --no-install-recommends \ + gcc python3-dev libssl-dev postgresql-client git-core && \ pipenv install --deploy --system && \ apt-get remove -y gcc python3-dev libssl-dev && \ - apt-get autoremove -y + apt-get purge -y --auto-remove && \ + rm -rf /var/lib/apt/lists/ * COPY app ./ From 5f793db6065c189db3afb14caae760aec3b7ac4a Mon Sep 17 00:00:00 2001 From: Dan Funk Date: Wed, 20 May 2020 15:07:16 -0400 Subject: [PATCH 07/89] It is critical that the Pipfile python version matches the python version on the server. --- Pipfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Pipfile b/Pipfile index 6c9960d2..375ba042 100644 --- a/Pipfile +++ b/Pipfile @@ -38,4 +38,4 @@ xlrd = "*" ldap3 = "*" [requires] -python_version = "3.7" +python_version = "3.6.9" From 616a47bd10ad73875af3244fe1e417d24e40e7ee Mon Sep 17 00:00:00 2001 From: Aaron Louie Date: Wed, 20 May 2020 15:17:03 -0400 Subject: [PATCH 08/89] Fixes file paths --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index f17074a2..f8c26e05 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,10 +13,10 @@ RUN pip install pipenv && \ apt-get purge -y --auto-remove && \ rm -rf /var/lib/apt/lists/ * -COPY app ./ +COPY . /app -ENV FLASK_APP=./crc/__init__.py -CMD ["pipenv", "run", "python", "./run.py"] +ENV FLASK_APP=/app/crc/__init__.py +CMD ["pipenv", "run", "python", "/app/run.py"] # expose ports EXPOSE 5000 From de7cebbd7d7d349e462b8b9ee5ab863e03b4f1fa Mon Sep 17 00:00:00 2001 From: Dan Funk Date: Wed, 20 May 2020 15:32:40 -0400 Subject: [PATCH 09/89] o --- crconnect.wsgi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crconnect.wsgi b/crconnect.wsgi index 6c7ce423..069796e4 100644 --- a/crconnect.wsgi +++ b/crconnect.wsgi @@ -1,4 +1,4 @@ -opython_home = '/usr/local/envs/crcpython3' +python_home = '/usr/local/envs/crcpython3' import os import sys From b91579400045a999735709d8942a999ff36a9532 Mon Sep 17 00:00:00 2001 From: Aaron Louie Date: Wed, 20 May 2020 15:57:54 -0400 Subject: [PATCH 10/89] Fixes more paths. --- Dockerfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index f8c26e05..160a8f41 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,20 +2,21 @@ FROM python:3.7-slim WORKDIR /app -COPY Pipfile Pipfile.lock ./ +COPY Pipfile Pipfile.lock /app/ RUN pip install pipenv && \ apt-get update && \ apt-get install -y --no-install-recommends \ gcc python3-dev libssl-dev postgresql-client git-core && \ - pipenv install --deploy --system && \ + pipenv install --dev && \ apt-get remove -y gcc python3-dev libssl-dev && \ apt-get purge -y --auto-remove && \ rm -rf /var/lib/apt/lists/ * -COPY . /app +COPY . /app/ ENV FLASK_APP=/app/crc/__init__.py +CMD ["pipenv", "run", "flask", "db", "upgrade"] CMD ["pipenv", "run", "python", "/app/run.py"] # expose ports From d0f380e2242b50a7cc3a16bcf7d1b93da97b0721 Mon Sep 17 00:00:00 2001 From: Aaron Louie Date: Wed, 20 May 2020 16:49:32 -0400 Subject: [PATCH 11/89] Uses Python 3.6.9 --- .travis.yml | 2 +- Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index fba238f6..9ca51691 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ language: python python: - - "3.7" + - "3.6.9" services: - postgresql diff --git a/Dockerfile b/Dockerfile index 160a8f41..e759c07d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.7-slim +FROM python:3.6.9-slim WORKDIR /app From abc90cdcb17cd482e72a442ad6f46d603a9550bd Mon Sep 17 00:00:00 2001 From: Carlos Lopez Date: Wed, 20 May 2020 15:10:22 -0600 Subject: [PATCH 12/89] Adding serialiazer for study files --- crc/models/file.py | 8 +++++++- crc/models/study.py | 18 ++++++++++++++++++ crc/models/workflow.py | 3 ++- 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/crc/models/file.py b/crc/models/file.py index 6d83e4b2..942b2a80 100644 --- a/crc/models/file.py +++ b/crc/models/file.py @@ -6,7 +6,7 @@ from marshmallow_sqlalchemy import SQLAlchemyAutoSchema from sqlalchemy import func, Index from sqlalchemy.dialects.postgresql import UUID -from crc import db +from crc import db, ma class FileType(enum.Enum): @@ -139,3 +139,9 @@ class LookupDataSchema(SQLAlchemyAutoSchema): include_relationships = False include_fk = False # Includes foreign keys + +class SimpleFileSchema(ma.Schema): + + class Meta: + model = FileModel + fields = ["name"] diff --git a/crc/models/study.py b/crc/models/study.py index 6716df8e..79186f3f 100644 --- a/crc/models/study.py +++ b/crc/models/study.py @@ -5,6 +5,7 @@ from sqlalchemy import func from crc import db, ma from crc.api.common import ApiErrorSchema +from crc.models.file import FileModel, SimpleFileSchema from crc.models.protocol_builder import ProtocolBuilderStatus, ProtocolBuilderStudy from crc.models.workflow import WorkflowSpecCategoryModel, WorkflowState, WorkflowStatus, WorkflowSpecModel, \ WorkflowModel @@ -39,6 +40,10 @@ class StudyModel(db.Model): if self.on_hold: self.protocol_builder_status = ProtocolBuilderStatus.HOLD + def files(self): + _files = FileModel.query.filter_by(workflow_id=self.workflow[0].id) + return _files + class WorkflowMetadata(object): def __init__(self, id, name, display_name, description, spec_version, category_id, state: WorkflowState, status: WorkflowStatus, @@ -154,3 +159,16 @@ class StudySchema(ma.Schema): def make_study(self, data, **kwargs): """Can load the basic study data for updates to the database, but categories are write only""" return Study(**data) + + +class StudyFilesSchema(ma.Schema): + + # files = fields.List(fields.Nested(SimpleFileSchema), dump_only=True) + files = fields.Method('_files') + + class Meta: + model = Study + additional = ["id", "title", "last_updated", "primary_investigator_id"] + + def _files(self, obj): + return [file.name for file in obj.files()] diff --git a/crc/models/workflow.py b/crc/models/workflow.py index ea072e93..e5152eac 100644 --- a/crc/models/workflow.py +++ b/crc/models/workflow.py @@ -73,10 +73,11 @@ class WorkflowModel(db.Model): bpmn_workflow_json = db.Column(db.JSON) status = db.Column(db.Enum(WorkflowStatus)) study_id = db.Column(db.Integer, db.ForeignKey('study.id')) + study = db.relationship("StudyModel", backref='workflow') workflow_spec_id = db.Column(db.String, db.ForeignKey('workflow_spec.id')) workflow_spec = db.relationship("WorkflowSpecModel") spec_version = db.Column(db.String) total_tasks = db.Column(db.Integer, default=0) completed_tasks = db.Column(db.Integer, default=0) # task_history = db.Column(db.ARRAY(db.String), default=[]) # The history stack of user completed tasks. - last_updated = db.Column(db.DateTime) \ No newline at end of file + last_updated = db.Column(db.DateTime) From d76c18d329b2ebd6c64b6e2b9437263c387ae764 Mon Sep 17 00:00:00 2001 From: Aaron Louie Date: Wed, 20 May 2020 20:13:09 -0400 Subject: [PATCH 13/89] Installs curl --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index e759c07d..fae1657f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,8 @@ COPY Pipfile Pipfile.lock /app/ RUN pip install pipenv && \ apt-get update && \ apt-get install -y --no-install-recommends \ - gcc python3-dev libssl-dev postgresql-client git-core && \ + gcc python3-dev libssl-dev \ + curl postgresql-client git-core && \ pipenv install --dev && \ apt-get remove -y gcc python3-dev libssl-dev && \ apt-get purge -y --auto-remove && \ From 4628834106554486c1f0c447751eb46575cfb1ea Mon Sep 17 00:00:00 2001 From: Dan Funk Date: Thu, 21 May 2020 12:11:35 -0400 Subject: [PATCH 14/89] just a few more logging details. --- crc/api/user.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/crc/api/user.py b/crc/api/user.py index 83245d19..74a11841 100644 --- a/crc/api/user.py +++ b/crc/api/user.py @@ -1,7 +1,7 @@ import json import connexion -from flask import redirect, g +from flask import redirect, g, request from crc import sso, app, db from crc.api.common import ApiError @@ -35,9 +35,12 @@ def get_current_user(): @sso.login_handler def sso_login(user_info): - app.logger.info("Login from Shibboleth happening. " + json.dump(user_info)) + redirect = request.args.get('redirect') + app.logger.info("SSO_LOGIN: Full URL: " + request.url) + app.logger.info("SSO_LOGIN: User Details: " + json.dump(user_info)) + app.logger.info("SSO_LOGIN: Will try to redirect to : " + redirect) # TODO: Get redirect URL from Shibboleth request header - _handle_login(user_info) + _handle_login(user_info, redirect) def _handle_login(user_info, redirect_url=app.config['FRONTEND_AUTH_CALLBACK']): @@ -86,8 +89,10 @@ def _handle_login(user_info, redirect_url=app.config['FRONTEND_AUTH_CALLBACK']): # Return the frontend auth callback URL, with auth token appended. auth_token = user.encode_auth_token().decode() if redirect_url is not None: + app.logger.info("SSO_LOGIN: REDIRECTING TO: " + redirect_url) return redirect('%s/%s' % (redirect_url, auth_token)) else: + app.logger.info("SSO_LOGIN: NO REDIRECT, JUST RETURNING AUTH TOKEN.") return auth_token def backdoor( From 0265db7146a923a5c7850180911ab8077b675059 Mon Sep 17 00:00:00 2001 From: Dan Funk Date: Thu, 21 May 2020 16:02:45 -0400 Subject: [PATCH 15/89] adding an /sso endpoint for testing. --- crc/api/user.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crc/api/user.py b/crc/api/user.py index 74a11841..e7688dfc 100644 --- a/crc/api/user.py +++ b/crc/api/user.py @@ -42,6 +42,10 @@ def sso_login(user_info): # TODO: Get redirect URL from Shibboleth request header _handle_login(user_info, redirect) +@app.route('/sso') +def index(): + return str(request.headers) + def _handle_login(user_info, redirect_url=app.config['FRONTEND_AUTH_CALLBACK']): """On successful login, adds user to database if the user is not already in the system, From b3ae9ee77057a10720603af82a432567a4367d6a Mon Sep 17 00:00:00 2001 From: Dan Funk Date: Thu, 21 May 2020 16:28:34 -0400 Subject: [PATCH 16/89] changing the mapping, because 'Uid' not 'uid' --- config/default.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/config/default.py b/config/default.py index dd19f2ab..cb70e726 100644 --- a/config/default.py +++ b/config/default.py @@ -30,16 +30,31 @@ SWAGGER_AUTH_KEY = environ.get('SWAGGER_AUTH_KEY', default="SWAGGER") #: Default attribute map for single signon. SSO_LOGIN_URL = '/login' SSO_ATTRIBUTE_MAP = { - 'eppn': (False, 'eppn'), # dhf8r@virginia.edu - 'uid': (True, 'uid'), # dhf8r + 'Eppn': (False, 'eppn'), # dhf8r@virginia.edu + 'Uid': (True, 'uid'), # dhf8r 'givenName': (False, 'first_name'), # Daniel - 'mail': (False, 'email_address'), # dhf8r@Virginia.EDU - 'sn': (False, 'last_name'), # Funk + 'Sn': (False, 'last_name'), # Funk 'affiliation': (False, 'affiliation'), # 'staff@virginia.edu;member@virginia.edu' 'displayName': (False, 'display_name'), # Daniel Harold Funk 'title': (False, 'title') # SOFTWARE ENGINEER V } +# This what I see coming back: +# X-Remote-Cn: Daniel Harold Funk (dhf8r) +# X-Remote-Sn: Funk +# X-Remote-Givenname: Daniel +# X-Remote-Uid: dhf8r +# Eppn: dhf8r@virginia.edu +# Cn: Daniel Harold Funk (dhf8r) +# Sn: Funk +# Givenname: Daniel +# Uid: dhf8r +# X-Remote-User: dhf8r@virginia.edu +# X-Forwarded-For: 128.143.0.10 +# X-Forwarded-Host: dev.crconnect.uvadcos.io +# X-Forwarded-Server: dev.crconnect.uvadcos.io +# Connection: Keep-Alive + # %s/%i placeholders expected for uva_id and study_id in various calls. PB_BASE_URL = environ.get('PB_BASE_URL', default="http://localhost:5001/pb/") PB_USER_STUDIES_URL = environ.get('PB_USER_STUDIES_URL', default=PB_BASE_URL + "user_studies?uva_id=%s") From 4627318818cfaa4eeec10d1033d322b9095bf177 Mon Sep 17 00:00:00 2001 From: Dan Funk Date: Fri, 22 May 2020 07:55:58 -0400 Subject: [PATCH 17/89] Dropping flask_sso library in favor of reading from the headers directly. Updating login to read from ldap once it has the user_id. Adding more information to the sso endpoint. --- Pipfile | 1 - Pipfile.lock | 31 +++++++++--------------------- crc/__init__.py | 1 - crc/api/user.py | 50 +++++++++++++++++++++++++++++++++++++++++-------- 4 files changed, 51 insertions(+), 32 deletions(-) diff --git a/Pipfile b/Pipfile index 375ba042..77c70afc 100644 --- a/Pipfile +++ b/Pipfile @@ -31,7 +31,6 @@ sphinx = "*" recommonmark = "*" psycopg2-binary = "*" docxtpl = "*" -flask-sso = "*" python-dateutil = "*" pandas = "*" xlrd = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 904d32bd..036d4bf9 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,11 +1,11 @@ { "_meta": { "hash": { - "sha256": "26d23456010d3e5a559386d412cef3beacd92d5a4e474f2afdb0737ea0f20f04" + "sha256": "1ca737db75750ea4351c15b4b0b26155d90bc5522705ed293a0c2773600b6a0a" }, "pipfile-spec": 6, "requires": { - "python_version": "3.7" + "python_version": "3.6.9" }, "sources": [ { @@ -96,12 +96,6 @@ ], "version": "==3.6.3.0" }, - "blinker": { - "hashes": [ - "sha256:471aee25f3992bd325afa3772f1063dbdbbca947a041b8b89466dc00d606f8b6" - ], - "version": "==1.4" - }, "celery": { "hashes": [ "sha256:108a0bf9018a871620936c33a3ee9f6336a89f8ef0a0f567a9001f4aa361415f", @@ -307,13 +301,6 @@ ], "version": "==2.4.1" }, - "flask-sso": { - "hashes": [ - "sha256:541a8a2387c6eac4325c53f8f7f863a03173b37aa558a37a430010d7fc1a3633" - ], - "index": "pypi", - "version": "==0.4.0" - }, "future": { "hashes": [ "sha256:b1bead90b70cf6ec3f0710ae53a525360fa360d306a86583adc6bf83a4db537d" @@ -711,10 +698,10 @@ }, "six": { "hashes": [ - "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a", - "sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c" + "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", + "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced" ], - "version": "==1.14.0" + "version": "==1.15.0" }, "snowballstemmer": { "hashes": [ @@ -783,7 +770,7 @@ "spiffworkflow": { "editable": true, "git": "https://github.com/sartography/SpiffWorkflow.git", - "ref": "2c9698894f7e526a91bf3ca8c4b9fc9b6b01e807" + "ref": "cb098ee6d55b85bf7795997f4ad5f78c27d15381" }, "sqlalchemy": { "hashes": [ @@ -955,10 +942,10 @@ }, "six": { "hashes": [ - "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a", - "sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c" + "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", + "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced" ], - "version": "==1.14.0" + "version": "==1.15.0" }, "wcwidth": { "hashes": [ diff --git a/crc/__init__.py b/crc/__init__.py index aa301108..7cdc4ead 100644 --- a/crc/__init__.py +++ b/crc/__init__.py @@ -31,7 +31,6 @@ session = db.session migrate = Migrate(app, db) ma = Marshmallow(app) -sso = SSO(app=app) from crc import models from crc import api diff --git a/crc/api/user.py b/crc/api/user.py index e7688dfc..5c9bc108 100644 --- a/crc/api/user.py +++ b/crc/api/user.py @@ -3,10 +3,10 @@ import json import connexion from flask import redirect, g, request -from crc import sso, app, db +from crc import app, db from crc.api.common import ApiError from crc.models.user import UserModel, UserModelSchema - +from crc.services.ldap_service import LdapService """ .. module:: crc.api.user @@ -32,21 +32,55 @@ def verify_token(token): def get_current_user(): return UserModelSchema().dump(g.user) +def sso_login(): + # This what I see coming back: + # X-Remote-Cn: Daniel Harold Funk (dhf8r) + # X-Remote-Sn: Funk + # X-Remote-Givenname: Daniel + # X-Remote-Uid: dhf8r + # Eppn: dhf8r@virginia.edu + # Cn: Daniel Harold Funk (dhf8r) + # Sn: Funk + # Givenname: Daniel + # Uid: dhf8r + # X-Remote-User: dhf8r@virginia.edu + # X-Forwarded-For: 128.143.0.10 + # X-Forwarded-Host: dev.crconnect.uvadcos.io + # X-Forwarded-Server: dev.crconnect.uvadcos.io + # Connection: Keep-Alive + uid = request.headers.get("Uid") + if not uid: + uid = request.headers.get("X-Remote-Uid") + + if not uid: + raise ApiError("invalid_sso_credentials", "'Uid' nor 'X-Remote-Uid' were present in the headers: %s" + % str(request.headers)) -@sso.login_handler -def sso_login(user_info): redirect = request.args.get('redirect') app.logger.info("SSO_LOGIN: Full URL: " + request.url) - app.logger.info("SSO_LOGIN: User Details: " + json.dump(user_info)) + app.logger.info("SSO_LOGIN: User Id: " + uid) app.logger.info("SSO_LOGIN: Will try to redirect to : " + redirect) + + ldap_service = LdapService() + info = ldap_service.user_info(uid) + + user = UserModel(uid=uid, email_address=info.email, display_name=info.display_name, + affiliation=info.affiliation, title=info.title) + # TODO: Get redirect URL from Shibboleth request header - _handle_login(user_info, redirect) + _handle_login(user, redirect) @app.route('/sso') -def index(): - return str(request.headers) +def sso(): + response = "" + response += "

Headers

" + response += str(request.headers) + response += "

Environment

" + response += str(request.environ) + return response +@app.route('/login') def _handle_login(user_info, redirect_url=app.config['FRONTEND_AUTH_CALLBACK']): """On successful login, adds user to database if the user is not already in the system, then returns the frontend auth callback URL, with auth token appended. From 1cc9a60bfe9ddef1509c9d24108d1d73d416b37d Mon Sep 17 00:00:00 2001 From: Dan Funk Date: Fri, 22 May 2020 08:01:33 -0400 Subject: [PATCH 18/89] clearing out the remaining references to the flask_sso library. --- crc/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/crc/__init__.py b/crc/__init__.py index 7cdc4ead..9f1c4ee3 100644 --- a/crc/__init__.py +++ b/crc/__init__.py @@ -6,7 +6,6 @@ from flask_cors import CORS from flask_marshmallow import Marshmallow from flask_migrate import Migrate from flask_sqlalchemy import SQLAlchemy -from flask_sso import SSO logging.basicConfig(level=logging.INFO) From b490005af78e7c5fc31bf65e0552e1f015a1e6dd Mon Sep 17 00:00:00 2001 From: Dan Funk Date: Fri, 22 May 2020 09:50:18 -0400 Subject: [PATCH 19/89] dropping the remaining config stuff for flask_sso. updaing the user 'sso' endpoint to provide additional information for debugging. Pulling information from ldap to stay super consistent on where we get our information. --- config/default.py | 28 --------------- crc/api/user.py | 68 +++++++++++++----------------------- crc/services/ldap_service.py | 37 +++++++++++++------- tests/test_authentication.py | 19 +++++++++- tests/test_ldap_service.py | 2 +- 5 files changed, 69 insertions(+), 85 deletions(-) diff --git a/config/default.py b/config/default.py index cb70e726..d2486f86 100644 --- a/config/default.py +++ b/config/default.py @@ -27,34 +27,6 @@ TOKEN_AUTH_SECRET_KEY = environ.get('TOKEN_AUTH_SECRET_KEY', default="Shhhh!!! T FRONTEND_AUTH_CALLBACK = environ.get('FRONTEND_AUTH_CALLBACK', default="http://localhost:4200/session") SWAGGER_AUTH_KEY = environ.get('SWAGGER_AUTH_KEY', default="SWAGGER") -#: Default attribute map for single signon. -SSO_LOGIN_URL = '/login' -SSO_ATTRIBUTE_MAP = { - 'Eppn': (False, 'eppn'), # dhf8r@virginia.edu - 'Uid': (True, 'uid'), # dhf8r - 'givenName': (False, 'first_name'), # Daniel - 'Sn': (False, 'last_name'), # Funk - 'affiliation': (False, 'affiliation'), # 'staff@virginia.edu;member@virginia.edu' - 'displayName': (False, 'display_name'), # Daniel Harold Funk - 'title': (False, 'title') # SOFTWARE ENGINEER V -} - -# This what I see coming back: -# X-Remote-Cn: Daniel Harold Funk (dhf8r) -# X-Remote-Sn: Funk -# X-Remote-Givenname: Daniel -# X-Remote-Uid: dhf8r -# Eppn: dhf8r@virginia.edu -# Cn: Daniel Harold Funk (dhf8r) -# Sn: Funk -# Givenname: Daniel -# Uid: dhf8r -# X-Remote-User: dhf8r@virginia.edu -# X-Forwarded-For: 128.143.0.10 -# X-Forwarded-Host: dev.crconnect.uvadcos.io -# X-Forwarded-Server: dev.crconnect.uvadcos.io -# Connection: Keep-Alive - # %s/%i placeholders expected for uva_id and study_id in various calls. PB_BASE_URL = environ.get('PB_BASE_URL', default="http://localhost:5001/pb/") PB_USER_STUDIES_URL = environ.get('PB_USER_STUDIES_URL', default=PB_BASE_URL + "user_studies?uva_id=%s") diff --git a/crc/api/user.py b/crc/api/user.py index 5c9bc108..6924eb27 100644 --- a/crc/api/user.py +++ b/crc/api/user.py @@ -6,7 +6,7 @@ from flask import redirect, g, request from crc import app, db from crc.api.common import ApiError from crc.models.user import UserModel, UserModelSchema -from crc.services.ldap_service import LdapService +from crc.services.ldap_service import LdapService, LdapUserInfo """ .. module:: crc.api.user @@ -32,6 +32,7 @@ def verify_token(token): def get_current_user(): return UserModelSchema().dump(g.user) +@app.route('/login') def sso_login(): # This what I see coming back: # X-Remote-Cn: Daniel Harold Funk (dhf8r) @@ -59,67 +60,48 @@ def sso_login(): redirect = request.args.get('redirect') app.logger.info("SSO_LOGIN: Full URL: " + request.url) app.logger.info("SSO_LOGIN: User Id: " + uid) - app.logger.info("SSO_LOGIN: Will try to redirect to : " + redirect) + app.logger.info("SSO_LOGIN: Will try to redirect to : " + str(redirect)) ldap_service = LdapService() info = ldap_service.user_info(uid) - user = UserModel(uid=uid, email_address=info.email, display_name=info.display_name, - affiliation=info.affiliation, title=info.title) - - # TODO: Get redirect URL from Shibboleth request header - _handle_login(user, redirect) + return _handle_login(info, redirect) @app.route('/sso') def sso(): response = "" response += "

Headers

" - response += str(request.headers) + response += "