From d8f7b6cc9f0a3a2c0ea607e7860e35b0550d130d Mon Sep 17 00:00:00 2001 From: Aaron Louie Date: Mon, 11 May 2020 12:05:16 -0400 Subject: [PATCH 01/11] Adds dev, demo, and training branches --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 913400c6..e99f0dfc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,7 +37,7 @@ deploy: skip_cleanup: true on: all_branches: true - condition: $TRAVIS_BRANCH =~ ^(testing|staging|master)$ + condition: $TRAVIS_BRANCH =~ ^(dev|testing|demo|training|staging|master)$ notifications: email: From 02118800caabff781b35cbe0e0fe49bce81ba451 Mon Sep 17 00:00:00 2001 From: Aaron Louie Date: Mon, 11 May 2020 16:22:42 -0400 Subject: [PATCH 02/11] Prints out LDAP URL --- config/default.py | 1 + 1 file changed, 1 insertion(+) diff --git a/config/default.py b/config/default.py index 0e9c3a28..af343cee 100644 --- a/config/default.py +++ b/config/default.py @@ -52,3 +52,4 @@ print('DEVELOPMENT = ', DEVELOPMENT) print('TESTING = ', TESTING) print('PRODUCTION = ', PRODUCTION) print('PB_BASE_URL = ', PB_BASE_URL) +print('LDAP_URL = ', LDAP_URL) From 73ce70b574d875d10a672b84308d9b8359a0ea0b Mon Sep 17 00:00:00 2001 From: Aaron Louie Date: Tue, 12 May 2020 10:43:52 -0400 Subject: [PATCH 03/11] Allows CORS connections from frontend --- config/default.py | 5 ++++- config/testing.py | 1 - config/travis-testing.py | 1 - crc/__init__.py | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/config/default.py b/config/default.py index af343cee..e7a01691 100644 --- a/config/default.py +++ b/config/default.py @@ -1,4 +1,5 @@ import os +import re from os import environ basedir = os.path.abspath(os.path.dirname(__file__)) @@ -6,7 +7,7 @@ basedir = os.path.abspath(os.path.dirname(__file__)) JSON_SORT_KEYS = False # CRITICAL. Do not sort the data when returning values to the front end. NAME = "CR Connect Workflow" -CORS_ENABLED = False +CORS_ALLOW_ORIGINS = re.split(r',\s*', environ.get('CORS_ALLOW_ORIGINS', default="localhost:4200, localhost:5002")) DEVELOPMENT = environ.get('DEVELOPMENT', default="true") == "true" TESTING = environ.get('TESTING', default="false") == "true" PRODUCTION = (environ.get('PRODUCTION', default="false") == "true") or (not DEVELOPMENT and not TESTING) @@ -48,8 +49,10 @@ LDAP_URL = environ.get('LDAP_URL', default="ldap.virginia.edu") LDAP_TIMEOUT_SEC = environ.get('LDAP_TIMEOUT_SEC', default=3) print('=== USING DEFAULT CONFIG: ===') print('DB_HOST = ', DB_HOST) +print('CORS_ALLOW_ORIGINS = ', CORS_ALLOW_ORIGINS) print('DEVELOPMENT = ', DEVELOPMENT) print('TESTING = ', TESTING) print('PRODUCTION = ', PRODUCTION) print('PB_BASE_URL = ', PB_BASE_URL) print('LDAP_URL = ', LDAP_URL) + diff --git a/config/testing.py b/config/testing.py index 295156ce..823a0178 100644 --- a/config/testing.py +++ b/config/testing.py @@ -2,7 +2,6 @@ import os basedir = os.path.abspath(os.path.dirname(__file__)) NAME = "CR Connect Workflow" -CORS_ENABLED = False DEVELOPMENT = True TESTING = True SQLALCHEMY_DATABASE_URI = "postgresql://crc_user:crc_pass@localhost:5432/crc_test" diff --git a/config/travis-testing.py b/config/travis-testing.py index 6231533d..b0da6538 100644 --- a/config/travis-testing.py +++ b/config/travis-testing.py @@ -2,7 +2,6 @@ import os basedir = os.path.abspath(os.path.dirname(__file__)) NAME = "CR Connect Workflow" -CORS_ENABLED = False DEVELOPMENT = True TESTING = True SQLALCHEMY_DATABASE_URI = "postgresql://postgres:@localhost:5432/crc_test" diff --git a/crc/__init__.py b/crc/__init__.py index 25f1326e..a2add8cc 100644 --- a/crc/__init__.py +++ b/crc/__init__.py @@ -36,7 +36,7 @@ from crc import models from crc import api connexion_app.add_api('api.yml') -cors = CORS(connexion_app.app) +cors = CORS(connexion_app.app, resources={r"/v1.0/*": {"origins": app.config['CORS_ALLOW_ORIGINS']}}) @app.cli.command() From ac3f0b401bbaf89f6e30e1069755e4f30699c6b5 Mon Sep 17 00:00:00 2001 From: Aaron Louie Date: Tue, 12 May 2020 12:23:47 -0400 Subject: [PATCH 04/11] Converts list of allowed origins to regexes --- crc/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crc/__init__.py b/crc/__init__.py index a2add8cc..05400ccb 100644 --- a/crc/__init__.py +++ b/crc/__init__.py @@ -36,7 +36,10 @@ from crc import models from crc import api connexion_app.add_api('api.yml') -cors = CORS(connexion_app.app, resources={r"/v1.0/*": {"origins": app.config['CORS_ALLOW_ORIGINS']}}) + +# Convert list of allowed origins to list of regexes +origins_re = [r"^https?:\/\/%s(.*)" % o for o in app.config['CORS_ALLOW_ORIGINS']] +cors = CORS(connexion_app.app, resources={r"/*": {"origins": origins_re}}) @app.cli.command() From 77b19083739bfe797e9c60e679a7f4d4040aba58 Mon Sep 17 00:00:00 2001 From: Aaron Louie Date: Tue, 12 May 2020 14:27:17 -0400 Subject: [PATCH 05/11] Escapes dot characters in CORS domains --- crc/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crc/__init__.py b/crc/__init__.py index 05400ccb..2ec760b7 100644 --- a/crc/__init__.py +++ b/crc/__init__.py @@ -38,7 +38,8 @@ from crc import api connexion_app.add_api('api.yml') # Convert list of allowed origins to list of regexes -origins_re = [r"^https?:\/\/%s(.*)" % o for o in app.config['CORS_ALLOW_ORIGINS']] +origins_re = [r"^https?:\/\/%s(.*)" % o.replace('.', '\.') for o in app.config['CORS_ALLOW_ORIGINS']] +print('Allowing connections from origins matching the following regexes:', origins_re) cors = CORS(connexion_app.app, resources={r"/*": {"origins": origins_re}}) From cdfe45dc00f5646f3228820bc1a5d1cde6598a8d Mon Sep 17 00:00:00 2001 From: Aaron Louie Date: Tue, 12 May 2020 22:42:02 -0400 Subject: [PATCH 06/11] Turns on flask_cors debugging to troubleshoot CORS issue --- crc/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crc/__init__.py b/crc/__init__.py index 2ec760b7..be25bca8 100644 --- a/crc/__init__.py +++ b/crc/__init__.py @@ -4,9 +4,10 @@ import os import connexion from flask_cors import CORS from flask_marshmallow import Marshmallow -from flask_sqlalchemy import SQLAlchemy from flask_migrate import Migrate +from flask_sqlalchemy import SQLAlchemy from flask_sso import SSO + logging.basicConfig(level=logging.INFO) connexion_app = connexion.FlaskApp(__name__) @@ -39,7 +40,7 @@ connexion_app.add_api('api.yml') # Convert list of allowed origins to list of regexes origins_re = [r"^https?:\/\/%s(.*)" % o.replace('.', '\.') for o in app.config['CORS_ALLOW_ORIGINS']] -print('Allowing connections from origins matching the following regexes:', origins_re) +logging.getLogger('flask_cors').level = logging.DEBUG cors = CORS(connexion_app.app, resources={r"/*": {"origins": origins_re}}) From caf791118fcc946aedd5c7324fc4f163b3c82ba0 Mon Sep 17 00:00:00 2001 From: Aaron Louie Date: Wed, 13 May 2020 11:00:43 -0400 Subject: [PATCH 07/11] Allows all resources --- crc/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crc/__init__.py b/crc/__init__.py index be25bca8..a8b6f8af 100644 --- a/crc/__init__.py +++ b/crc/__init__.py @@ -41,7 +41,7 @@ connexion_app.add_api('api.yml') # Convert list of allowed origins to list of regexes origins_re = [r"^https?:\/\/%s(.*)" % o.replace('.', '\.') for o in app.config['CORS_ALLOW_ORIGINS']] logging.getLogger('flask_cors').level = logging.DEBUG -cors = CORS(connexion_app.app, resources={r"/*": {"origins": origins_re}}) +cors = CORS(connexion_app.app, origins=origins_re) @app.cli.command() From 3ef07bd8c90275843da1938c25a0de0071ad1480 Mon Sep 17 00:00:00 2001 From: Aaron Louie Date: Wed, 13 May 2020 12:32:57 -0400 Subject: [PATCH 08/11] Allows all origins (for troubleshooting) --- crc/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crc/__init__.py b/crc/__init__.py index a8b6f8af..e91b3143 100644 --- a/crc/__init__.py +++ b/crc/__init__.py @@ -41,7 +41,8 @@ connexion_app.add_api('api.yml') # Convert list of allowed origins to list of regexes origins_re = [r"^https?:\/\/%s(.*)" % o.replace('.', '\.') for o in app.config['CORS_ALLOW_ORIGINS']] logging.getLogger('flask_cors').level = logging.DEBUG -cors = CORS(connexion_app.app, origins=origins_re) +cors = CORS(connexion_app.app) +# cors = CORS(connexion_app.app, origins=origins_re) @app.cli.command() From b72bc07ec4ed73668866e917fb697f0d6aa1d26e Mon Sep 17 00:00:00 2001 From: Aaron Louie Date: Wed, 13 May 2020 21:15:13 -0400 Subject: [PATCH 09/11] Builds RRT Docker image. Deploys to RRT instance. --- .travis.yml | 2 +- deploy.sh | 42 ++++++++++++++++++++++++++++++++++-------- 2 files changed, 35 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index e99f0dfc..fba238f6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,7 +37,7 @@ deploy: skip_cleanup: true on: all_branches: true - condition: $TRAVIS_BRANCH =~ ^(dev|testing|demo|training|staging|master)$ + condition: $TRAVIS_BRANCH =~ ^(dev|testing|demo|training|staging|master|rrt\/.*)$ notifications: email: diff --git a/deploy.sh b/deploy.sh index 04533e71..d597f52c 100755 --- a/deploy.sh +++ b/deploy.sh @@ -1,11 +1,39 @@ #!/bin/bash +function branch_to_tag () { + if [ "$1" == "latest" ]; then echo "production"; else echo "$1" ; fi +} + +function branch_to_deploy_group() { + if [[ $1 =~ ^(rrt\/.*)$ ]]; then echo "rrt"; else echo "crconnect" ; fi +} + +function branch_to_deploy_stage () { + if [ "$1" == "master" ]; then echo "production"; else echo "$1" ; fi +} + +REPO="sartography/cr-connect-workflow" +TAG=$(branch_to_tag "$TRAVIS_BRANCH") +COMMIT=${TRAVIS_COMMIT::8} + +DEPLOY_APP="backend" +DEPLOY_GROUP=$(branch_to_deploy_group "$TRAVIS_BRANCH") +DEPLOY_STAGE=$(branch_to_deploy_stage "$TRAVIS_BRANCH") + +if [ "$DEPLOY_GROUP" == "rrt" ]; then + IFS='/' read -ra ARR <<< "$TRAVIS_BRANCH" # Split branch on '/' character + TAG=$(branch_to_tag "rrt_${ARR[1]}") + DEPLOY_STAGE=$(branch_to_deploy_stage "${ARR[1]}") +fi + +DEPLOY_PATH="$DEPLOY_GROUP/$DEPLOY_STAGE/$DEPLOY_APP" +echo "REPO = $REPO" +echo "TAG = $TAG" +echo "COMMIT = $COMMIT" +echo "DEPLOY_PATH = $DEPLOY_PATH" + # Build and push Docker image to Docker Hub echo "$DOCKER_TOKEN" | docker login -u "$DOCKER_USERNAME" --password-stdin || exit 1 -REPO="sartography/cr-connect-workflow" -TAG=$(if [ "$TRAVIS_BRANCH" == "master" ]; then echo "latest"; else echo "$TRAVIS_BRANCH" ; fi) -COMMIT=${TRAVIS_COMMIT::8} - docker build -f Dockerfile -t "$REPO:$COMMIT" . || exit 1 docker tag "$REPO:$COMMIT" "$REPO:$TAG" || exit 1 docker tag "$REPO:$COMMIT" "$REPO:travis-$TRAVIS_BUILD_NUMBER" || exit 1 @@ -15,8 +43,6 @@ docker push "$REPO" || exit 1 echo "Publishing to Docker Hub..." sleep 30 -# Notify DC/OS that Docker image has been updated +# Notify UVA DCOS that Docker image has been updated echo "Refreshing DC/OS..." -STAGE=$(if [ "$TRAVIS_BRANCH" == "master" ]; then echo "production"; else echo "$TRAVIS_BRANCH" ; fi) -echo "STAGE = $STAGE" -aws sqs send-message --region "$AWS_DEFAULT_REGION" --queue-url "$AWS_SQS_URL" --message-body "crconnect/$STAGE/backend" || exit 1 +aws sqs send-message --region "$AWS_DEFAULT_REGION" --queue-url "$AWS_SQS_URL" --message-body "$DEPLOY_PATH" || exit 1 From 1511b6fe0d1b9962cff68ef77b9254b3fa93ca4e Mon Sep 17 00:00:00 2001 From: Aaron Louie Date: Thu, 14 May 2020 08:52:05 -0400 Subject: [PATCH 10/11] Only creates Docker tag for branch --- deploy.sh | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/deploy.sh b/deploy.sh index d597f52c..a1c5da4b 100755 --- a/deploy.sh +++ b/deploy.sh @@ -14,7 +14,6 @@ function branch_to_deploy_stage () { REPO="sartography/cr-connect-workflow" TAG=$(branch_to_tag "$TRAVIS_BRANCH") -COMMIT=${TRAVIS_COMMIT::8} DEPLOY_APP="backend" DEPLOY_GROUP=$(branch_to_deploy_group "$TRAVIS_BRANCH") @@ -29,14 +28,11 @@ fi DEPLOY_PATH="$DEPLOY_GROUP/$DEPLOY_STAGE/$DEPLOY_APP" echo "REPO = $REPO" echo "TAG = $TAG" -echo "COMMIT = $COMMIT" echo "DEPLOY_PATH = $DEPLOY_PATH" # Build and push Docker image to Docker Hub echo "$DOCKER_TOKEN" | docker login -u "$DOCKER_USERNAME" --password-stdin || exit 1 -docker build -f Dockerfile -t "$REPO:$COMMIT" . || exit 1 -docker tag "$REPO:$COMMIT" "$REPO:$TAG" || exit 1 -docker tag "$REPO:$COMMIT" "$REPO:travis-$TRAVIS_BUILD_NUMBER" || exit 1 +docker build -f Dockerfile -t "$REPO:$TAG" . || exit 1 docker push "$REPO" || exit 1 # Wait for Docker Hub From f2c9fd5fc4d31523f19e1c81171507ade19caa8c Mon Sep 17 00:00:00 2001 From: Dan Funk Date: Thu, 14 May 2020 15:07:05 -0400 Subject: [PATCH 11/11] adding a default url. And some debugging information to see if we hit he endpoint in the logs. --- config/default.py | 1 + crc/api/user.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/config/default.py b/config/default.py index e7a01691..41a24949 100644 --- a/config/default.py +++ b/config/default.py @@ -27,6 +27,7 @@ FRONTEND_AUTH_CALLBACK = environ.get('FRONTEND_AUTH_CALLBACK', default="http://l SWAGGER_AUTH_KEY = environ.get('SWAGGER_AUTH_KEY', default="SWAGGER") #: Default attribute map for single signon. +SSO_LOGIN_URL = '/login' SSO_ATTRIBUTE_MAP = { 'eppn': (False, 'eppn'), # dhf8r@virginia.edu 'uid': (True, 'uid'), # dhf8r diff --git a/crc/api/user.py b/crc/api/user.py index 411d87cb..83245d19 100644 --- a/crc/api/user.py +++ b/crc/api/user.py @@ -1,3 +1,5 @@ +import json + import connexion from flask import redirect, g @@ -33,6 +35,7 @@ def get_current_user(): @sso.login_handler def sso_login(user_info): + app.logger.info("Login from Shibboleth happening. " + json.dump(user_info)) # TODO: Get redirect URL from Shibboleth request header _handle_login(user_info)