sartography
/
cr-connect-workflow
mirror of https://github.com/sartography/cr-connect-workflow.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'dev' into rrt/production

This commit is contained in:
Aaron Louie 2020-05-27 10:07:04 -04:00
parent 0e973b988a 1d52a0214a
commit 7721b9f83f
16 changed files with 90 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
config/default.py
View File

@ -31,7 +31,7 @@ FRONTEND_AUTH_CALLBACK = environ.get('FRONTEND_AUTH_CALLBACK', default="http://l
SWAGGER_AUTH_KEY = environ.get('SWAGGER_AUTH_KEY', default="SWAGGER")
# %s/%i placeholders expected for uva_id and study_id in various calls.
PB_ENABLED = environ.get('PB_ENABLED', default=False)
PB_ENABLED = environ.get('PB_ENABLED', default="false") == "true"
PB_BASE_URL = environ.get('PB_BASE_URL', default="http://localhost:5001/pb/").strip('/') + '/' # Trailing slash required
PB_USER_STUDIES_URL = environ.get('PB_USER_STUDIES_URL', default=PB_BASE_URL + "user_studies?uva_id=%s")
PB_INVESTIGATORS_URL = environ.get('PB_INVESTIGATORS_URL', default=PB_BASE_URL + "investigators?studyid=%i")
@ -40,13 +40,5 @@ PB_STUDY_DETAILS_URL = environ.get('PB_STUDY_DETAILS_URL', default=PB_BASE_URL +
LDAP_URL = environ.get('LDAP_URL', default="ldap.virginia.edu").strip('/') # No trailing slash or http://
LDAP_TIMEOUT_SEC = int(environ.get('LDAP_TIMEOUT_SEC', default=3))
print('=== USING DEFAULT CONFIG: ===')
print('DB_HOST = ', DB_HOST)
print('CORS_ALLOW_ORIGINS = ', CORS_ALLOW_ORIGINS)
print('DEVELOPMENT = ', DEVELOPMENT)
print('TESTING = ', TESTING)
print('PRODUCTION = ', PRODUCTION)
print('PB_BASE_URL = ', PB_BASE_URL)
print('LDAP_URL = ', LDAP_URL)
print('APPLICATION_ROOT = ', APPLICATION_ROOT)

10
crc/__init__.py
View File

@ -40,6 +40,16 @@ connexion_app.add_api('api.yml', base_path='/v1.0')
origins_re = [r"^https?:\/\/%s(.*)" % o.replace('.', '\.') for o in app.config['CORS_ALLOW_ORIGINS']]
cors = CORS(connexion_app.app, origins=origins_re)
print('=== USING THESE CONFIG SETTINGS: ===')
print('DB_HOST = ', )
print('CORS_ALLOW_ORIGINS = ', app.config['CORS_ALLOW_ORIGINS'])
print('DEVELOPMENT = ', app.config['DEVELOPMENT'])
print('TESTING = ', app.config['TESTING'])
print('PRODUCTION = ', app.config['PRODUCTION'])
print('PB_BASE_URL = ', app.config['PB_BASE_URL'])
print('LDAP_URL = ', app.config['LDAP_URL'])
print('APPLICATION_ROOT = ', app.config['APPLICATION_ROOT'])
print('PB_ENABLED = ', app.config['PB_ENABLED'])
@app.cli.command()
def load_example_data():

3
crc/api/workflow.py
View File

@ -4,6 +4,7 @@ from crc import session
from crc.api.common import ApiError, ApiErrorSchema
from crc.models.api_models import WorkflowApi, WorkflowApiSchema, NavigationItem, NavigationItemSchema
from crc.models.file import FileModel, LookupDataSchema
from crc.models.stats import TaskEventModel
from crc.models.workflow import WorkflowModel, WorkflowSpecModelSchema, WorkflowSpecModel, WorkflowSpecCategoryModel, \
WorkflowSpecCategoryModelSchema
from crc.services.file_service import FileService
@ -77,6 +78,8 @@ def delete_workflow_specification(spec_id):
for file in files:
FileService.delete_file(file.id)
session.query(TaskEventModel).filter(TaskEventModel.workflow_spec_id == spec_id).delete()
# Delete all stats and workflow models related to this specification
for workflow in session.query(WorkflowModel).filter_by(workflow_spec_id=spec_id):
StudyService.delete_workflow(workflow)

11
crc/services/lookup_service.py
View File

@ -1,4 +1,6 @@
from pandas import ExcelFile
from sqlalchemy import func, desc
from sqlalchemy.sql.functions import GenericFunction
from crc import db
from crc.api.common import ApiError
@ -7,6 +9,9 @@ from crc.models.file import FileDataModel, LookupFileModel, LookupDataModel
from crc.services.file_service import FileService
from crc.services.ldap_service import LdapService
class TSRank(GenericFunction):
package = 'full_text'
name = 'ts_rank'
class LookupService(object):
@ -122,9 +127,9 @@ class LookupService(object):
else:
query = "%s:*" % query
db_query = db_query.filter(LookupDataModel.label.match(query))
# db_query = db_query.filter(text("lookup_data.label @@ to_tsquery('simple', '%s')" % query))
db_query = db_query.order_by(desc(func.full_text.ts_rank(
func.to_tsvector('simple', LookupDataModel.label),
func.to_tsquery('simple', query))))
return db_query.limit(limit).all()
@staticmethod

11
crc/services/protocol_builder.py
View File

@ -9,12 +9,18 @@ from crc.models.protocol_builder import ProtocolBuilderStudySchema, ProtocolBuil
class ProtocolBuilderService(object):
ENABLED = app.config['PB_ENABLED']
STUDY_URL = app.config['PB_USER_STUDIES_URL']
INVESTIGATOR_URL = app.config['PB_INVESTIGATORS_URL']
REQUIRED_DOCS_URL = app.config['PB_REQUIRED_DOCS_URL']
STUDY_DETAILS_URL = app.config['PB_STUDY_DETAILS_URL']
@staticmethod
def is_enabled():
if isinstance(app.config['PB_ENABLED'], str):
return app.config['PB_ENABLED'].lower() == "true"
else:
return app.config['PB_ENABLED'] is True
@staticmethod
def get_studies(user_id) -> {}:
ProtocolBuilderService.__enabled_or_raise()
@ -43,7 +49,7 @@ class ProtocolBuilderService(object):
@staticmethod
def __enabled_or_raise():
if not ProtocolBuilderService.ENABLED:
if not ProtocolBuilderService.is_enabled():
raise ApiError("protocol_builder_disabled", "The Protocol Builder Service is currently disabled.")
@staticmethod
@ -59,4 +65,3 @@ class ProtocolBuilderService(object):
"Received an invalid response from the protocol builder (status %s): %s when calling "
"url '%s'." %
(response.status_code, response.text, url))

52
crc/services/study_service.py
View File

@ -117,7 +117,7 @@ class StudyService(object):
that is available.."""
# Get PB required docs, if Protocol Builder Service is enabled.
if ProtocolBuilderService.ENABLED:
if ProtocolBuilderService.is_enabled():
try:
pb_docs = ProtocolBuilderService.get_required_docs(study_id=study_id)
except requests.exceptions.ConnectionError as ce:
@ -133,7 +133,7 @@ class StudyService(object):
documents = {}
for code, doc in doc_dictionary.items():
if ProtocolBuilderService.ENABLED:
if ProtocolBuilderService.is_enabled():
pb_data = next((item for item in pb_docs if int(item['AUXDOCID']) == int(doc['id'])), None)
doc['required'] = False
if pb_data:
@ -216,34 +216,36 @@ class StudyService(object):
"""Assures that the studies we have locally for the given user are
in sync with the studies available in protocol builder. """
if not ProtocolBuilderService.ENABLED:
return
if ProtocolBuilderService.is_enabled():
# Get studies matching this user from Protocol Builder
pb_studies: List[ProtocolBuilderStudy] = ProtocolBuilderService.get_studies(user.uid)
app.logger.info("The Protocol Builder is enabled. app.config['PB_ENABLED'] = " +
str(app.config['PB_ENABLED']))
# Get studies from the database
db_studies = session.query(StudyModel).filter_by(user_uid=user.uid).all()
# Get studies matching this user from Protocol Builder
pb_studies: List[ProtocolBuilderStudy] = ProtocolBuilderService.get_studies(user.uid)
# Update all studies from the protocol builder, create new studies as needed.
# Futher assures that every active study (that does exist in the protocol builder)
# has a reference to every available workflow (though some may not have started yet)
for pb_study in pb_studies:
db_study = next((s for s in db_studies if s.id == pb_study.STUDYID), None)
if not db_study:
db_study = StudyModel(id=pb_study.STUDYID)
session.add(db_study)
db_studies.append(db_study)
db_study.update_from_protocol_builder(pb_study)
StudyService._add_all_workflow_specs_to_study(db_study)
# Get studies from the database
db_studies = session.query(StudyModel).filter_by(user_uid=user.uid).all()
# Mark studies as inactive that are no longer in Protocol Builder
for study in db_studies:
pb_study = next((pbs for pbs in pb_studies if pbs.STUDYID == study.id), None)
if not pb_study:
study.protocol_builder_status = ProtocolBuilderStatus.ABANDONED
# Update all studies from the protocol builder, create new studies as needed.
# Futher assures that every active study (that does exist in the protocol builder)
# has a reference to every available workflow (though some may not have started yet)
for pb_study in pb_studies:
db_study = next((s for s in db_studies if s.id == pb_study.STUDYID), None)
if not db_study:
db_study = StudyModel(id=pb_study.STUDYID)
session.add(db_study)
db_studies.append(db_study)
db_study.update_from_protocol_builder(pb_study)
StudyService._add_all_workflow_specs_to_study(db_study)
db.session.commit()
# Mark studies as inactive that are no longer in Protocol Builder
for study in db_studies:
pb_study = next((pbs for pbs in pb_studies if pbs.STUDYID == study.id), None)
if not pb_study:
study.protocol_builder_status = ProtocolBuilderStatus.ABANDONED
db.session.commit()
@staticmethod
def __update_status_of_workflow_meta(workflow_metas, status):

BIN
tests/data/enum_options_from_file/customer_list.xls
View File

Binary file not shown.

19
tests/test_lookup_service.py
View File

@ -1,10 +1,10 @@
from tests.base_test import BaseTest
from crc import session
from crc.models.file import FileDataModel, FileModel, LookupFileModel, LookupDataModel
from crc.services.file_service import FileService
from crc.services.lookup_service import LookupService
from crc.services.workflow_processor import WorkflowProcessor
from crc.services.workflow_service import WorkflowService
from tests.base_test import BaseTest
class TestLookupService(BaseTest):
@ -21,7 +21,7 @@ class TestLookupService(BaseTest):
self.assertEqual(1, len(lookup_records))
lookup_record = lookup_records[0]
lookup_data = session.query(LookupDataModel).filter(LookupDataModel.lookup_file_model == lookup_record).all()
self.assertEquals(19, len(lookup_data))
self.assertEquals(23, len(lookup_data))
# Using the same table with different lookup lable or value, does create additional records.
LookupService.get_lookup_table_from_data_model(file_data_model, "CUSTOMER_NAME", "CUSTOMER_NUMBER")
lookup_records = session.query(LookupFileModel).all()
@ -51,8 +51,6 @@ class TestLookupService(BaseTest):
self.assertEquals(1, len(results), "case does not matter.")
self.assertEquals("UVA - INTERNAL - GM USE ONLY", results[0].label)
results = LookupService._run_lookup_query(lookup_table, "medici", limit=10)
self.assertEquals(1, len(results), "partial words are picked up.")
self.assertEquals("The Medicines Company", results[0].label)
@ -73,7 +71,16 @@ class TestLookupService(BaseTest):
self.assertEquals(7, len(results), "short terms get multiple correct results.")
self.assertEquals("Genetics Savings & Clone, Inc.", results[0].label)
results = LookupService._run_lookup_query(lookup_table, "reaction design", limit=10)
self.assertEquals(5, len(results), "all results come back for two terms.")
self.assertEquals("Reaction Design", results[0].label, "The first result is the most relevant")
self.assertEquals("Reaction Then Design ", results[1].label, "The first result is the most relevant")
self.assertEquals("Design Then Reaction", results[2].label, "The first result is the most relevant")
self.assertEquals("Just Reaction", results[3].label, "The first result is the most relevant")
self.assertEquals("Just Design", results[4].label, "The first result is the most relevant")
# Fixme: Stop words are taken into account on the query side, and haven't found a fix yet.
#results = WorkflowService.run_lookup_query(lookup_table.id, "in", limit=10)
#self.assertEquals(7, len(results), "stop words are not removed.")
#self.assertEquals("Genetics Savings & Clone, Inc.", results[0].label)

9
tests/test_protocol_builder.py
View File

@ -1,5 +1,6 @@
from unittest.mock import patch
from crc import app
from tests.base_test import BaseTest
from crc.services.protocol_builder import ProtocolBuilderService
@ -10,7 +11,7 @@ class TestProtocolBuilder(BaseTest):
@patch('crc.services.protocol_builder.requests.get')
def test_get_studies(self, mock_get):
ProtocolBuilderService.ENABLED = True
app.config['PB_ENABLED'] = True
mock_get.return_value.ok = True
mock_get.return_value.text = self.protocol_builder_response('user_studies.json')
response = ProtocolBuilderService.get_studies(self.test_uid)
@ -18,7 +19,7 @@ class TestProtocolBuilder(BaseTest):
@patch('crc.services.protocol_builder.requests.get')
def test_get_investigators(self, mock_get):
ProtocolBuilderService.ENABLED = True
app.config['PB_ENABLED'] = True
mock_get.return_value.ok = True
mock_get.return_value.text = self.protocol_builder_response('investigators.json')
response = ProtocolBuilderService.get_investigators(self.test_study_id)
@ -30,7 +31,7 @@ class TestProtocolBuilder(BaseTest):
@patch('crc.services.protocol_builder.requests.get')
def test_get_required_docs(self, mock_get):
ProtocolBuilderService.ENABLED = True
app.config['PB_ENABLED'] = True
mock_get.return_value.ok = True
mock_get.return_value.text = self.protocol_builder_response('required_docs.json')
response = ProtocolBuilderService.get_required_docs(self.test_study_id)
@ -40,7 +41,7 @@ class TestProtocolBuilder(BaseTest):
@patch('crc.services.protocol_builder.requests.get')
def test_get_details(self, mock_get):
ProtocolBuilderService.ENABLED = True
app.config['PB_ENABLED'] = True
mock_get.return_value.ok = True
mock_get.return_value.text = self.protocol_builder_response('study_details.json')
response = ProtocolBuilderService.get_study_details(self.test_study_id)

4
tests/test_study_api.py
View File

@ -3,7 +3,7 @@ from tests.base_test import BaseTest
from datetime import datetime, timezone
from unittest.mock import patch
from crc import session
from crc import session, app
from crc.models.protocol_builder import ProtocolBuilderStatus, \
ProtocolBuilderStudySchema
from crc.models.stats import TaskEventModel
@ -105,7 +105,7 @@ class TestStudyApi(BaseTest):
def test_get_all_studies(self, mock_studies, mock_details, mock_docs, mock_investigators):
# Enable the protocol builder for these tests, as the master_workflow and other workflows
# depend on using the PB for data.
ProtocolBuilderService.ENABLED = True
app.config['PB_ENABLED'] = True
self.load_example_data()
s = StudyModel(
id=54321, # This matches one of the ids from the study_details_json data.

6
tests/test_study_service.py
View File

@ -2,7 +2,7 @@ import json
from datetime import datetime
from unittest.mock import patch
from crc import db
from crc import db, app
from crc.models.protocol_builder import ProtocolBuilderStatus
from crc.models.study import StudyModel
from crc.models.user import UserModel
@ -57,7 +57,7 @@ class TestStudyService(BaseTest):
@patch('crc.services.protocol_builder.ProtocolBuilderService.get_required_docs') # mock_docs
def test_total_tasks_updated(self, mock_docs):
"""Assure that as a users progress is available when getting a list of studies for that user."""
app.config['PB_ENABLED'] = True
docs_response = self.protocol_builder_response('required_docs.json')
mock_docs.return_value = json.loads(docs_response)
@ -106,7 +106,7 @@ class TestStudyService(BaseTest):
@patch('crc.services.protocol_builder.ProtocolBuilderService.get_required_docs') # mock_docs
def test_get_required_docs(self, mock_docs):
app.config['PB_ENABLED'] = True
# mock out the protocol builder
docs_response = self.protocol_builder_response('required_docs.json')
mock_docs.return_value = json.loads(docs_response)

4
tests/test_tasks_api.py
View File

@ -308,7 +308,7 @@ class TestTasksApi(BaseTest):
self.load_example_data()
# Enable the protocol builder.
ProtocolBuilderService.ENABLED = True
app.config['PB_ENABLED'] = True
# This depends on getting a list of investigators back from the protocol builder.
mock_get.return_value.ok = True
@ -426,7 +426,7 @@ class TestTasksApi(BaseTest):
def test_parallel_multi_instance(self, mock_get):
# Assure we get nine investigators back from the API Call, as set in the investigators.json file.
ProtocolBuilderService.ENABLED = True
app.config['PB_ENABLED'] = True
mock_get.return_value.ok = True
mock_get.return_value.text = self.protocol_builder_response('investigators.json')

2
tests/test_workflow_processor.py
View File

@ -374,7 +374,7 @@ class TestWorkflowProcessor(BaseTest):
mock_details.return_value = json.loads(details_response)
self.load_example_data(use_crc_data=True)
ProtocolBuilderService.ENABLED = True
app.config['PB_ENABLED'] = True
study = session.query(StudyModel).first()
workflow_spec_model = db.session.query(WorkflowSpecModel).\

4
tests/test_workflow_service.py
View File

@ -68,7 +68,7 @@ class TestWorkflowService(BaseTest):
task = processor.next_task()
WorkflowService.process_options(task, task.task_spec.form.fields[0])
options = task.task_spec.form.fields[0].options
self.assertEquals(19, len(options))
self.assertEquals(23, len(options))
self.assertEquals('1000', options[0]['id'])
self.assertEquals("UVA - INTERNAL - GM USE ONLY", options[0]['name'])
@ -86,7 +86,7 @@ class TestWorkflowService(BaseTest):
self.assertEquals("CUSTOMER_NAME", lookup_record.label_column)
self.assertEquals("CUSTOMER_NAME", lookup_record.label_column)
lookup_data = session.query(LookupDataModel).filter(LookupDataModel.lookup_file_model == lookup_record).all()
self.assertEquals(19, len(lookup_data))
self.assertEquals(23, len(lookup_data))
self.assertEquals("1000", lookup_data[0].value)
self.assertEquals("UVA - INTERNAL - GM USE ONLY", lookup_data[0].label)

2
tests/test_workflow_spec_api.py
View File

@ -1,9 +1,9 @@
import json
from tests.base_test import BaseTest
from crc import session
from crc.models.file import FileModel
from crc.models.workflow import WorkflowSpecModel, WorkflowSpecModelSchema, WorkflowModel, WorkflowSpecCategoryModel
from tests.base_test import BaseTest
class TestWorkflowSpec(BaseTest):

6
tests/test_workflow_spec_validation_api.py
View File

@ -4,7 +4,7 @@ from unittest.mock import patch
from tests.base_test import BaseTest
from crc.services.protocol_builder import ProtocolBuilderService
from crc import session
from crc import session, app
from crc.api.common import ApiErrorSchema
from crc.models.protocol_builder import ProtocolBuilderStudySchema
from crc.models.workflow import WorkflowSpecModel
@ -21,7 +21,7 @@ class TestWorkflowSpecValidation(BaseTest):
return ApiErrorSchema(many=True).load(json_data)
def test_successful_validation_of_test_workflows(self):
ProtocolBuilderService.ENABLED = False # Assure this is disabled.
app.config['PB_ENABLED'] = False # Assure this is disabled.
self.assertEqual(0, len(self.validate_workflow("parallel_tasks")))
self.assertEqual(0, len(self.validate_workflow("decision_table")))
self.assertEqual(0, len(self.validate_workflow("docx")))
@ -48,7 +48,7 @@ class TestWorkflowSpecValidation(BaseTest):
mock_investigators.return_value = json.loads(investigators_response)
self.load_example_data(use_crc_data=True)
ProtocolBuilderService.ENABLED=True
app.config['PB_ENABLED'] = True
workflows = session.query(WorkflowSpecModel).all()
errors = []
for w in workflows: