From da7cae51b803e0c746e72db21c4e65c524efd8b0 Mon Sep 17 00:00:00 2001 From: Dan Funk Date: Thu, 7 May 2020 13:57:24 -0400 Subject: [PATCH 1/3] Adding a new reference file that provides greater details about the investigators related to a study. Improving the study_info script documentation to provide detailed examples of values returned based on arguments. Making the tests a little more targetted and less subject to breaking through better mocks. Allow all tests to pass even when ther protocol builder mock isn't running locally. Removing the duplication of reference files in tests and static, as this seems silly to me at the moment. --- Pipfile.lock | 2 +- crc/models/file.py | 3 +- crc/scripts/complete_template.py | 2 +- crc/scripts/study_info.py | 199 +++++++++++------- crc/services/file_service.py | 47 ++--- crc/services/study_service.py | 38 +++- crc/static/reference/investigators.xlsx | Bin 0 -> 5193 bytes example_data.py | 9 +- tests/base_test.py | 6 +- tests/data/ldap_response.json | 76 +++++++ tests/data/reference/irb_documents.xlsx | Bin 10016 -> 0 bytes tests/test_files_api.py | 8 +- tests/test_study_details_documents.py | 17 +- tests/test_study_service.py | 26 +++ tests/test_tasks_api.py | 18 +- .../test_workflow_processor_multi_instance.py | 75 ++++--- tests/test_workflow_spec_validation_api.py | 36 +++- 17 files changed, 395 insertions(+), 167 deletions(-) create mode 100644 crc/static/reference/investigators.xlsx delete mode 100644 tests/data/reference/irb_documents.xlsx diff --git a/Pipfile.lock b/Pipfile.lock index fe8efd54..9449e2ea 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -783,7 +783,7 @@ "spiffworkflow": { "editable": true, "git": "https://github.com/sartography/SpiffWorkflow.git", - "ref": "6608bb1d9cc77b906bf668804470e850ec798414" + "ref": "4cabff6e53ede311835153b17e027ddc90f02699" }, "sqlalchemy": { "hashes": [ diff --git a/crc/models/file.py b/crc/models/file.py index c2c2b045..6d83e4b2 100644 --- a/crc/models/file.py +++ b/crc/models/file.py @@ -3,8 +3,7 @@ from typing import cast from marshmallow_enum import EnumField from marshmallow_sqlalchemy import SQLAlchemyAutoSchema -from sqlalchemy import func, Index, text -from sqlalchemy.dialects import postgresql +from sqlalchemy import func, Index from sqlalchemy.dialects.postgresql import UUID from crc import db diff --git a/crc/scripts/complete_template.py b/crc/scripts/complete_template.py index a68bdd5f..81de06b8 100644 --- a/crc/scripts/complete_template.py +++ b/crc/scripts/complete_template.py @@ -52,7 +52,7 @@ Takes two arguments: message="The CompleteTemplate script requires 2 arguments. The first argument is " "the name of the docx template to use. The second " "argument is a code for the document, as " - "set in the reference document %s. " % FileService.IRB_PRO_CATEGORIES_FILE) + "set in the reference document %s. " % FileService.DOCUMENT_LIST) task_study_id = task.workflow.data[WorkflowProcessor.STUDY_ID_KEY] file_name = args[0] diff --git a/crc/scripts/study_info.py b/crc/scripts/study_info.py index ee576e9a..fb1e328d 100644 --- a/crc/scripts/study_info.py +++ b/crc/scripts/study_info.py @@ -1,72 +1,147 @@ -from ldap3.core.exceptions import LDAPSocketOpenError +import json -from crc import session, app +from crc import session from crc.api.common import ApiError from crc.models.study import StudyModel, StudySchema from crc.models.workflow import WorkflowStatus -from crc.scripts.script import Script, ScriptValidationError -from crc.services.file_service import FileService -from crc.services.ldap_service import LdapService +from crc.scripts.script import Script from crc.services.protocol_builder import ProtocolBuilderService from crc.services.study_service import StudyService -from crc.services.workflow_processor import WorkflowProcessor class StudyInfo(Script): + """Please see the detailed description that is provided below. """ - """Just your basic class that can pull in data from a few api endpoints and do a basic task.""" pb = ProtocolBuilderService() type_options = ['info', 'investigators', 'details', 'approvals', 'documents', 'protocol'] + # This is used for test/workflow validation, as well as documentation. + example_data = { + "StudyInfo": { + "info": { + "id": 12, + "title": "test", + "primary_investigator_id": 21, + "user_uid": "dif84", + "sponsor": "sponsor", + "ind_number": "1234", + "inactive": False + }, + "investigators": { + 'PI': { + 'label': 'Primary Investigator', + 'display': 'Always', + 'unique': 'Yes', + 'user_id': 'dhf8r', + 'display_name': 'Dan Funk', + 'given_name': 'Dan', + 'email': 'dhf8r@virginia.edu', + 'telephone_number': '+1 (434) 924-1723', + 'title': "E42:He's a hoopy frood", + 'department': 'E0:EN-Eng Study of Parallel Universes', + 'affiliation': 'faculty', + 'sponsor_type': 'Staff'}, + 'SC_I': { + 'label': 'Study Coordinator I', + 'display': 'Always', + 'unique': 'Yes', + 'user_id': None}, + 'DC': { + 'label': 'Department Contact', + 'display': 'Optional', + 'unique': 'Yes', + 'user_id': 'asd3v', + 'error': 'Unable to locate a user with id asd3v in LDAP'} + }, + "documents": { + 'AD_CoCApp': {'category1': 'Ancillary Document', 'category2': 'CoC Application', 'category3': '', + 'Who Uploads?': 'CRC', 'id': '12', + 'description': 'Certificate of Confidentiality Application', 'required': False, + 'study_id': 1, 'code': 'AD_CoCApp', 'display_name': 'Ancillary Document / CoC Application', + 'count': 0, 'files': []}, + 'UVACompl_PRCAppr': {'category1': 'UVA Compliance', 'category2': 'PRC Approval', 'category3': '', + 'Who Uploads?': 'CRC', 'id': '6', 'description': "Cancer Center's PRC Approval Form", + 'required': True, 'study_id': 1, 'code': 'UVACompl_PRCAppr', + 'display_name': 'UVA Compliance / PRC Approval', 'count': 1, 'files': [ + {'file_id': 10, + 'task_id': 'fakingthisout', + 'workflow_id': 2, + 'workflow_spec_id': 'docx'}], + 'status': 'complete'} + }, + "details": + {}, + "approvals": { + "study_id": 12, + "workflow_id": 321, + "display_name": "IRB API Details", + "name": "irb_api_details", + "status": WorkflowStatus.not_started.value, + "workflow_spec_id": "irb_api_details", + }, + 'protocol': { + id: 0, + } + } + } + + def example_to_string(self, key): + return json.dumps(self.example_data['StudyInfo'][key], indent=2, separators=(',', ': ')) + def get_description(self): - return """StudyInfo [TYPE], where TYPE is one of 'info', 'investigators', or 'details', 'approvals', - 'documents' or 'protocol'. - Adds details about the current study to the Task Data. The type of information required should be - provided as an argument. 'info' returns the basic information such as the title. 'Investigators' provides - detailed information about each investigator in th study. 'Details' provides a large number - of details about the study, as gathered within the protocol builder, and 'documents', - lists all the documents that can be a part of the study, with documents from Protocol Builder - marked as required, and details about any files that were uploaded' . - """ + return """ +StudyInfo [TYPE], where TYPE is one of 'info', 'investigators', 'details', 'approvals', +'documents' or 'protocol'. + +Adds details about the current study to the Task Data. The type of information required should be +provided as an argument. The following arguments are available: + +### Info ### +Returns the basic information such as the id and title +``` +{info_example} +``` + +### Investigators ### +Returns detailed information about related personnel. +The order returned is guaranteed to match the order provided in the investigators.xslx reference file. +If possible, detailed information is added in from LDAP about each personnel based on their user_id. +``` +{investigators_example} +``` + +### Details ### +Returns detailed information about variable keys read in from the Protocol Builder. + +### Approvals ### +Returns data about the status of approvals related to a study. +``` +{approvals_example} +``` + +### Documents ### +Returns a list of all documents that might be related to a study, reading all columns from the irb_documents.xsl +file. Including information about any files that were uploaded or generated that relate to a given document. +Please note this is just a few examples, ALL known document types are returned in an actual call. +``` +{documents_example} +``` + +### Protocol ### +Returns information specific to the protocol. + + + """.format(info_example=self.example_to_string("info"), + investigators_example=self.example_to_string("investigators"), + approvals_example=self.example_to_string("approvals"), + documents_example=self.example_to_string("documents"), + ) def do_task_validate_only(self, task, study_id, *args, **kwargs): """For validation only, pretend no results come back from pb""" self.check_args(args) - # Assure the reference file exists (a bit hacky, but we want to raise this error early, and cleanly.) - FileService.get_file_reference_dictionary() - data = { - "study":{ - "info": { - "id": 12, - "title": "test", - "primary_investigator_id":21, - "user_uid": "dif84", - "sponsor": "sponsor", - "ind_number": "1234", - "inactive": False - }, - "investigators": - { - "INVESTIGATORTYPE": "PI", - "INVESTIGATORTYPEFULL": "Primary Investigator", - "NETBADGEID": "dhf8r" - }, - "details": - {}, - "approvals": { - "study_id": 12, - "workflow_id": 321, - "display_name": "IRB API Details", - "name": "irb_api_details", - "status": WorkflowStatus.not_started.value, - "workflow_spec_id": "irb_api_details", - }, - 'protocol': { - id: 0, - } - } - } - self.add_data_to_task(task=task, data=data["study"]) + self.add_data_to_task(task=task, data=self.example_data["StudyInfo"]) + # Make sure all avaialble document information shows up or we can get errors on validation. self.add_data_to_task(task, {"documents": StudyService().get_documents_status(study_id)}) def do_task(self, task, study_id, *args, **kwargs): @@ -82,8 +157,7 @@ class StudyInfo(Script): schema = StudySchema() self.add_data_to_task(task, {cmd: schema.dump(study)}) if cmd == 'investigators': - pb_response = self.pb.get_investigators(study_id) - self.add_data_to_task(task, {cmd: self.organize_investigators_by_type(pb_response)}) + self.add_data_to_task(task, {cmd: StudyService().get_investigators(study_id)}) if cmd == 'details': self.add_data_to_task(task, {cmd: self.pb.get_study_details(study_id)}) if cmd == 'approvals': @@ -101,22 +175,3 @@ class StudyInfo(Script): "one of %s" % ",".join(StudyInfo.type_options)) - def organize_investigators_by_type(self, pb_investigators): - """Convert array of investigators from protocol builder into a dictionary keyed on the type""" - output = {} - for i in pb_investigators: - dict = {"user_id": i["NETBADGEID"], "type_full": i["INVESTIGATORTYPEFULL"]} - dict.update(self.get_ldap_dict_if_available(i["NETBADGEID"])) - output[i["INVESTIGATORTYPE"]] = dict - return output - - def get_ldap_dict_if_available(self, user_id): - try: - ldap_service = LdapService() - return ldap_service.user_info(user_id).__dict__ - except ApiError: - app.logger.info(str(ApiError)) - return {} - except LDAPSocketOpenError: - app.logger.info("Failed to connect to LDAP Server.") - return {} diff --git a/crc/services/file_service.py b/crc/services/file_service.py index 7698b5a2..b19b7d6e 100644 --- a/crc/services/file_service.py +++ b/crc/services/file_service.py @@ -16,7 +16,8 @@ import hashlib class FileService(object): """Provides consistent management and rules for storing, retrieving and processing files.""" - IRB_PRO_CATEGORIES_FILE = "irb_documents.xlsx" + DOCUMENT_LIST = "irb_documents.xlsx" + INVESTIGATOR_LIST = "investigators.xlsx" @staticmethod def add_workflow_spec_file(workflow_spec: WorkflowSpecModel, @@ -31,12 +32,18 @@ class FileService(object): return FileService.update_file(file_model, binary_data, content_type) + @staticmethod + def is_allowed_document(code): + data_model = FileService.get_reference_file_data(FileService.DOCUMENT_LIST) + xls = ExcelFile(data_model.data) + df = xls.parse(xls.sheet_names[0]) + return code in df['code'].values @staticmethod def add_form_field_file(study_id, workflow_id, task_id, form_field_key, name, content_type, binary_data): """Create a new file and associate it with a user task form field within a workflow. Please note that the form_field_key MUST be a known file in the irb_documents.xslx reference document.""" - if not FileService.irb_document_reference_exists(form_field_key): + if not FileService.is_allowed_document(form_field_key): raise ApiError("invalid_form_field_key", "When uploading files, the form field id must match a known document in the " "irb_docunents.xslx reference file. This code is not found in that file '%s'" % form_field_key) @@ -52,32 +59,21 @@ class FileService(object): return FileService.update_file(file_model, binary_data, content_type) @staticmethod - def irb_document_reference_exists(code): - data_model = FileService.get_reference_file_data(FileService.IRB_PRO_CATEGORIES_FILE) + def get_reference_data(reference_file_name, index_column, int_columns=[]): + """ Opens a reference file (assumes that it is xls file) and returns the data as a + dictionary, each row keyed on the given index_column name. If there are columns + that should be represented as integers, pass these as an array of int_columns, lest + you get '1.0' rather than '1' """ + data_model = FileService.get_reference_file_data(reference_file_name) xls = ExcelFile(data_model.data) df = xls.parse(xls.sheet_names[0]) - return code in df['code'].values - - @staticmethod - def get_file_reference_dictionary(): - """Loads up the xsl file that contains the IRB Pro Categories and converts it to - a Panda's data frame for processing.""" - data_model = FileService.get_reference_file_data(FileService.IRB_PRO_CATEGORIES_FILE) - xls = ExcelFile(data_model.data) - df = xls.parse(xls.sheet_names[0]) - df['id'] = df['id'].fillna(0) - df = df.astype({'id': 'Int64'}) + for c in int_columns: + df[c] = df[c].fillna(0) + df = df.astype({c: 'Int64'}) df = df.fillna('') df = df.applymap(str) - df = df.set_index('code') - # IF we need to convert the column names to something more sensible. - # df.columns = [snakeCase(x) for x in df.columns] + df = df.set_index(index_column) return json.loads(df.to_json(orient='index')) -# # Pandas is lovely, but weird. Here we drop records without an Id, and convert it to an integer. -# df = df.drop_duplicates(subset='Id').astype({'Id': 'Int64'}) - # Now we index on the ID column and convert to a dictionary, where the key is the id, and the value - # is a dictionary with all the remaining data in it. It's kinda pretty really. -# all_dict = df.set_index('Id').to_dict('index') @staticmethod def add_task_file(study_id, workflow_id, workflow_spec_id, task_id, name, content_type, binary_data, @@ -115,12 +111,12 @@ class FileService(object): @staticmethod def update_file(file_model, binary_data, content_type): - file_data_model = session.query(FileDataModel).\ + file_data_model = session.query(FileDataModel). \ filter_by(file_model_id=file_model.id, version=file_model.latest_version ).with_for_update().first() md5_checksum = UUID(hashlib.md5(binary_data).hexdigest()) - if(file_data_model is not None and md5_checksum == file_data_model.md5_hash): + if (file_data_model is not None and md5_checksum == file_data_model.md5_hash): # This file does not need to be updated, it's the same file. return file_model @@ -187,7 +183,6 @@ class FileService(object): .filter(FileDataModel.version == file_model.latest_version) \ .first() - @staticmethod def get_reference_file_data(file_name): file_model = session.query(FileModel). \ diff --git a/crc/services/study_service.py b/crc/services/study_service.py index e60d615d..58390e56 100644 --- a/crc/services/study_service.py +++ b/crc/services/study_service.py @@ -3,8 +3,9 @@ import json from typing import List from SpiffWorkflow import WorkflowException +from ldap3.core.exceptions import LDAPSocketOpenError -from crc import db, session +from crc import db, session, app from crc.api.common import ApiError from crc.models.file import FileModel, FileModelSchema from crc.models.protocol_builder import ProtocolBuilderStudy, ProtocolBuilderStatus @@ -13,6 +14,7 @@ from crc.models.study import StudyModel, Study, Category, WorkflowMetadata from crc.models.workflow import WorkflowSpecCategoryModel, WorkflowModel, WorkflowSpecModel, WorkflowState, \ WorkflowStatus from crc.services.file_service import FileService +from crc.services.ldap_service import LdapService from crc.services.protocol_builder import ProtocolBuilderService from crc.services.workflow_processor import WorkflowProcessor @@ -111,7 +113,8 @@ class StudyService(object): pb_docs = ProtocolBuilderService.get_required_docs(study_id=study_id) # Loop through all known document types, get the counts for those files, and use pb_docs to mark those required. - doc_dictionary = FileService.get_file_reference_dictionary() + doc_dictionary = FileService.get_reference_data(FileService.DOCUMENT_LIST, 'code', ['id']) + documents = {} for code, doc in doc_dictionary.items(): @@ -149,6 +152,37 @@ class StudyService(object): return documents + @staticmethod + def get_investigators(study_id): + + # Loop through all known investigator types as set in the reference file + inv_dictionary = FileService.get_reference_data(FileService.INVESTIGATOR_LIST, 'code') + + # Get PB required docs + pb_investigators = ProtocolBuilderService.get_investigators(study_id=study_id) + + """Convert array of investigators from protocol builder into a dictionary keyed on the type""" + for i_type in inv_dictionary: + pb_data = next((item for item in pb_investigators if item['INVESTIGATORTYPE'] == i_type), None) + if pb_data: + inv_dictionary[i_type]['user_id'] = pb_data["NETBADGEID"] + inv_dictionary[i_type].update(StudyService.get_ldap_dict_if_available(pb_data["NETBADGEID"])) + else: + inv_dictionary[i_type]['user_id'] = None + + return inv_dictionary + + @staticmethod + def get_ldap_dict_if_available(user_id): + try: + ldap_service = LdapService() + return ldap_service.user_info(user_id).__dict__ + except ApiError as ae: + app.logger.info(str(ae)) + return {"error": str(ae)} + except LDAPSocketOpenError: + app.logger.info("Failed to connect to LDAP Server.") + return {} @staticmethod def get_protocol(study_id): diff --git a/crc/static/reference/investigators.xlsx b/crc/static/reference/investigators.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..fdff0c19e4174c3f99c21994944733e10e218903 GIT binary patch literal 5193 zcmaJ_1yoh-(x$t+B&0#%0FsC9Zs~66K6H1d#D^l?rEmZNX(SF(5+WsCM-VB=|KNAO zE7$*CXRo#QUVH7Cz2BL6pP6UWl@O7L;Ly;};4D4M)Zy+41?=9(k<--O+R=j(n&%oaZHQ+ z^8tg{=%en`sXP;!$rdZ@oZP6#PCqt8HpCu2n4=%A3(wc&1EEiw+E5?(VY;Ayu2B!# zzhd}))SM$zQ@6AjfS|5~il%n>!*3F{Zbk$+IQ9SO8KS>E<6&p+Zf&LI>Hf^w*5l4G zFEYASfBHmNCIdKU|4b80G@6J`_qQW7)pO({If zR5<$s2KR3*E@zL$%^S`Si;BDPF%<+1qlRd45Fwf`@xi81%hddxV?E+*ecaN5y(5R@ zBZs}a&Hb_Q%=%{42(Df0+>bIUYm4MdkvUSS4l#JhhFV!t!wegetr!8r5GlsQC^z9) zMQJ@f{}z(le$+9Kv{l2G)QS*osI*xeQ3NNZ=8$U>0LxS+g|M>z9KwSJl+(5#s~V>k zGdro_UfWF#MwS9n)sSd>KeSK9Hyf`+Lf9HPv~=%Ku)yYu=|mSDTJoaB44M~zg&JqI zH%4}g$I!-%75+3<%JM2cef+T6P+CRdV+o1ZSUj9HUmi{1#^48`^K@#8>=D>; zIq)k80}$Fd*Xqrw{K%r$x~;BmGIX&wru);VcoXPx*F00Ma0Qbn2O6T?o@6U8<#WYG zKi*jdP-O8|1kxt0pV-O4OuW z;B_~A!n+$No=aNiTe~A&RXrYdUsA5-G!VY`5yz_XQBgdTn0t@ne&(0ZIi=pnxt+Ec zo}FaQ+?%=4QG<{C)Gv|c-TYi70JA- zJFDsEuMLalRK{-3&dm33gK-BYy22Y|V*CvI74b^JnL8cf2)X(_r*)$TH%;G~9*>8D z{?g|d#I(hj;C*tecMN_f_lQVgF|^!yO^m*QSzKAgp9;6MXh%4wG5^_=sK4B+*^j3` z9Eottpr|BWN$f0{pmIRRTWSjCp-0pp7_bZK-V~GrT?MqqH+A%cQ%H@@JUX5WF-x*7 z3Iq#g0M}TnahY?Sf+rWeYaGbznhnh#4xh)0kYqbaaAxa;g&1uLTaoJ+v4o;MnFhv9 zmL)&BG{KvG0Yb_V?CT%bW_=3?w$mI=(jO{8wr`v)YC=4H9j$+MAJA+Gm(R3hQ3sdAF_&@aOwWLy zYUfp7dgoN~W-_S&@hCr+lRct!S}+v@B!z1|HGHk8RI(oC8yY>kjEdv!(^^U9!=S0r zO)nsXPD?9HOuiw5bg(t_-UBUTK)6@gm7qI{#fLl?Ez_B-l^uPe3$G>4titOWoi6iv zanG*w$B4h48x3MOXk{X6HZcfjd8_x23!vGQ_hj@6tdFo&G$K^!xjQ)?6S(%zLSo*a zBjJGPxh*#^zero4Xiyd6;6=BtP<|j1qQjZy@#%wz4}ksIvQxxK;?bBC#&4=2ZZbyA zl#p#nP4>-}!fH2nAerDQ2?*1bvBF~$7Z=CzyocAWl>rQeT7D$qW6fv~WwqRn5U6gL zkv{6v76*yA*r)3|KhN!LyuKN4OKBlbEL#cz1HN-aKp>N+S()kLvTk$&HfLT)9uT6? zj$?h{Z%t&;3G4GAjE@x!?KS8zCk(VVEkp)Xp@Y|vIiidSi=P6I=$97)&0bYqZM@xd z>UR{|>uB>Vt_|DE<&CXR;j>h2ac;vZW|ze@Nv%n0*0f98_5Q2boodxfP8~xbAA6pu zejA(_$Yz&oFto^tlv{laDrhJoQ&S_D^`JkMC>C~L=5wmd$Pk0bNM70ymcDI<&z7f= z2lMGqLaO zy+z*c&^e8FLlGgK(>+EDdSj}XP1@3*#;xM3tCTgoFxWKJ-U{a8FRVPw<<7+clmRWe z##VHbbFtOmJWy9+MOXXlr6(V)iZ?TMzzbYE+hDaeMs8m60|`>Z@nf&?O$p1ics`n$ z)#baqm*0a-cfcC&7lePxdrVl~|IJDM<{@cgiiliz(FcKXDgGcZv9#5jj+pVku(wgNhQ=WQCybqx@32iv%_&ad~oavM!Yd>9;jfCuGGo}V z665xwMf*)ccBhwcYqa|dF~Y5g+8YnS7Bt!9oAxx7oe{g~t_fy4F&GO$MYE|4YhHmZ z8~8_7f(LuU%EeOM-Nn^|)7;hdCnHHwQ-d**{)2!xw)rL%Gxbu|Q6$McrtSh-=cn=u zk6CUrTC~of=(J8ikMG~#1c7OmAs=*5im6o@o)s0;7kUNcWQ+cN2@5;^Jf7(wjR$44sLdX3BC6D6dW@_!Pp@E z5WLCLJ#v6E&sW{*@;tug+mvuX^?X4`4F^ipQ7T~sH~qaXqSM{*s8)57$p^ef znch})qI#b&t19Cv&FN3B-=8~uYj>2zE~(V*hcB7ZqR*f-!;xOWwll1?G>J}Ds}fX1 zwGx-)#u)OEVe<4;a~ZlMy7e=q1iL+(%oRbT!K`19hhAYD9?A8rrRw zEZf!EguIOuyO0g@5%5c9Z>=p_&A42a=PKyMXIzU<6$&qX9okWSYeK;<}jg)>uT*`{FBZ7 zq}X`U2iI|fx*07SA$W@Ohy-%w(t`AsE|l!CPSD)68R1tB$I^VzPLwywicVPZ?j5fPOS(tH^Cw5QcES^wN$M8gQj&+@TPXiNsy;vE6CV zN~LjQ44+Ii2yLfIt}8y}2#fbsrnWE;1>n?#I@SV~s&LroE_Fod8yS{``8b#Ok%1&T ziG$9~7eoaFY!Nf|#DcX(!0*T}gG18YvkT@?-mbvY1tV$I;6W^4bl~ zXtGq0fR`Yzgn!6?z8=URS1GzJk2A6k(LcTE9BcrFy$QlFiC5JJ?F+A#M!uoxGxLjS z?+Cti+Y(r%3Zxwx2(CpJVaLuEu4goBj5hrGX`Vh0XW2_-_n=RMHJYaX7zV_sX!dnP z3A`|`>%y8*btQNNBDh~upZimfyQ$BA%Kx4L{ch(zS-+c+{1OV-mtl7P-^Ap13-{aQ zJ5v8ks$kgyv+!Rj{_j@qH&A!2^)GQi{kN5W8tvbe?^mI_9_yFnqyN2#e|KEJtKP@y zyE64lvatT6X#K8uAGYpb{Fl_je&Kge|6Tk3hPs2cUoww#Px~LB`(6EhX58(%UlIiS Zm;bT{)s>J@?p{KN9b&Ms9RK|2{{c`(wvYe- literal 0 HcmV?d00001 diff --git a/example_data.py b/example_data.py index dd53c75e..7cf8246b 100644 --- a/example_data.py +++ b/example_data.py @@ -239,7 +239,14 @@ class ExampleDataLoader: def load_reference_documents(self): file_path = os.path.join(app.root_path, 'static', 'reference', 'irb_documents.xlsx') file = open(file_path, "rb") - FileService.add_reference_file(FileService.IRB_PRO_CATEGORIES_FILE, + FileService.add_reference_file(FileService.DOCUMENT_LIST, + binary_data=file.read(), + content_type=CONTENT_TYPES['xls']) + file.close() + + file_path = os.path.join(app.root_path, 'static', 'reference', 'investigators.xlsx') + file = open(file_path, "rb") + FileService.add_reference_file(FileService.INVESTIGATOR_LIST, binary_data=file.read(), content_type=CONTENT_TYPES['xls']) file.close() diff --git a/tests/base_test.py b/tests/base_test.py index ec946b10..3511b361 100644 --- a/tests/base_test.py +++ b/tests/base_test.py @@ -207,14 +207,12 @@ class BaseTest(unittest.TestCase): study = session.query(StudyModel).first() spec = self.load_test_spec(workflow_name, category_id=category_id) workflow_model = StudyService._create_workflow_model(study, spec) - #processor = WorkflowProcessor(workflow_model) - #workflow = session.query(WorkflowModel).filter_by(study_id=study.id, workflow_spec_id=workflow_name).first() return workflow_model def create_reference_document(self): - file_path = os.path.join(app.root_path, '..', 'tests', 'data', 'reference', 'irb_documents.xlsx') + file_path = os.path.join(app.root_path, 'static', 'reference', 'irb_documents.xlsx') file = open(file_path, "rb") - FileService.add_reference_file(FileService.IRB_PRO_CATEGORIES_FILE, + FileService.add_reference_file(FileService.DOCUMENT_LIST, binary_data=file.read(), content_type=CONTENT_TYPES['xls']) file.close() diff --git a/tests/data/ldap_response.json b/tests/data/ldap_response.json index 3dc2b4e9..f42fee94 100644 --- a/tests/data/ldap_response.json +++ b/tests/data/ldap_response.json @@ -74,6 +74,82 @@ "Staff" ] } + }, + { + "attributes": { + "cn": [ + "Dan Funk (dhf8r)" + ], + "displayName": "Dan Funk", + "givenName": [ + "Dan" + ], + "mail": [ + "dhf8r@virginia.edu" + ], + "objectClass": [ + "top", + "person", + "organizationalPerson", + "inetOrgPerson", + "uvaPerson", + "uidObject" + ], + "telephoneNumber": [ + "+1 (434) 924-1723" + ], + "title": [ + "E42:He's a hoopy frood" + ], + "uvaDisplayDepartment": [ + "E0:EN-Eng Study of Parallel Universes" + ], + "uvaPersonIAMAffiliation": [ + "faculty" + ], + "uvaPersonSponsoredType": [ + "Staff" + ] + }, + "dn": "uid=dhf8r,ou=People,o=University of Virginia,c=US", + "raw": { + "cn": [ + "Dan Funk (dhf84)" + ], + "displayName": [ + "Dan Funk" + ], + "givenName": [ + "Dan" + ], + "mail": [ + "dhf8r@virginia.edu" + ], + "objectClass": [ + "top", + "person", + "organizationalPerson", + "inetOrgPerson", + "uvaPerson", + "uidObject" + ], + "telephoneNumber": [ + "+1 (434) 924-1723" + ], + "title": [ + "E42:He's a hoopy frood" + ], + "uvaDisplayDepartment": [ + "E0:EN-Eng Study of Parallel Universes" + ], + "uvaPersonIAMAffiliation": [ + "faculty" + ], + "uvaPersonSponsoredType": [ + "Staff" + ] + } } + ] } \ No newline at end of file diff --git a/tests/data/reference/irb_documents.xlsx b/tests/data/reference/irb_documents.xlsx deleted file mode 100644 index 0aaaae272f0139697454176fdf77989e86efc094..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 10016 zcmaKSbwC`-t~V5y#arB=E$&hZ#clCLi@OweEAC#3ySuwXarfdZ?rud2lyB+1@15TB z-FLEo3_ClM$^4SZWJu(tU|?~e5D^ifq(jr>q5dEO$lop&OgeT(7WPce)^_Il*4E~X zE|wOI&pl<)GUnaPsDY=CFU4+pB@qn5x!b}8D;{s5`xG`NdN{w%);)PL5^5kD9Zkec z%w4ui{5W8bI0)_$MujiVh7IWKbwX_Oc|18@$5$E6URg7xj;bWZDrv=@_r{fEQm?t> z{>Z4`_2s=*?jTUMXl}6C?^uCp_(F0z6?*igVwna^treYoi+$HqrNJBkO((}agLwde zG-^i@%{B>&(qE%vVnw*JyxZp6=BFT@ryQ#A%K&jYBhBa-lbpJtSR*Z!$$K6>P+eX$ zB#D+)tbjkike5P0lsmg~n}Y0{`UMn}{Qrpz&cBecH`TKr|QVTpHFH zD|1B69MVRe9kX0P4dT=Y3tcjCqr`bI%9H6Oe)trc1;*>NZ+X?FynVKPx!ZwG$2j9{ z)U0x%fA+KY5V|Dp0Tfnvx3gWc3ZJm6qMeSA%;nV0ri;L*lIYIT2#ei&H=&h|-Zl{h z+k_p!xr9r#pFp**`gR}PeDs7f4_>ZS6xuKnYdByue3?5@F!tfCwipkAB`3F8++g*V z@vwf{Oq@I#91J`>P97|&9w22^rht1}(mw{fZ!nHakhx7iT7@pD8j zM&_UC2`-OAzCOeF{#gdvvX+tr*AVuZOOtX0g=`pPI;Wf>A#om0)mmEws zC^uj#{2J^b37{@f%+`?Hm1arHO-_r0e;0IN%6s?;wr{NTjg%aK{W?yeSB-e@C8ZUh zwiJKa_9;h%^L#OewVha%Shb*M-1Du~@!}%X>mu6l{`URWI=i&nO0{g5SFCxl2Kt>N z{_E_ow79Q*CxrFXp@b>Dsmu61v68~5kIEbKW}xs$CK)#r(wT6@)VaHDQ*fdDhYpdc zrKdjYf^?x;M6w#~P^^elB*XM@Cgw)Qt(FOpdZkVz53m~OHmRc=x(C+Tm~gw4rqr*< zcdAO^*eHG?p%{bA*sAwn_FMWYGeaVt%xL}a6sF#cg!^SXDTTt;B_x`mY|0_No!?8f9L3|?L z-16)aH}`xRw|l>Vh(Gj`9~4N#A!0BtGZVSK8@dFAN;Km zztM1EEJngtO7q6C^!%UmWt}VbKm#xRwp_ADbCi@8SAAo-bop_(@{}Uk;gTYZ<4+>zdQ7Y7Gfp;E9d-w27XSt{=6p?7&&l z5+lju3yT^ltYJG%B$x%U`hYzCxfbKMJaOioo(1}bF3fAP=QF_gQCW6cY0*MOCp9Gd z=$afE5*#Fj=+H#xm|oLbq8YTWCU@3&aDXtnGPECx5n)Z6Yx~UHnxCP_Mr3x;Edmc) zJ`K)%T~?MVx3S7}`i0KGYJXnIytpr&04A-hZRsm=(Jez3B7F-cJ zkm~M|Q$*lcg6J*|bc@cK$Ao9j$3Fe;h)c?lS7YU^al z-V5Jnk;mwOhTL<12{v<}`o|BrOnBZlQaO#gCBjTxr%x}yM0>Ji$t>#&rR@udqV8%> zp0Wig8P=V?X98Ln?N3L%>YbrC6q~ z@0^%EiVr8@$85fFXu^sHMa5evl?_B zHGszfi}*^k@6#Oe^jrzldsa|NbNvnEhPaMfVX#k`Xoex~nX9^hu(d>-9o6>9Yq?>x zx;2ypVLv4M!$HIiIl_kt%JQ#?YlckAY+t5IuG~`T#Y`EYD!s{e0#*0V&eE1YP6*a= z#7G9au>PXbvqXA**#uBVHZpE95@mPpNLHg;XZ;*at7WyD$Ss zHZxdZk)GPg&rU_)6pdAQd@3;1!ycY8v+BnzoK;2-rX(vO)@Q165^9<~#J6W0SIfi8 z=igT3`c7B7hW^aYI^(Nr9eWUkahCRxq~~qjPNL6?8IbLzVd``aLz?7BC*Ymz2k2Cs z&H_eVB4bwwBv;nE8+8R4uoY}##sdaa7j?UvILbdp59H_Up2KDBBG2tS&L{4((-h~w z>5tMBEr9p)r+}ZfrtuT9JlkEX!e9KN7!d0@y-vv`>b&KQ^H{+xYqkf?Z(5^RJ#vMB zD<^CY-u8M9GNceb1yUkHLDg{lQ;>o8uOQ<&@Uk~GGIFqI`rWWR2O4c@tM(}>B)zLQ z6%%_|axL*a*6K@X`uzuExia~(4Q%&f^oDHF0;2O)7jvg@gY^z$O^jekYtkt-o?o8p zFw3bhn%s}SqJDh3l4oD4E+6ZwiP?8iQsVlNU1~Mn*WuatQ7vu+v0-&v6W#3Hb?ypI+@0}9uESOie$IxN<9wP6 z(I=_HnVuBcoIxI@lJoMehS4tsKkPoP;3WJQeQYjV)QZkm+b;X~aIDp|5IHV)k9L|e zS*=+oda2mP9M(3Xy43Z#VbD74F#FMQ+H2Lba5V)zy*)iOL-xAkvuE>LnV`s_LYjD! zO;Y34vFY5HtI(&|-p%W^)0|_iM;D`4SDta!83#W|$5QOh>r0U@&6bu=`5*E;TMJeB z35Wt8)^VR0l7HU$<+rx6h27Sx)sIszJ#0Ew(ONC)+upjLo-D zGm8JF13>D;zPgifcux|#kdeQ$WTkqiHkZ*H)m%L7`Ch4aFY0i#*rEkpEaS#2ZGFRK zQ^9jO9zEZo;ohTksb1T8^`^yW$+9K?7f9(be!tTp#bP?_CN7<}wRCNGyk#5T(reY) zPcw7Y}z5?CR2UbFQWA zq2A@KiWr!S_hH0hi+E*F!S7PBMzrBE16;4QonqN=a}>6+Y(0ALsKU6|Qt-H0u1OyX ztUq=yzvTZZQ^#2H>uKrQzQN03$xZcjdYXuW_}XFp)ind(J5YNq+NXa?XNj&{9_HVPa5!EDDy6>tSY*X9Yzc$il-r# z6wR;SNUl0mzZbaAxn{M#yqwPWB;t7NtTS(WAD_{}2f(Vtcz;ZnBet5VGG6L9S4v3G z(AM=}cf+*beRt(>F~P8+9es>b1Jiq?X4{E;s%ATg{G@gs__DKVBlP8ZmHt@h-bWS| zhYU=2Sxk3-Zup*32`CmikibYNzQrJy(2JCd19{LfRVBl2c4z_1y)4Upt7c`45xwA8 zHsVoGf{<@!fMGr+oGd0>tELSqq+cXk0yfwm*6EH*NM8_#M-=D64%$1MI#>cnP(cM z`o+&qn^YtK2n&l8PZJqn3nCzV+i&Z0=^NEuKY{U9A&?cu94~faUZ9ENk*$941KNs^ zaHa@8QF}n3VB5iClweccF&pRb2efq|zrX{bnN9^wvx&rFZBCFC_&%JkCTc~=G%>bN zIM*NkvFN~va(lLCeEtgFL49*v=4ZeJGtH!VV6JxEMSt#c^BKPT>NjCf7b!N4Lccf& zrnGTtUZRO(ldb-npm(mn%xh4#Y;Z0-w_RRUwB6yL$&tSwq9js-bgoDmo+N;I2WS9c zpFBQ~fxw=PG7ae_=H)Luz$jXW2iu!Z@HzF6NA3v;=PD87l`$z3^ThEiXSrJuH}}VL z_h4#0OX)}1Tr8X$sNpbv?&^B|dw4g53(=%9F$KqRj=L3KbAS5Z2D>|qmxClS#e?60m_B(PS_q`-iVs1iT4k?9!ha z#c!6~9HRJa{dciq10|3EJ0G(2K;3-JyR$j&M6VQfb@TV$hlM^prB^o;|&Ou z1l;VkYeb!drI<{((gLEf2prOdx}U{fJwIkT3Ndg<2R`d8t>9lPIv-AY*hZo7$U)@F zM*H1VxLs-6JLWE|COXn5mGoAD6Srp%?l&Uwn>oTbP&L12wba0?dE`|cCtH=ZGg-L` z!qBlx1HTKxl(4(7NSl`CE~|U!EZ+%r6Py3rK$rrNBI05-N=|F)WX&24p3yXIB-H&8 z*C#eqHgmI}UVv8%5j}**P#58uGg$m!_5i`6Op(3vw_1NUvG7kwd{asnJ-3<+QQIUJ z-z*Ig=d%m&<^uvl!gzi+=uxOXO9L3e?%OPt5ad9%P}nUv2txq|RtXfoc_N9+d5C%d zu16n@Y0^UblRcxnU|kx_KxY~uwQdiuwb>hlA(Nc6IEvJ|*P`sk0PbNXl#jYib-#S7 z7uk(zFgrmMBP&3W%D|SVZ)NxBu}k^;3PAq^s=VO7QfO#_S5uWj4bWstfhn(nG?-mq zDTr!f5%mUzUa48Nxxa0#4l;lz%z45&Bx1YbI*tCrYKsJ1PO9$j%cHk*e=DnH~VXQn!i` zrLy211%fE@^Ikwtf{ul9^AeI~^%IUQ8kh6KB=`Ub zmgQJ1j1aKeu_kkP0(D4(myLZT_%soMT&eZdim{RvsVUTo3FSLNxv_P9Q-`Ummo;#j zMq#6w*AmTGsr8SFvC=n`D7rrIVv9s(G*O~*%PJ4W2J008T~@Ao1d$!X5x# zp_W3=5ybXgx3+w#Rcd3#bmHT z@BsQ_M>EC+lhodSx?Lom5Kjg}1BTfWh7J5qID4oeOr@!)_4kUgBC~-a6FO~r7U*eg zM6=JLg=dE>p(X^U`qe~3^bJ{;59B5g7z3Utq)uHwvwL`ui{Ha*c^>jjL@@=j!xA7j z5gn$U7hm^r)g_;W%SSK9gaJESCR;*yY*}d%S?eKx30>8=Gq<>6k@+D1P7%$DF1A;8 zA@p-4SPf>(3a19&Vv>ZYDocPatcsy$p=}o8%kR)$2SW}VfYoYRmtqCSXF8ab9BW>g zx~p8oW&Ep!`20I`ET@@~K(Hl|bkEWo8zInZkryox{sF<__%mZ)rLp?0YcQ;{1rp+r zqQ6novM|n!32Sz@>3*(MyHalTkUYa$J~hK3VYTo|qHD6lfn8(J9bE&?TCAf04mo%kex z{JE$Kof1(L#{#%>Oc+x>(rX-W;^wEO@w#EgU?4#bj-Dz7k+IVIuCXQzTzAHl1u^`F zCg$y0nqVWu%IfLg3Y}><$}CCP%_u5T>gs@GDN9gp>?am!cTN3!U|Okf+zy1^^sHJpt)dW(N+dp&4!9S8uqnOJBYzA z?;wwyhVc^DrP#0rW4_)ubE5YTV|&ds9p9BxDi}kH8It|Dr1^1VG*-@Vv8Ug4FnA87 zEKRX%zm!t6bi;ZB-o7>t`nlgBX9cv~-tpJOx8Tl~SU1nSO-^_*+-!yJxpKov zXVZaic_A~YnV=I2rdu7&cJfLg@EopQs6UN3+r9tMe==tRHaQM*dul%_AkA2{;wN!5 zTIKLK6@66mXrwE44)k8oN}hrmct2#aDFd#R(XKassd6`3Kw8{u?QEpM%cjA8cVQ&g zLZea==GMm9VO=kzth!TkZ)qDK6_m1Jr|1+Vp5AWZmmp_#AhTdO{cH5%y}Bx2&SbVG zC=Z`Y>x&;dEn&aLtvx#`mQFv!9_qjU+2lg> z2&%X|$mqanxG8B}@K`uB)_&?Bm3_lx)EwFpaQ8&(BrW#1x_V=DQz!c-=+Mmbb7_j) zqpi&^{^u7jv6e(?DH;?MqyIm>c+vlQ@%~S$AwqG`YKa-6?S>9^#gak$6Jb1zYUCFv zFh;>((pRxp=Ak$P=keK3E0{)kG>gLMobOOpTa#SXS5nRQi96M@lt;SszVXU8G5Rm| z9xWO0B(jWCsKO@A3+NlDj1?Pcio(&Ld#Tw!|FWwD}mw;)m zG_CO1>^aq%p;y1gH2I1drFN|lV?)y0Wu1OZMM<{;T z-o=im-JFSnt|&@sr19p8GlIUV-iYx!uRh7eQ5Lb%$ZKdmx`3*^P;MOAG$Nmax>p=q z9s2{vrTl!`@CVv~49YyGD8)w(zBoL{bgYu$qeAlEmT{wnkMs(`sXI&Bh+$V!d8$HS~4 zJuR4IPm;+ItVI*|V{T1D{Se+4Bj}cr&)#GhGcui#OBd9fsWWxnV7f=?>TTzq1x|k- zQASLiQQhp-KXOW4^%L&dXy{}!aBrcZpsZp3$!Gxo&c!&mS{VIK!W<`RSuc^{1fJf| zVce9Yu}dxu+Riu))ut4wE1FH!qUakC#-XD~5Z=t4>Js(KDq9ZHGQ`;#x%dPn*njH` zVN~_a5ibTP+Jz##As176O>4xOv2lNQwE0e~4=V|)oGNW2l;AVIS8#Y4cN^W?Iz+@N zX36onN!W;;mu znBl70L_Fj)!JL&m{H5tVIoD8u2%MkS2}`v>bJu~LMWl>?@JmjB@KFFaZAhKcFusz! zV!A5z7N~3IBC&au)NMCe$#1=pEz|Q`D@vZ?{0KN=i#nF|E1l%6$Gcwq^}r34pQY(r z>fw~KY$&>;7hQa1v@i%&DoAfj`x}e+ni#%aA#Fd5rG=9&82=y_P+b^Kqd!C@;@)ke zaRVum8x58u*?r+r)^03sxyq#*v!mHl>DMTum$C4uG=CR4&`e1iYdr4lgfx=kx3vNj zPc3_K*gG&4_gGGUj&pXEEBOwg3P2HK#>Y~dB*b>dtjadaDV#Vo3%^0@nz_Gd*a^a0 z+{Z7iE|UmI;RaksC~n%nm)B|{Ec(Ie!fzZX&Q_7QaBo38O7!o~k%6&;n?y%!PN!fK zK5=ZyI(bvm?YOb;$q%s3PRq@$N5y%`QQl>FhF9&cm6NNs$_OWvr6uYIVi5*YPTk#_ zYH&FAvO@H|VaA5_NeAW%(z5;V^)<~^Uj3O`OJ0tu_iJ35A6fM-je zjz%!4fMgt_u>Z-@Q6QH7uVn1M60l!8Wq&7N-9u?_IKTpkS@|u}5-sHHB;Tm42F2FTThXsDzn0PtQT`b6qV2VTM?ufr$m>gf-|JJ`=K z1tha4pFltu`^V%g1Vbo8YXf;ZYa4qe14nxYYs=?LA)MWRfZHe5kXJ(0 z#bJMcRN0f@t{TejS8rBA`0*O78n`!UIK4ET@ZfbE!?+Erh?^qH$9e0vX}WM9Qr3z~`~p!DqduE~_G$H?mSi&iZ6= zBu||A7Fo>n^*~krvH)oo3zfK~$nlUA(cXpeo6ETY%`evuZXK4nqf*)SE}J-YdakJL z)nud9P;liP$cl5qD^q}=^&U}0|A)Mc^6uUPLhfqt5lqnL;>u_8?e@c%2;QWR)cmyy zAzTERGkV5905l9SSB%!nm(iyomjd$HRB=#0M5uG&eH!uIQwa3?tHJq? z?y0$wG)h7jdd|kYEr*Vrt6_^vKAf%5??QbsT*=J38%yo3K#ULg7va=jqno3@$&*B zsdWWYH?m#LYz0Tghs(x)Dmh&7RzT%I{4%@tHuy}^ToA?nTxk439@z>VnGq;%2Ra30 z_**W)TTV%7Jz;ukd?@>=;KA5ZknJ6MG%o?bLU=C+>4bJA*ZJNGzd6HKV(a71Iv0wc_!I(uwGHV&0AC!(Tpvj!*eWHcl7x=_PNCS0pI zui{wOG&mjV*)_eC}OC1MbOZn*aCUzmQSTYu_ zhLt(q8q32aX_)o1mKL*ZFHIm?TKdOxg7K%8^lWUNm6Y~HP6~zvqxaM^l=`YU#q>}e z$SH~`2gg^sVHq-LK{&|&bz#cWJx6HC-ulv|+j@tu{R6kgF}V@;XK^b+G1@a))6E8A z*HfJX>LONPMIL~J$If4Rb|bJO4}(0*NAsocIPn5`-^;v@$KM8*ah7mRPzNRHSuM@* z#B~Z)G$tZq~^Ye?RrQ%i-_q!fr*tEfc{=P4EAZS=(ky z_Ufn|@68~oL9Li1eOhqD9q{a@vwuCnoFw_q!r;LH??b}z1Y>bnK$ib4d_XKqzp-3Q zRdc$%Tgc|sa_TMgGgSKoBJ{@)AUq*({onN-&n`tn*xJg$$jU)S$<@ZlUh8)jCU*4M zr2S7lLI+f1xwpMBfKxD7;)NpbC=IL$O3tHNt;|-_9f5-0rbX(u)mWz^T-X{#uOS&? zI>f@XvW3RCAK_Ja@{U`LmWEBP+C)~vP*&QI&>z<_;$iNy zj5Oiek?(wi)WC#FnNZFr_HT72ANbe^(hcNO^tkI~8h-{~?n3chsN%k28e>4Vqn{#- z2V4E15gPTsX^W=-X|`7%5b}Br=8dBmxGC1&cr$GedqR72XM+Z@%} z=JjN|`*wq!J4%z0g~#4*x=iL|5L@W`y}4(XmiOjlOSPm2eRQeZ39>neR-S($CluYQ zXy^CM7tGAu!wJD)$(ZO@fEe-Qy>WeA`D zua4*M9R7U1Jg4RUA}^%h9R6#P?(bawjAx&pI)9NZgegRhKWnW1KKaiO;W^~}i}E4Z zJa6KE1K__;{qx#5QT%=4pWgoY2>eA2B>(k}{X2s{9ntfp@E5%!`;)=H zuZX`-|MPG@d#k@_mHdzC|Mp#fpZ%w4{ipQEl>hZO%S*w*KQDm{`FKNIXDHS4xBmx+ Cq4|;k diff --git a/tests/test_files_api.py b/tests/test_files_api.py index 646a2107..d8ac2a3a 100644 --- a/tests/test_files_api.py +++ b/tests/test_files_api.py @@ -1,12 +1,14 @@ import io import json from datetime import datetime +from unittest.mock import patch from crc import session from crc.models.file import FileModel, FileType, FileModelSchema, FileDataModel from crc.models.workflow import WorkflowSpecModel from crc.services.file_service import FileService from crc.services.workflow_processor import WorkflowProcessor +from example_data import ExampleDataLoader from tests.base_test import BaseTest @@ -102,7 +104,7 @@ class TestFilesApi(BaseTest): self.assertEqual("application/vnd.ms-excel", file.content_type) def test_set_reference_file_bad_extension(self): - file_name = FileService.IRB_PRO_CATEGORIES_FILE + file_name = FileService.DOCUMENT_LIST data = {'file': (io.BytesIO(b"abcdef"), "does_not_matter.ppt")} rv = self.app.put('/v1.0/reference_file/%s' % file_name, data=data, follow_redirects=True, content_type='multipart/form-data', headers=self.logged_in_headers()) @@ -119,7 +121,9 @@ class TestFilesApi(BaseTest): self.assertEqual(b"abcdef", data_out) def test_list_reference_files(self): - file_name = FileService.IRB_PRO_CATEGORIES_FILE + ExampleDataLoader.clean_db() + + file_name = FileService.DOCUMENT_LIST data = {'file': (io.BytesIO(b"abcdef"), file_name)} rv = self.app.put('/v1.0/reference_file/%s' % file_name, data=data, follow_redirects=True, content_type='multipart/form-data', headers=self.logged_in_headers()) diff --git a/tests/test_study_details_documents.py b/tests/test_study_details_documents.py index e85dc87d..bf47b620 100644 --- a/tests/test_study_details_documents.py +++ b/tests/test_study_details_documents.py @@ -24,7 +24,11 @@ class TestStudyDetailsDocumentsScript(BaseTest): convention that we are implementing for the IRB. """ - def test_validate_returns_error_if_reference_files_do_not_exist(self): + @patch('crc.services.protocol_builder.requests.get') + def test_validate_returns_error_if_reference_files_do_not_exist(self, mock_get): + mock_get.return_value.ok = True + mock_get.return_value.text = self.protocol_builder_response('required_docs.json') + self.load_example_data() self.create_reference_document() study = session.query(StudyModel).first() @@ -36,7 +40,7 @@ class TestStudyDetailsDocumentsScript(BaseTest): # Remove the reference file. file_model = db.session.query(FileModel). \ filter(FileModel.is_reference == True). \ - filter(FileModel.name == FileService.IRB_PRO_CATEGORIES_FILE).first() + filter(FileModel.name == FileService.DOCUMENT_LIST).first() if file_model: db.session.query(FileDataModel).filter(FileDataModel.file_model_id == file_model.id).delete() db.session.query(FileModel).filter(FileModel.id == file_model.id).delete() @@ -46,7 +50,12 @@ class TestStudyDetailsDocumentsScript(BaseTest): with self.assertRaises(ApiError): StudyInfo().do_task_validate_only(task, study.id, "documents") - def test_no_validation_error_when_correct_file_exists(self): + @patch('crc.services.protocol_builder.requests.get') + def test_no_validation_error_when_correct_file_exists(self, mock_get): + + mock_get.return_value.ok = True + mock_get.return_value.text = self.protocol_builder_response('required_docs.json') + self.load_example_data() self.create_reference_document() study = session.query(StudyModel).first() @@ -58,7 +67,7 @@ class TestStudyDetailsDocumentsScript(BaseTest): def test_load_lookup_data(self): self.create_reference_document() - dict = FileService.get_file_reference_dictionary() + dict = FileService.get_reference_data(FileService.DOCUMENT_LIST, 'code', ['id']) self.assertIsNotNone(dict) def get_required_docs(self): diff --git a/tests/test_study_service.py b/tests/test_study_service.py index 12837309..80fe10da 100644 --- a/tests/test_study_service.py +++ b/tests/test_study_service.py @@ -163,3 +163,29 @@ class TestStudyService(BaseTest): # 'workflow_id': 456, # 'workflow_spec_id': 'irb_api_details', # 'status': 'complete', + + @patch('crc.services.protocol_builder.ProtocolBuilderService.get_investigators') # mock_docs + def test_get_personnel(self, mock_docs): + self.load_example_data() + + # mock out the protocol builder + docs_response = self.protocol_builder_response('investigators.json') + mock_docs.return_value = json.loads(docs_response) + + workflow = self.create_workflow('docx') # The workflow really doesnt matter in this case. + investigators = StudyService().get_investigators(workflow.study_id) + + self.assertEquals(9, len(investigators)) + + # dhf8r is in the ldap mock data. + self.assertEquals("dhf8r", investigators['PI']['user_id']) + self.assertEquals("Dan Funk", investigators['PI']['display_name']) # Data from ldap + self.assertEquals("Primary Investigator", investigators['PI']['label']) # Data from xls file. + self.assertEquals("Always", investigators['PI']['display']) # Data from xls file. + + # asd3v is not in ldap, so an error should be returned. + self.assertEquals("asd3v", investigators['DC']['user_id']) + self.assertEquals("Unable to locate a user with id asd3v in LDAP", investigators['DC']['error']) # Data from ldap + + # No value is provided for Department Chair + self.assertIsNone(investigators['DEPT_CH']['user_id']) diff --git a/tests/test_tasks_api.py b/tests/test_tasks_api.py index ea888fdd..cd2b6e87 100644 --- a/tests/test_tasks_api.py +++ b/tests/test_tasks_api.py @@ -1,5 +1,6 @@ import json import os +import random from unittest.mock import patch from crc import session, app @@ -297,7 +298,7 @@ class TestTasksApi(BaseTest): self.assertEquals(1, len(tasks)) self.assertEquals("UserTask", tasks[0].type) self.assertEquals(MultiInstanceType.sequential, tasks[0].mi_type) - self.assertEquals(3, tasks[0].mi_count) + self.assertEquals(9, tasks[0].mi_count) def test_lookup_endpoint_for_task_field_enumerations(self): @@ -382,7 +383,7 @@ class TestTasksApi(BaseTest): @patch('crc.services.protocol_builder.requests.get') def test_parallel_multi_instance(self, mock_get): - # Assure we get three investigators back from the API Call, as set in the investigators.json file. + # Assure we get nine investigators back from the API Call, as set in the investigators.json file. mock_get.return_value.ok = True mock_get.return_value.text = self.protocol_builder_response('investigators.json') @@ -391,19 +392,14 @@ class TestTasksApi(BaseTest): workflow = self.create_workflow('multi_instance_parallel') tasks = self.get_workflow_api(workflow).user_tasks - self.assertEquals(3, len(tasks)) + self.assertEquals(9, len(tasks)) self.assertEquals("UserTask", tasks[0].type) self.assertEquals("MutiInstanceTask", tasks[0].name) self.assertEquals("Gather more information", tasks[0].title) - self.complete_form(workflow, tasks[0], {"investigator":{"email": "dhf8r@virginia.edu"}}) - tasks = self.get_workflow_api(workflow).user_tasks - - self.complete_form(workflow, tasks[2], {"investigator":{"email": "abc@virginia.edu"}}) - tasks = self.get_workflow_api(workflow).user_tasks - - self.complete_form(workflow, tasks[1], {"investigator":{"email": "def@virginia.edu"}}) - tasks = self.get_workflow_api(workflow).user_tasks + for i in random.sample(range(9), 9): + self.complete_form(workflow, tasks[i], {"investigator":{"email": "dhf8r@virginia.edu"}}) + tasks = self.get_workflow_api(workflow).user_tasks workflow = self.get_workflow_api(workflow) self.assertEquals(WorkflowStatus.complete, workflow.status) diff --git a/tests/test_workflow_processor_multi_instance.py b/tests/test_workflow_processor_multi_instance.py index 211f09ca..f2252129 100644 --- a/tests/test_workflow_processor_multi_instance.py +++ b/tests/test_workflow_processor_multi_instance.py @@ -13,6 +13,23 @@ from tests.base_test import BaseTest class TestWorkflowProcessorMultiInstance(BaseTest): """Tests the Workflow Processor as it deals with a Multi-Instance task""" + mock_investigator_response = {'PI': { + 'label': 'Primary Investigator', + 'display': 'Always', + 'unique': 'Yes', + 'user_id': 'dhf8r', + 'display_name': 'Dan Funk'}, + 'SC_I': { + 'label': 'Study Coordinator I', + 'display': 'Always', + 'unique': 'Yes', + 'user_id': None}, + 'DC': { + 'label': 'Department Contact', + 'display': 'Optional', + 'unique': 'Yes', + 'user_id': 'asd3v', + 'error': 'Unable to locate a user with id asd3v in LDAP'}} def _populate_form_with_random_data(self, task): WorkflowProcessor.populate_form_with_random_data(task) @@ -21,11 +38,10 @@ class TestWorkflowProcessorMultiInstance(BaseTest): workflow_model = StudyService._create_workflow_model(study_model, spec_model) return WorkflowProcessor(workflow_model) - @patch('crc.services.protocol_builder.requests.get') - def test_create_and_complete_workflow(self, mock_get): + @patch('crc.services.study_service.StudyService.get_investigators') + def test_create_and_complete_workflow(self, mock_study_service): # This depends on getting a list of investigators back from the protocol builder. - mock_get.return_value.ok = True - mock_get.return_value.text = self.protocol_builder_response('investigators.json') + mock_study_service.return_value = self.mock_investigator_response self.load_example_data() workflow_spec_model = self.load_test_spec("multi_instance") @@ -40,16 +56,8 @@ class TestWorkflowProcessorMultiInstance(BaseTest): task = next_user_tasks[0] - self.assertEquals( - { - 'DC': {'user_id': 'asd3v', 'type_full': 'Department Contact'}, - 'IRBC': {'user_id': 'asdf32', 'type_full': 'IRB Coordinator'}, - 'PI': {'user_id': 'dhf8r', 'type_full': 'Primary Investigator'} - }, - task.data['StudyInfo']['investigators']) - self.assertEqual(WorkflowStatus.user_input_required, processor.get_status()) - self.assertEquals("asd3v", task.data["investigator"]["user_id"]) + self.assertEquals("dhf8r", task.data["investigator"]["user_id"]) self.assertEqual("MutiInstanceTask", task.get_name()) api_task = WorkflowService.spiff_task_to_api_task(task) @@ -79,23 +87,21 @@ class TestWorkflowProcessorMultiInstance(BaseTest): processor.do_engine_steps() task = processor.bpmn_workflow.last_task - self.assertEquals( - { - 'DC': {'user_id': 'asd3v', 'type_full': 'Department Contact', 'email': 'asd3v@virginia.edu'}, - 'IRBC': {'user_id': 'asdf32', 'type_full': 'IRB Coordinator', "email": "asdf32@virginia.edu"}, - 'PI': {'user_id': 'dhf8r', 'type_full': 'Primary Investigator', "email": "dhf8r@virginia.edu"} - }, + expected = self.mock_investigator_response + expected['PI']['email'] = "asd3v@virginia.edu" + expected['SC_I']['email'] = "asdf32@virginia.edu" + expected['DC']['email'] = "dhf8r@virginia.edu" + self.assertEquals(expected, task.data['StudyInfo']['investigators']) self.assertEqual(WorkflowStatus.complete, processor.get_status()) - - @patch('crc.services.protocol_builder.requests.get') - def test_create_and_complete_workflow_parallel(self, mock_get): + @patch('crc.services.study_service.StudyService.get_investigators') + def test_create_and_complete_workflow_parallel(self, mock_study_service): """Unlike the test above, the parallel task allows us to complete the items in any order.""" - mock_get.return_value.ok = True - mock_get.return_value.text = self.protocol_builder_response('investigators.json') + # This depends on getting a list of investigators back from the protocol builder. + mock_study_service.return_value = self.mock_investigator_response self.load_example_data() workflow_spec_model = self.load_test_spec("multi_instance_parallel") @@ -110,16 +116,8 @@ class TestWorkflowProcessorMultiInstance(BaseTest): # We can complete the tasks out of order. task = next_user_tasks[2] - self.assertEquals( - { - 'DC': {'user_id': 'asd3v', 'type_full': 'Department Contact'}, - 'IRBC': {'user_id': 'asdf32', 'type_full': 'IRB Coordinator'}, - 'PI': {'user_id': 'dhf8r', 'type_full': 'Primary Investigator'} - }, - task.data['StudyInfo']['investigators']) - self.assertEqual(WorkflowStatus.user_input_required, processor.get_status()) - self.assertEquals("dhf8r", task.data["investigator"]["user_id"]) # The last of the tasks + self.assertEquals("asd3v", task.data["investigator"]["user_id"]) # The last of the tasks api_task = WorkflowService.spiff_task_to_api_task(task) self.assertEquals(MultiInstanceType.parallel, api_task.mi_type) @@ -142,12 +140,11 @@ class TestWorkflowProcessorMultiInstance(BaseTest): processor.do_engine_steps() # Completing the tasks out of order, still provides the correct information. - self.assertEquals( - { - 'DC': {'user_id': 'asd3v', 'type_full': 'Department Contact', 'email': 'asd3v@virginia.edu'}, - 'IRBC': {'user_id': 'asdf32', 'type_full': 'IRB Coordinator', "email": "asdf32@virginia.edu"}, - 'PI': {'user_id': 'dhf8r', 'type_full': 'Primary Investigator', "email": "dhf8r@virginia.edu"} - }, + expected = self.mock_investigator_response + expected['PI']['email'] = "asd3v@virginia.edu" + expected['SC_I']['email'] = "asdf32@virginia.edu" + expected['DC']['email'] = "dhf8r@virginia.edu" + self.assertEquals(expected, task.data['StudyInfo']['investigators']) self.assertEqual(WorkflowStatus.complete, processor.get_status()) diff --git a/tests/test_workflow_spec_validation_api.py b/tests/test_workflow_spec_validation_api.py index cbdbffc1..676a8a96 100644 --- a/tests/test_workflow_spec_validation_api.py +++ b/tests/test_workflow_spec_validation_api.py @@ -1,9 +1,11 @@ import json import unittest +from unittest.mock import patch from crc import session from crc.api.common import ApiErrorSchema from crc.models.file import FileModel +from crc.models.protocol_builder import ProtocolBuilderStudySchema from crc.models.workflow import WorkflowSpecModel, WorkflowSpecModelSchema, WorkflowModel, WorkflowSpecCategoryModel from tests.base_test import BaseTest @@ -18,7 +20,22 @@ class TestWorkflowSpecValidation(BaseTest): json_data = json.loads(rv.get_data(as_text=True)) return ApiErrorSchema(many=True).load(json_data) - def test_successful_validation_of_test_workflows(self): + @patch('crc.services.protocol_builder.ProtocolBuilderService.get_investigators') # mock_studies + @patch('crc.services.protocol_builder.ProtocolBuilderService.get_required_docs') # mock_docs + @patch('crc.services.protocol_builder.ProtocolBuilderService.get_study_details') # mock_details + @patch('crc.services.protocol_builder.ProtocolBuilderService.get_studies') # mock_studies + def test_successful_validation_of_test_workflows(self, mock_studies, mock_details, mock_docs, mock_investigators): + + # Mock Protocol Builder responses + studies_response = self.protocol_builder_response('user_studies.json') + mock_studies.return_value = ProtocolBuilderStudySchema(many=True).loads(studies_response) + details_response = self.protocol_builder_response('study_details.json') + mock_details.return_value = json.loads(details_response) + docs_response = self.protocol_builder_response('required_docs.json') + mock_docs.return_value = json.loads(docs_response) + investigators_response = self.protocol_builder_response('investigators.json') + mock_investigators.return_value = json.loads(investigators_response) + self.assertEqual(0, len(self.validate_workflow("parallel_tasks"))) self.assertEqual(0, len(self.validate_workflow("decision_table"))) self.assertEqual(0, len(self.validate_workflow("docx"))) @@ -28,7 +45,22 @@ class TestWorkflowSpecValidation(BaseTest): self.assertEqual(0, len(self.validate_workflow("study_details"))) self.assertEqual(0, len(self.validate_workflow("two_forms"))) - def test_successful_validation_of_auto_loaded_workflows(self): + @patch('crc.services.protocol_builder.ProtocolBuilderService.get_investigators') # mock_studies + @patch('crc.services.protocol_builder.ProtocolBuilderService.get_required_docs') # mock_docs + @patch('crc.services.protocol_builder.ProtocolBuilderService.get_study_details') # mock_details + @patch('crc.services.protocol_builder.ProtocolBuilderService.get_studies') # mock_studies + def test_successful_validation_of_auto_loaded_workflows(self, mock_studies, mock_details, mock_docs, mock_investigators): + + # Mock Protocol Builder responses + studies_response = self.protocol_builder_response('user_studies.json') + mock_studies.return_value = ProtocolBuilderStudySchema(many=True).loads(studies_response) + details_response = self.protocol_builder_response('study_details.json') + mock_details.return_value = json.loads(details_response) + docs_response = self.protocol_builder_response('required_docs.json') + mock_docs.return_value = json.loads(docs_response) + investigators_response = self.protocol_builder_response('investigators.json') + mock_investigators.return_value = json.loads(investigators_response) + self.load_example_data() workflows = session.query(WorkflowSpecModel).all() errors = [] From 99503fb8324078c44fa6615bba13cc6f96aeb53d Mon Sep 17 00:00:00 2001 From: Dan Funk Date: Mon, 11 May 2020 17:42:24 -0400 Subject: [PATCH 2/3] missed a file in the last commit. --- crc/scripts/study_info.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crc/scripts/study_info.py b/crc/scripts/study_info.py index 538b4d88..ae5a10ed 100644 --- a/crc/scripts/study_info.py +++ b/crc/scripts/study_info.py @@ -5,6 +5,7 @@ from crc.api.common import ApiError from crc.models.study import StudyModel, StudySchema from crc.models.workflow import WorkflowStatus from crc.scripts.script import Script +from crc.services.file_service import FileService from crc.services.protocol_builder import ProtocolBuilderService from crc.services.study_service import StudyService @@ -141,7 +142,8 @@ Returns information specific to the protocol. """For validation only, pretend no results come back from pb""" self.check_args(args) # Assure the reference file exists (a bit hacky, but we want to raise this error early, and cleanly.) - FileService.get_file_reference_dictionary() + FileService.get_reference_file_data(FileService.DOCUMENT_LIST) + FileService.get_reference_file_data(FileService.INVESTIGATOR_LIST) data = { "study":{ "info": { From e3573bb7df1a952b14962ec4ec56d4bcb7eb97c6 Mon Sep 17 00:00:00 2001 From: Dan Funk Date: Tue, 12 May 2020 08:48:26 -0400 Subject: [PATCH 3/3] Really just flailing now as I try to get tests to pass in Jenkins when they all work just fine on my box. --- Pipfile.lock | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index 9449e2ea..8050cb3a 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -478,11 +478,11 @@ }, "marshmallow": { "hashes": [ - "sha256:56663fa1d5385c14c6a1236badd166d6dee987a5f64d2b6cc099dadf96eb4f09", - "sha256:f12203bf8d94c410ab4b8d66edfde4f8a364892bde1f6747179765559f93d62a" + "sha256:c2673233aa21dde264b84349dc2fd1dce5f30ed724a0a00e75426734de5b84ab", + "sha256:f88fe96434b1f0f476d54224d59333eba8ca1a203a2695683c1855675c4049a7" ], "index": "pypi", - "version": "==3.5.2" + "version": "==3.6.0" }, "marshmallow-enum": { "hashes": [ @@ -783,7 +783,7 @@ "spiffworkflow": { "editable": true, "git": "https://github.com/sartography/SpiffWorkflow.git", - "ref": "4cabff6e53ede311835153b17e027ddc90f02699" + "ref": "f626ac6d4f035f3a65a058320efd8d33d1ec652a" }, "sqlalchemy": { "hashes": [ @@ -938,11 +938,11 @@ }, "pytest": { "hashes": [ - "sha256:0e5b30f5cb04e887b91b1ee519fa3d89049595f428c1db76e73bd7f17b09b172", - "sha256:84dde37075b8805f3d1f392cc47e38a0e59518fb46a431cfdaf7cf1ce805f970" + "sha256:95c710d0a72d91c13fae35dce195633c929c3792f54125919847fdcdf7caa0d3", + "sha256:eb2b5e935f6a019317e455b6da83dd8650ac9ffd2ee73a7b657a30873d67a698" ], "index": "pypi", - "version": "==5.4.1" + "version": "==5.4.2" }, "six": { "hashes": [