diff --git a/Pipfile b/Pipfile index 4fc1631c..0a498025 100644 --- a/Pipfile +++ b/Pipfile @@ -39,13 +39,14 @@ requests = "*" sentry-sdk = {extras = ["flask"],version = "==0.14.4"} sphinx = "*" swagger-ui-bundle = "*" -spiffworkflow = {git = "https://github.com/sartography/SpiffWorkflow.git", ref = "master"} +spiffworkflow = {git = "https://github.com/sartography/SpiffWorkflow.git",ref = "master"} webtest = "*" werkzeug = "*" xlrd = "*" xlsxwriter = "*" pygithub = "*" python-box = "*" +python-levenshtein = "*" [requires] python_version = "3.8" diff --git a/Pipfile.lock b/Pipfile.lock index 1290c7c7..251d34ba 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "81609b24d5be29db98b75d9d2597c20cee854a0ce6036fbd9b2dd58707852e3f" + "sha256": "621d57ec513f24c665dd34e08ae5a19fc27784e87c55bb4d2a91a1d48a473081" }, "pipfile-spec": 6, "requires": { @@ -690,6 +690,13 @@ ], "version": "==1.0.4" }, + "python-levenshtein": { + "hashes": [ + "sha256:033a11de5e3d19ea25c9302d11224e1a1898fe5abd23c61c7c360c25195e3eb1" + ], + "index": "pypi", + "version": "==0.12.0" + }, "pytz": { "hashes": [ "sha256:a494d53b6d39c3c6e44c3bec237336e14305e4f29bbf800b599253057fbb79ed", diff --git a/crc/models/file.py b/crc/models/file.py index 8afed6cd..8693b7e5 100644 --- a/crc/models/file.py +++ b/crc/models/file.py @@ -166,10 +166,12 @@ class LookupDataModel(db.Model): # query with: # search_results = LookupDataModel.query.filter(LookupDataModel.label.match("INTERNAL")).all() + __ts_vector__ = func.to_tsvector('simple', label) + __table_args__ = ( Index( 'ix_lookupdata_tsv', - func.to_tsvector('simple', label), # Use simple, not english to keep stop words in place. + __ts_vector__, # Use simple, not english to keep stop words in place. postgresql_using='gin' ), ) diff --git a/crc/scripts/ldap.py b/crc/scripts/ldap.py index 577d4a75..2e7d1e6c 100644 --- a/crc/scripts/ldap.py +++ b/crc/scripts/ldap.py @@ -1,19 +1,24 @@ import copy +from flask import g + from crc import app from crc.api.common import ApiError from crc.scripts.script import Script from crc.services.ldap_service import LdapService +from crc.services.user_service import UserService class Ldap(Script): """This Script allows to be introduced as part of a workflow and called from there, taking - a UID (or several) as input and looking it up through LDAP to return the person's details """ + a UID (or several) as input and looking it up through LDAP to return the person's details. + If no user id is specified, returns information about the current user.""" def get_description(self): return """ Attempts to create a dictionary with person details, using the -provided argument (a UID) and look it up through LDAP. +provided argument (a UID) and look it up through LDAP. If no UID is +provided, then returns information about the current user. Examples: supervisor_info = ldap(supervisor_uid) // Sets the supervisor information to ldap details for the given uid. @@ -26,13 +31,15 @@ supervisor_info = ldap(supervisor_uid) // Sets the supervisor information to l return self.set_users_info_in_task(task, args) def set_users_info_in_task(self, task, args): - if len(args) != 1: - raise ApiError(code="missing_argument", - message="Ldap takes a single argument, the " - "UID for the person we want to look up") - uid = args[0] - user_info_dict = {} - + if len(args) > 1: + raise ApiError(code="invalid_argument", + message="Ldap takes at most one argument, the " + "UID for the person we want to look up.") + if len(args) < 1: + if UserService.has_user(): + uid = UserService.current_user().uid + else: + uid = args[0] user_info = LdapService.user_info(uid) user_info_dict = { "display_name": user_info.display_name, diff --git a/crc/services/lookup_service.py b/crc/services/lookup_service.py index c9eb1dd8..cfe00615 100644 --- a/crc/services/lookup_service.py +++ b/crc/services/lookup_service.py @@ -181,20 +181,22 @@ class LookupService(object): if len(query) > 0: if ' ' in query: terms = query.split(' ') - new_terms = ["'%s'" % query] + new_terms = [] for t in terms: new_terms.append("%s:*" % t) - new_query = ' | '.join(new_terms) + new_query = ' & '.join(new_terms) + new_query = "'%s' | %s" % (query, new_query) else: new_query = "%s:*" % query - # Run the full text query - db_query = db_query.filter(LookupDataModel.label.match(new_query)) - # But hackishly order by like, which does a good job of - # pulling more relevant matches to the top. + db_query = db_query.filter( + LookupDataModel.__ts_vector__.match(new_query, postgresql_regconfig='simple')) + + # Hackishly order by like, which does a good job of pulling more relevant matches to the top. db_query = db_query.order_by(desc(LookupDataModel.label.like("%" + query + "%"))) logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO) + logging.info(db_query) result = db_query.limit(limit).all() logging.getLogger('sqlalchemy.engine').setLevel(logging.ERROR) return result diff --git a/crc/static/bpmn/irb_api_personnel/irb_api_personnel.bpmn b/crc/static/bpmn/irb_api_personnel/irb_api_personnel.bpmn index 4787f50d..6293a96c 100644 --- a/crc/static/bpmn/irb_api_personnel/irb_api_personnel.bpmn +++ b/crc/static/bpmn/irb_api_personnel/irb_api_personnel.bpmn @@ -1,5 +1,5 @@ - + Flow_0kcrx5l @@ -27,7 +27,7 @@ #### {{ pi.PI.display_name }} ##### Title: {{ pi.PI.title }} -##### Department: {{ PI_department }} +##### Department: {{ pi.PI.department }} ##### Affiliation: {{ pi.PI.affiliation }} @@ -43,23 +43,24 @@ - Flow_07f7kkd + Flow_147b9li Flow_1mplloa + Flow_1dcsioh - Flow_1vgepkq + Flow_17uqguj pi = {x:investigators[x] for x in investigators.keys() if x[:2] == 'PI'} - Flow_1vgepkq + Flow_17uqguj Flow_147b9li Flow_00prawo - + len(pi.keys()) == 1 @@ -92,122 +93,80 @@ ##### Title: {{ sc.title }} - - ##### Department: {{ sc.department }} ##### Affiliation: {{ sc.affiliation }} - - + + - - - Flow_0xifvai Flow_1n0k4pd - + len(scs.keys()) == 0 - - Flow_147b9li - Flow_1grahhv - LDAP_dept = pi.PI.department -length_LDAP_dept = len(LDAP_dept) -E0_start = LDAP_dept.find("E0:") + 3 -E0_slice = LDAP_dept[E0_start:length_LDAP_dept] -E0_first_hyphen = E0_slice.find("-") -E0_dept_start = E0_first_hyphen + 1 -E0_school = E0_slice[0:E0_first_hyphen] -isSpace = " " in E0_slice -E0_spec = "" -E0_dept = "" - -if isSpace: - E0_first_space = E0_slice.find(" ") - E0_spec_start = E0_first_space + 1 - E0_spec_end = len(E0_slice) - E0_dept = E0_slice[E0_dept_start:E0_first_space] - E0_spec = E0_slice[E0_spec_start:E0_spec_end] - - - - - - - Flow_1grahhv - Flow_07f7kkd - - - - - - - - - - - - - - - - + + + - - - - + + + + - + - - + + - - + + - + - - + + - - - + + + - + - - + + - + + + + + - - + + @@ -220,37 +179,31 @@ if isSpace: - + - + - + - + - + - + - + - - - - - - - + diff --git a/tests/data/enum_options_from_file/customer_list.xls b/tests/data/enum_options_from_file/customer_list.xls index d697bb67..1ed72dd7 100644 Binary files a/tests/data/enum_options_from_file/customer_list.xls and b/tests/data/enum_options_from_file/customer_list.xls differ diff --git a/tests/data/irb_api_personnel/irb_api_personnel.bpmn b/tests/data/irb_api_personnel/irb_api_personnel.bpmn new file mode 100644 index 00000000..e6ea57f7 --- /dev/null +++ b/tests/data/irb_api_personnel/irb_api_personnel.bpmn @@ -0,0 +1,206 @@ + + + + + Flow_0kcrx5l + + + Flow_0kcrx5l + Flow_1dcsioh + investigators = study_info('investigators') + + + ## The following information was gathered: +{% if pi|length == 1 %} +### PI: {{ pi.PI.display_name }} + * Edit Acess? {{ pi.edit_access }} + * Send Emails? {{ pi.emails }} + * Experience: {{ pi.experience }} +{% else %} +### No PI in PB +{% endif %} + Flow_1n0k4pd + Flow_1oqem42 + + + ### Please provide supplemental information for: +#### {{ pi.PI.display_name }} +##### Title: {{ pi.PI.title }} + +##### Department: {{ pi.PI.department }} +##### Affiliation: {{ pi.PI.affiliation }} + + + + + + + + + + + + + + + Flow_147b9li + Flow_1mplloa + + + + + + + Flow_1dcsioh + Flow_17uqguj + pi = {x:investigators[x] for x in investigators.keys() if x[:2] == 'PI'} + + + Flow_17uqguj + Flow_147b9li + Flow_00prawo + + + + len(pi.keys()) == 0 + + + No PI entered in PB + Flow_00prawo + Flow_14ti38o + + + + Flow_1mplloa + Flow_14ti38o + Flow_0elbjpd + scs = {x:investigators[x] for x in investigators.keys() if x[:3] == 'SC_'} + + + Flow_0elbjpd + Flow_0xifvai + Flow_1oqem42 + + + + + ### Please provide supplemental information for: +#### {{ sc.display_name }} +##### Title: {{ sc.title }} + + +##### Department: {{ sc.department }} +##### Affiliation: {{ sc.affiliation }} + + + + + + + Flow_0xifvai + Flow_1n0k4pd + + + + len(scs.keys()) == 0 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/ldap/test_ldap_lookup_script.py b/tests/ldap/test_ldap_lookup_script.py index 0a88a899..0f8f48b1 100644 --- a/tests/ldap/test_ldap_lookup_script.py +++ b/tests/ldap/test_ldap_lookup_script.py @@ -1,9 +1,12 @@ +from flask import g + from tests.base_test import BaseTest +from crc import db +from crc.models.user import UserModel from crc.services.workflow_processor import WorkflowProcessor from crc.scripts.ldap import Ldap from crc.api.common import ApiError -from crc import db, mail class TestLdapLookupScript(BaseTest): @@ -44,6 +47,19 @@ class TestLdapLookupScript(BaseTest): with(self.assertRaises(ApiError)): user_details = script.do_task(task, workflow.study_id, workflow.id, "PIComputingID") + def test_get_current_user_details(self): + self.load_example_data() + self.create_reference_document() + workflow = self.create_workflow('empty_workflow') + processor = WorkflowProcessor(workflow) + task = processor.next_task() + + script = Ldap() + g.user = db.session.query(UserModel).filter(UserModel.uid=='dhf8r').first() + user_details = script.do_task(task, workflow.study_id, workflow.id) + self.assertEqual(user_details['display_name'], 'Dan Funk') + + def test_bpmn_task_receives_user_details(self): workflow = self.create_workflow('ldap_replace') diff --git a/tests/test_lookup_service.py b/tests/test_lookup_service.py index a27427f4..0b7a8ddb 100644 --- a/tests/test_lookup_service.py +++ b/tests/test_lookup_service.py @@ -114,11 +114,11 @@ class TestLookupService(BaseTest): self.assertEqual("Genetics Savings & Clone, Inc.", results[0].label) results = LookupService.lookup(workflow, "AllTheNames", "reaction design", limit=10) - self.assertEqual(5, len(results), "all results come back for two terms.") + self.assertEqual(3, len(results), "all results come back for two terms.") self.assertEqual("Reaction Design", results[0].label, "Exact matches come first.") results = LookupService.lookup(workflow, "AllTheNames", "1 Something", limit=10) - self.assertEqual("1 Something", results[0].label, "Exact matches are prefered") + self.assertEqual("1 Something", results[0].label, "Exact matches are preferred") results = LookupService.lookup(workflow, "AllTheNames", "1 (!-Something", limit=10) self.assertEqual("1 Something", results[0].label, "special characters don't flake out") @@ -126,16 +126,12 @@ class TestLookupService(BaseTest): results = LookupService.lookup(workflow, "AllTheNames", "1 Something", limit=10) self.assertEqual("1 Something", results[0].label, "double spaces should not be an issue.") + results = LookupService.lookup(workflow, "AllTheNames", "in", limit=10) + self.assertEqual(10, len(results), "stop words are not removed.") + self.assertEqual("Genetics Savings & Clone, Inc.", results[0].label) + + results = LookupService.lookup(workflow, "AllTheNames", "other", limit=10) + self.assertEqual("Other", results[0].label, "Can't find the word 'other', which is an english stop word") -# 1018 10000 Something Industry -# 1019 1000 Something Industry -# 1020 1 Something Industry -# 1021 10 Something Industry -# 1022 10000 Something Industry - - # Fixme: Stop words are taken into account on the query side, and haven't found a fix yet. - #results = WorkflowService.run_lookup_query(lookup_table.id, "in", limit=10) - #self.assertEqual(7, len(results), "stop words are not removed.") - #self.assertEqual("Genetics Savings & Clone, Inc.", results[0].label) diff --git a/tests/workflow/test_workflow_processor_multi_instance.py b/tests/workflow/test_workflow_processor_multi_instance.py index 1473ed3a..fc074208 100644 --- a/tests/workflow/test_workflow_processor_multi_instance.py +++ b/tests/workflow/test_workflow_processor_multi_instance.py @@ -38,6 +38,27 @@ class TestWorkflowProcessorMultiInstance(BaseTest): workflow_model = StudyService._create_workflow_model(study_model, spec_model) return WorkflowProcessor(workflow_model) + + @patch('crc.services.study_service.StudyService.get_investigators') + def test_load_irb_from_db(self, mock_study_service): + # This depends on getting a list of investigators back from the protocol builder. + mock_study_service.return_value = self.mock_investigator_response + + self.load_example_data() + workflow_spec_model = self.create_workflow("irb_api_personnel") + study = session.query(StudyModel).first() + processor = WorkflowProcessor(workflow_spec_model) + processor.do_engine_steps() + task_list = processor.get_ready_user_tasks() + processor.complete_task(task_list[0]) + processor.do_engine_steps() + nav_list = processor.bpmn_workflow.get_nav_list() + processor.save() + # reload after save + processor = WorkflowProcessor(workflow_spec_model) + nav_list2 = processor.bpmn_workflow.get_nav_list() + self.assertEqual(nav_list,nav_list2) + @patch('crc.services.study_service.StudyService.get_investigators') def test_create_and_complete_workflow(self, mock_study_service): # This depends on getting a list of investigators back from the protocol builder. diff --git a/tests/workflow/test_workflow_service.py b/tests/workflow/test_workflow_service.py index 9ae49b5a..89606959 100644 --- a/tests/workflow/test_workflow_service.py +++ b/tests/workflow/test_workflow_service.py @@ -75,9 +75,9 @@ class TestWorkflowService(BaseTest): task = processor.next_task() WorkflowService.process_options(task, task.task_spec.form.fields[0]) options = task.task_spec.form.fields[0].options - self.assertEqual(28, len(options)) - self.assertEqual('1000', options[0]['id']) - self.assertEqual("UVA - INTERNAL - GM USE ONLY", options[0]['name']) + self.assertEqual(29, len(options)) + self.assertEqual('0', options[0]['id']) + self.assertEqual("Other", options[0]['name']) def test_random_data_populate_form_on_auto_complete(self): self.load_example_data()