diff --git a/Pipfile b/Pipfile index 0e5e21dd..6b28197a 100644 --- a/Pipfile +++ b/Pipfile @@ -39,6 +39,7 @@ requests = "*" sentry-sdk = {extras = ["flask"],version = "==0.14.4"} sphinx = "*" spiffworkflow = {editable = true,git = "https://github.com/sartography/SpiffWorkflow.git",ref = "master"} +#spiffworkflow = {editable = true,path="/home/kelly/sartography/SpiffWorkflow/"} swagger-ui-bundle = "*" webtest = "*" werkzeug = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 909cf764..ff3840b4 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -322,7 +322,6 @@ "sha256:05b31d2034dd3f2a685cbbae4cfc4ed906b2a733cff7964ada450fd5e462b84e", "sha256:bfc7150eaf809b1c283879302f04c42791136060c6eeb12c0c6674fb1291fae5" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.4.4" }, "future": { @@ -412,12 +411,8 @@ }, "ldap3": { "hashes": [ - "sha256:298769ab0232b3a3efa1e84881096c24526fe37911c83a11285f222fe4975efd", - "sha256:81df4ac8b6df10fb1f05b17c18d0cb8c4c344d5a03083c382824960ed959cf5b", - "sha256:53aaae5bf14f3827c69600ddf4d61b88f49c055bb93060e9702c5bafd206c744", - "sha256:4fd2db72d0412cc16ee86be01332095e86e361329c3579b314231eb2e56c7871", - "sha256:52ab557b3c4908db4a90bea16731aa714b1b54e039b54fd4c4b83994c6c48c0c", - "sha256:17f04298b70bf7ecaa5db8a7d8622b5a962ef7fc2b245b2eea705ac1c24338c0" + "sha256:17f04298b70bf7ecaa5db8a7d8622b5a962ef7fc2b245b2eea705ac1c24338c0", + "sha256:81df4ac8b6df10fb1f05b17c18d0cb8c4c344d5a03083c382824960ed959cf5b" ], "index": "pypi", "version": "==2.7" @@ -428,6 +423,7 @@ "sha256:08fc93257dcfe9542c0a6883a25ba4971d78297f63d7a5a26ffa34861ca78730", "sha256:107781b213cf7201ec3806555657ccda67b1fccc4261fb889ef7fc56976db81f", "sha256:121b665b04083a1e85ff1f5243d4a93aa1aaba281bc12ea334d5a187278ceaf1", + "sha256:1fa21263c3aba2b76fd7c45713d4428dbcc7644d73dcf0650e9d344e433741b3", "sha256:2b30aa2bcff8e958cd85d907d5109820b01ac511eae5b460803430a7404e34d7", "sha256:4b4a111bcf4b9c948e020fd207f915c24a6de3f1adc7682a2d92660eb4e84f1a", "sha256:5591c4164755778e29e69b86e425880f852464a21c7bb53c7ea453bbe2633bbe", @@ -438,6 +434,7 @@ "sha256:786aad2aa20de3dbff21aab86b2fb6a7be68064cbbc0219bde414d3a30aa47ae", "sha256:7ad7906e098ccd30d8f7068030a0b16668ab8aa5cda6fcd5146d8d20cbaa71b5", "sha256:80a38b188d20c0524fe8959c8ce770a8fdf0e617c6912d23fc97c68301bb9aba", + "sha256:8f0ec6b9b3832e0bd1d57af41f9238ea7709bbd7271f639024f2fc9d3bb01293", "sha256:92282c83547a9add85ad658143c76a64a8d339028926d7dc1998ca029c88ea6a", "sha256:94150231f1e90c9595ccc80d7d2006c61f90a5995db82bccbca7944fd457f0f6", "sha256:9dc9006dcc47e00a8a6a029eb035c8f696ad38e40a27d073a003d7d1443f5d88", @@ -445,8 +442,10 @@ "sha256:aa8eba3db3d8761db161003e2d0586608092e217151d7458206e243be5a43843", "sha256:bea760a63ce9bba566c23f726d72b3c0250e2fa2569909e2d83cda1534c79443", "sha256:c3f511a3c58676147c277eff0224c061dd5a6a8e1373572ac817ac6324f1b1e0", + "sha256:c9d317efde4bafbc1561509bfa8a23c5cab66c44d49ab5b63ff690f5159b2304", "sha256:cc411ad324a4486b142c41d9b2b6a722c534096963688d879ea6fa8a35028258", "sha256:cdc13a1682b2a6241080745b1953719e7fe0850b40a5c71ca574f090a1391df6", + "sha256:cfd7c5dd3c35c19cec59c63df9571c67c6d6e5c92e0fe63517920e97f61106d1", "sha256:e1cacf4796b20865789083252186ce9dc6cc59eca0c2e79cca332bdff24ac481", "sha256:e70d4e467e243455492f5de463b72151cc400710ac03a0678206a5f27e79ddef", "sha256:ecc930ae559ea8a43377e8b60ca6f8d61ac532fc57efb915d899de4a67928efd", @@ -510,11 +509,11 @@ }, "marshmallow": { "hashes": [ - "sha256:0f3a630f6a2fd124929f1bdcb5df65bd14cc8f49f52a18d0bdcfa0c42414e4a7", - "sha256:ba949379cb6ef73655f72075e82b31cf57012a5557ede642fc8614ab0354f869" + "sha256:67bf4cae9d3275b3fc74bd7ff88a7c98ee8c57c94b251a67b031dc293ecc4b76", + "sha256:a2a5eefb4b75a3b43f05be1cca0b6686adf56af7465c3ca629e5ad8d1e1fe13d" ], "index": "pypi", - "version": "==3.7.0" + "version": "==3.7.1" }, "marshmallow-enum": { "hashes": [ @@ -646,19 +645,8 @@ }, "pyasn1": { "hashes": [ - "sha256:08c3c53b75eaa48d71cf8c710312316392ed40899cb34710d092e96745a358b7", - "sha256:03840c999ba71680a131cfaee6fab142e1ed9bbd9c693e285cc6aca0d555e576", - "sha256:6e7545f1a61025a4e58bb336952c5061697da694db1cae97b116e9c46abcf7c8", - "sha256:78fa6da68ed2727915c4767bb386ab32cdba863caa7dbe473eaae45f9959da86", - "sha256:aef77c9fb94a3ac588e87841208bdec464471d9871bd5050a287cc9a475cd0ba", - "sha256:5c9414dcfede6e441f7e8f81b43b34e834731003427e5b09e4e00e3172a10f00", - "sha256:e89bf84b5437b532b0803ba5c9a5e054d21fec423a89952a74f87fa2c9b7bce2", - "sha256:0458773cfe65b153891ac249bcf1b5f8f320b7c2ce462151f8fa74de8934becf", - "sha256:014c0e9976956a08139dc0712ae195324a75e142284d5f87f1a87ee1b068a359", "sha256:39c7e2ec30515947ff4e87fb6f456dfc6e84857d34be479c9d4a4ba4bf46aa5d", - "sha256:fec3e9d8e36808a28efb59b489e4528c10ad0f480e57dcc32b4de5c9d8c9fdf3", - "sha256:99fcc3c8d804d1bc6d9a099921e39d827026409a58f2a720dcdb89374ea0c776", - "sha256:7ab8a544af125fb704feadb008c99a88805126fb525280b2270bb25cc1d78a12" + "sha256:aef77c9fb94a3ac588e87841208bdec464471d9871bd5050a287cc9a475cd0ba" ], "version": "==0.4.8" }, @@ -713,11 +701,9 @@ }, "python-editor": { "hashes": [ - "sha256:5f98b069316ea1c2ed3f67e7f5df6c0d8f10b689964a4a811ff64f0106819ec8", + "sha256:1bf6e860a8ad52a14c3ee1252d5dc25b2030618ed80c022598f00176adc8367d", "sha256:51fda6bcc5ddbbb7063b2af7509e43bd84bfc32a4ff71349ec7847713882327b", - "sha256:c3da2053dbab6b29c94e43c486ff67206eafbe7eb52dbec7390b5e2fb05aac77", - "sha256:ea87e17f6ec459e780e4221f295411462e0d0810858e055fc514684350a2f522", - "sha256:1bf6e860a8ad52a14c3ee1252d5dc25b2030618ed80c022598f00176adc8367d" + "sha256:5f98b069316ea1c2ed3f67e7f5df6c0d8f10b689964a4a811ff64f0106819ec8" ], "version": "==1.0.4" }, @@ -900,7 +886,7 @@ "spiffworkflow": { "editable": true, "git": "https://github.com/sartography/SpiffWorkflow.git", - "ref": "c72ced41e323aa69fcb6f7708e1869e98add716d" + "ref": "161b3e2ac62a824c6b771e9b817a2bd477af0d17" }, "sqlalchemy": { "hashes": [ @@ -937,12 +923,11 @@ }, "swagger-ui-bundle": { "hashes": [ - "sha256:49d2e12d60a6499e9d37ea37953b5d700f4e114edc7520fe918bae5eb693a20e", - "sha256:c5373b683487b1b914dccd23bcd9a3016afa2c2d1cda10f8713c0a9af0f91dd3", - "sha256:f776811855092c086dbb08216c8810a84accef8c76c796a135caa13645c5cc68" + "sha256:f5255f786cde67a2638111f4a7d04355836743198a83c4ecbe815d9fc384b0c8", + "sha256:f5691167f2e9f73ecbe8229a89454ae5ea958f90bb0d4583ed7adaae598c4122" ], "index": "pypi", - "version": "==0.0.6" + "version": "==0.0.8" }, "urllib3": { "hashes": [ diff --git a/config/default.py b/config/default.py index 5c8f8c51..b295bf4b 100644 --- a/config/default.py +++ b/config/default.py @@ -15,7 +15,8 @@ TEST_UID = environ.get('TEST_UID', default="dhf8r") ADMIN_UIDS = re.split(r',\s*', environ.get('ADMIN_UIDS', default="dhf8r,ajl2j,cah3us,cl3wf")) # Sentry flag -ENABLE_SENTRY = environ.get('ENABLE_SENTRY', default="false") == "true" +ENABLE_SENTRY = environ.get('ENABLE_SENTRY', default="false") == "true" # To be removed soon +SENTRY_ENVIRONMENT = environ.get('SENTRY_ENVIRONMENT', None) # Add trailing slash to base path APPLICATION_ROOT = re.sub(r'//', '/', '/%s/' % environ.get('APPLICATION_ROOT', default="/").strip('/')) diff --git a/crc/__init__.py b/crc/__init__.py index d56085d0..9081f739 100644 --- a/crc/__init__.py +++ b/crc/__init__.py @@ -52,8 +52,9 @@ origins_re = [r"^https?:\/\/%s(.*)" % o.replace('.', '\.') for o in app.config[' cors = CORS(connexion_app.app, origins=origins_re) # Sentry error handling -if app.config['ENABLE_SENTRY']: +if app.config['SENTRY_ENVIRONMENT']: sentry_sdk.init( + environment=app.config['SENTRY_ENVIRONMENT'], dsn="https://25342ca4e2d443c6a5c49707d68e9f40@o401361.ingest.sentry.io/5260915", integrations=[FlaskIntegration()] ) @@ -91,9 +92,3 @@ def clear_db(): from example_data import ExampleDataLoader ExampleDataLoader.clean_db() -@app.cli.command() -def rrt_data_fix(): - """Finds all the empty task event logs, and populates - them with good wholesome data.""" - from crc.services.workflow_service import WorkflowService - WorkflowService.fix_legacy_data_model_for_rrt() diff --git a/crc/api/common.py b/crc/api/common.py index f8673a5b..cb527c73 100644 --- a/crc/api/common.py +++ b/crc/api/common.py @@ -25,6 +25,7 @@ class ApiError(Exception): instance.task_name = task.task_spec.description or "" instance.file_name = task.workflow.spec.file or "" instance.task_data = task.data + app.logger.error(message, exc_info=True) return instance @classmethod @@ -35,6 +36,7 @@ class ApiError(Exception): instance.task_name = task_spec.description or "" if task_spec._wf_spec: instance.file_name = task_spec._wf_spec.file + app.logger.error(message, exc_info=True) return instance @classmethod diff --git a/crc/models/study.py b/crc/models/study.py index 47d4eb8f..7bb2db33 100644 --- a/crc/models/study.py +++ b/crc/models/study.py @@ -40,7 +40,7 @@ class StudyModel(db.Model): class WorkflowMetadata(object): - def __init__(self, id, name, display_name, description, spec_version, category_id, state: WorkflowState, status: WorkflowStatus, + def __init__(self, id, name, display_name, description, spec_version, category_id, category_display_name, state: WorkflowState, status: WorkflowStatus, total_tasks, completed_tasks, display_order): self.id = id self.name = name @@ -48,6 +48,7 @@ class WorkflowMetadata(object): self.description = description self.spec_version = spec_version self.category_id = category_id + self.category_display_name = category_display_name self.state = state self.status = status self.total_tasks = total_tasks @@ -64,6 +65,7 @@ class WorkflowMetadata(object): description=workflow.workflow_spec.description, spec_version=workflow.spec_version(), category_id=workflow.workflow_spec.category_id, + category_display_name=workflow.workflow_spec.category.display_name, state=WorkflowState.optional, status=workflow.status, total_tasks=workflow.total_tasks, @@ -79,7 +81,8 @@ class WorkflowMetadataSchema(ma.Schema): class Meta: model = WorkflowMetadata additional = ["id", "name", "display_name", "description", - "total_tasks", "completed_tasks", "display_order"] + "total_tasks", "completed_tasks", "display_order", + "category_id", "category_display_name"] unknown = INCLUDE diff --git a/crc/models/task_event.py b/crc/models/task_event.py index a6cb1a2d..e3914468 100644 --- a/crc/models/task_event.py +++ b/crc/models/task_event.py @@ -56,7 +56,6 @@ class TaskEventSchema(ma.Schema): study = fields.Nested(StudySchema, dump_only=True) workflow = fields.Nested(WorkflowMetadataSchema, dump_only=True) - class Meta: model = TaskEvent additional = ["id", "user_uid", "action", "task_id", "task_title", diff --git a/crc/services/workflow_processor.py b/crc/services/workflow_processor.py index 60040a95..165d3313 100644 --- a/crc/services/workflow_processor.py +++ b/crc/services/workflow_processor.py @@ -17,12 +17,13 @@ from SpiffWorkflow.dmn.parser.BpmnDmnParser import BpmnDmnParser from SpiffWorkflow.exceptions import WorkflowTaskExecException from SpiffWorkflow.specs import WorkflowSpec -from crc import session +from crc import session, app from crc.api.common import ApiError from crc.models.file import FileDataModel, FileModel, FileType from crc.models.workflow import WorkflowStatus, WorkflowModel, WorkflowSpecDependencyFile from crc.scripts.script import Script from crc.services.file_service import FileService +from crc import app class CustomBpmnScriptEngine(BpmnScriptEngine): @@ -30,17 +31,29 @@ class CustomBpmnScriptEngine(BpmnScriptEngine): Rather than execute arbitrary code, this assumes the script references a fully qualified python class such as myapp.RandomFact. """ - def execute(self, task: SpiffTask, script, **kwargs): + def execute(self, task: SpiffTask, script, data): """ - Assume that the script read in from the BPMN file is a fully qualified python class. Instantiate - that class, pass in any data available to the current task so that it might act on it. - Assume that the class implements the "do_task" method. - - This allows us to reference custom code from the BPMN diagram. + Functions in two modes. + 1. If the command is proceeded by #! then this is assumed to be a python script, and will + attempt to load that python module and execute the do_task method on that script. Scripts + must be located in the scripts package and they must extend the script.py class. + 2. If not proceeded by the #! this will attempt to execute the script directly and assumes it is + valid Python. """ # Shlex splits the whole string while respecting double quoted strings within + if not script.startswith('#!'): + try: + super().execute(task, script, data) + except SyntaxError as e: + raise ApiError.from_task('syntax_error', + f'If you are running a pre-defined script, please' + f' proceed the script with "#!", otherwise this is assumed to be' + f' pure python: {script}, {e.msg}', task=task) + else: + self.run_predefined_script(task, script[2:], data) # strip off the first two characters. + + def run_predefined_script(self, task: SpiffTask, script, data): commands = shlex.split(script) - printable_comms = commands path_and_command = commands[0].rsplit(".", 1) if len(path_and_command) == 1: module_name = "crc.scripts." + self.camel_to_snake(path_and_command[0]) @@ -59,10 +72,10 @@ class CustomBpmnScriptEngine(BpmnScriptEngine): if not isinstance(klass(), Script): raise ApiError.from_task("invalid_script", - "This is an internal error. The script '%s:%s' you called " % - (module_name, class_name) + - "does not properly implement the CRC Script class.", - task=task) + "This is an internal error. The script '%s:%s' you called " % + (module_name, class_name) + + "does not properly implement the CRC Script class.", + task=task) if task.workflow.data[WorkflowProcessor.VALIDATION_PROCESS_KEY]: """If this is running a validation, and not a normal process, then we want to mimic running the script, but not make any external calls or database changes.""" @@ -71,8 +84,8 @@ class CustomBpmnScriptEngine(BpmnScriptEngine): klass().do_task(task, study_id, workflow_id, *commands[1:]) except ModuleNotFoundError: raise ApiError.from_task("invalid_script", - "Unable to locate Script: '%s:%s'" % (module_name, class_name), - task=task) + "Unable to locate Script: '%s:%s'" % (module_name, class_name), + task=task) def evaluate_expression(self, task, expression): """ diff --git a/crc/services/workflow_service.py b/crc/services/workflow_service.py index c481a0a8..3205e800 100644 --- a/crc/services/workflow_service.py +++ b/crc/services/workflow_service.py @@ -376,7 +376,7 @@ class WorkflowService(object): try: task.title = spiff_task.workflow.script_engine.evaluate_expression(spiff_task, task.properties['display_name']) except Exception as e: - app.logger.info("Failed to set title on task due to type error." + str(e)) + app.logger.error("Failed to set title on task due to type error." + str(e), exc_info=True) elif task.title and ' ' in task.title: task.title = task.title.partition(' ')[2] return task @@ -516,41 +516,6 @@ class WorkflowService(object): db.session.add(task_event) db.session.commit() - @staticmethod - def fix_legacy_data_model_for_rrt(): - """ Remove this after use! This is just to fix RRT so the data is handled correctly. - - Utility that is likely called via the flask command line, it will loop through all the - workflows in the system and attempt to add the right data into the task action log so that - users do not have to re fill out all of the forms if they start over or go back in the workflow. - Viciously inefficient, but should only have to run one time for RRT""" - workflows = db.session.query(WorkflowModel).all() - for workflow_model in workflows: - task_logs = db.session.query(TaskEventModel) \ - .filter(TaskEventModel.workflow_id == workflow_model.id) \ - .filter(TaskEventModel.action == WorkflowService.TASK_ACTION_COMPLETE) \ - .order_by(TaskEventModel.date.desc()).all() - - processor = WorkflowProcessor(workflow_model) - # Grab all the data from last task completed, which will be everything in this - # rrt situation because of how we were keeping all the data at the time. - latest_data = processor.next_task().data - - # Move forward in the task spec tree, dropping any data that would have been - # added in subsequent tasks, just looking at form data, will not track the automated - # task data additions, hopefully this doesn't hang us. - for log in task_logs: -# if log.task_data is not None: # Only do this if the task event does not have data populated in it. -# continue - data = copy.deepcopy(latest_data) # Or you end up with insane crazy issues. - # In the simple case of RRT, there is exactly one task for the given task_spec - task = processor.bpmn_workflow.get_tasks_from_spec_name(log.task_name)[0] - data = WorkflowService.extract_form_data(data, task) - log.form_data = data - db.session.add(log) - - db.session.commit() - @staticmethod def extract_form_data(latest_data, task): """Removes data from latest_data that would be added by the child task or any of its children.""" diff --git a/crc/static/bpmn/core_info/core_info.bpmn b/crc/static/bpmn/core_info/core_info.bpmn index 8e790f98..8c69ffb3 100644 --- a/crc/static/bpmn/core_info/core_info.bpmn +++ b/crc/static/bpmn/core_info/core_info.bpmn @@ -1,5 +1,5 @@ - + Flow_1wqp7vf @@ -212,7 +212,7 @@ SequenceFlow_1r3yrhy Flow_09h1imz - StudyInfo details + #! StudyInfo details Flow_09h1imz diff --git a/crc/static/bpmn/data_security_plan/data_security_plan.bpmn b/crc/static/bpmn/data_security_plan/data_security_plan.bpmn index 0bf95e18..86426d6d 100644 --- a/crc/static/bpmn/data_security_plan/data_security_plan.bpmn +++ b/crc/static/bpmn/data_security_plan/data_security_plan.bpmn @@ -1,5 +1,5 @@ - + SequenceFlow_100w7co @@ -453,7 +453,7 @@ Indicate all the possible formats in which you will transmit your data outside o SequenceFlow_0k2r83n SequenceFlow_0t6xl9i SequenceFlow_16kyite - CompleteTemplate NEW_DSP_template.docx Study_DataSecurityPlan + #! CompleteTemplate NEW_DSP_template.docx Study_DataSecurityPlan ##### Instructions diff --git a/crc/static/bpmn/documents_approvals/documents_approvals.bpmn b/crc/static/bpmn/documents_approvals/documents_approvals.bpmn index bf39615b..12e85e34 100644 --- a/crc/static/bpmn/documents_approvals/documents_approvals.bpmn +++ b/crc/static/bpmn/documents_approvals/documents_approvals.bpmn @@ -41,8 +41,7 @@ {%- else -%} | {{doc.display_name}} | Not started | [?](/help/documents/{{doc.code}}) | No file yet | {%- endif %} -{% endif %}{% endfor %} - +{% endif %}{% endfor %} @@ -54,12 +53,12 @@ Flow_0c7ryff Flow_142jtxs - StudyInfo approvals + #! StudyInfo approvals Flow_1k3su2q Flow_0c7ryff - StudyInfo documents + #! StudyInfo documents diff --git a/crc/static/bpmn/ide_supplement/ide_supplement.bpmn b/crc/static/bpmn/ide_supplement/ide_supplement.bpmn index a886b4d4..7a83643b 100644 --- a/crc/static/bpmn/ide_supplement/ide_supplement.bpmn +++ b/crc/static/bpmn/ide_supplement/ide_supplement.bpmn @@ -1,5 +1,5 @@ - + SequenceFlow_1dhb8f4 @@ -36,7 +36,7 @@ SequenceFlow_1dhb8f4 SequenceFlow_1uzcl1f - StudyInfo details + #! StudyInfo details diff --git a/crc/static/bpmn/ids_full_submission/ids_full_submission.bpmn b/crc/static/bpmn/ids_full_submission/ids_full_submission.bpmn index 72fece25..25a9ad6e 100644 --- a/crc/static/bpmn/ids_full_submission/ids_full_submission.bpmn +++ b/crc/static/bpmn/ids_full_submission/ids_full_submission.bpmn @@ -1,5 +1,5 @@ - + SequenceFlow_1dexemq @@ -217,7 +217,7 @@ Protocol Owner: **(need to insert value here)** SequenceFlow_1dexemq Flow_1x9d2mo - StudyInfo documents + #! StudyInfo documents diff --git a/crc/static/bpmn/ind_update/ind_update.bpmn b/crc/static/bpmn/ind_update/ind_update.bpmn index 15845111..528a87ce 100644 --- a/crc/static/bpmn/ind_update/ind_update.bpmn +++ b/crc/static/bpmn/ind_update/ind_update.bpmn @@ -12,7 +12,7 @@ SequenceFlow_1dhb8f4 SequenceFlow_1uzcl1f - StudyInfo details + #! StudyInfo details diff --git a/crc/static/bpmn/irb_api_details/irb_api_details.bpmn b/crc/static/bpmn/irb_api_details/irb_api_details.bpmn index b5f0da02..b4f540f5 100644 --- a/crc/static/bpmn/irb_api_details/irb_api_details.bpmn +++ b/crc/static/bpmn/irb_api_details/irb_api_details.bpmn @@ -1,5 +1,5 @@ - + @@ -8,7 +8,7 @@ SequenceFlow_1fmyo77 SequenceFlow_18nr0gf - StudyInfo details + #! StudyInfo details diff --git a/crc/static/bpmn/irb_api_personnel/irb_api_personnel.bpmn b/crc/static/bpmn/irb_api_personnel/irb_api_personnel.bpmn index 99edb961..a5258cbe 100644 --- a/crc/static/bpmn/irb_api_personnel/irb_api_personnel.bpmn +++ b/crc/static/bpmn/irb_api_personnel/irb_api_personnel.bpmn @@ -1,5 +1,5 @@ - + Flow_0kcrx5l @@ -7,7 +7,7 @@ Flow_0kcrx5l Flow_1dcsioh - StudyInfo investigators + #! StudyInfo investigators ## The following information was gathered: @@ -54,28 +54,28 @@ - - + + - - + + - - + + - + - + - + - + diff --git a/crc/static/bpmn/research_rampup/research_rampup.bpmn b/crc/static/bpmn/research_rampup/research_rampup.bpmn index 19588731..4a04eb6d 100644 --- a/crc/static/bpmn/research_rampup/research_rampup.bpmn +++ b/crc/static/bpmn/research_rampup/research_rampup.bpmn @@ -1,5 +1,5 @@ - + SequenceFlow_05ja25w @@ -598,7 +598,7 @@ Use the EHS [Lab Safety Plan During COVID 19 template](https://www.google.com/ur This step is internal to the system and do not require and user interaction Flow_11uqavk Flow_0aqgwvu - CompleteTemplate ResearchRampUpPlan.docx RESEARCH_RAMPUP + #! CompleteTemplate ResearchRampUpPlan.docx RESEARCH_RAMPUP @@ -755,7 +755,7 @@ Notify the Area Monitor for This step is internal to the system and do not require and user interaction Flow_0j4rs82 Flow_07ge8uf - RequestApproval ApprvlApprvr1 ApprvlApprvr2 + #!RequestApproval ApprvlApprvr1 ApprvlApprvr2 #### Script Task @@ -764,7 +764,7 @@ This step is internal to the system and do not require and user interaction Flow_16y8glw Flow_0uc4o6c - UpdateStudy title:PIComputingID.label pi:PIComputingID.value + #! UpdateStudy title:PIComputingID.label pi:PIComputingID.value #### Weekly Personnel Schedule(s) diff --git a/crc/static/bpmn/top_level_workflow/top_level_workflow.bpmn b/crc/static/bpmn/top_level_workflow/top_level_workflow.bpmn index 6806fa5b..23d6ff72 100644 --- a/crc/static/bpmn/top_level_workflow/top_level_workflow.bpmn +++ b/crc/static/bpmn/top_level_workflow/top_level_workflow.bpmn @@ -11,7 +11,7 @@ SequenceFlow_1ees8ka SequenceFlow_17ct47v - StudyInfo documents + #! StudyInfo documents Flow_1m8285h @@ -62,7 +62,7 @@ Flow_0pwtiqm Flow_0eq6px2 - StudyInfo details + #! StudyInfo details Flow_14ce1d7 @@ -91,7 +91,7 @@ Flow_1qyrmzn Flow_0vo6ul1 - StudyInfo investigators + #! StudyInfo investigators diff --git a/docker_run.sh b/docker_run.sh index 8ad66274..ec80bb99 100755 --- a/docker_run.sh +++ b/docker_run.sh @@ -23,12 +23,6 @@ if [ "$RESET_DB_RRT" = "true" ]; then pipenv run flask load-example-rrt-data fi -if [ "$FIX_RRT_DATA" = "true" ]; then - echo 'Fixing RRT data...' - pipenv run flask rrt-data-fix -fi - - # THIS MUST BE THE LAST COMMAND! if [ "$APPLICATION_ROOT" = "/" ]; then pipenv run gunicorn --bind 0.0.0.0:$PORT0 wsgi:app diff --git a/example_data.py b/example_data.py index efdfe3b3..8b9b0c27 100644 --- a/example_data.py +++ b/example_data.py @@ -251,7 +251,6 @@ class ExampleDataLoader: master_spec=False, from_tests=True) - def create_spec(self, id, name, display_name="", description="", filepath=None, master_spec=False, category_id=None, display_order=None, from_tests=False): """Assumes that a directory exists in static/bpmn with the same name as the given id. diff --git a/tests/base_test.py b/tests/base_test.py index 6ea1966d..3f0b2405 100644 --- a/tests/base_test.py +++ b/tests/base_test.py @@ -19,7 +19,7 @@ from crc.models.protocol_builder import ProtocolBuilderStatus from crc.models.task_event import TaskEventModel from crc.models.study import StudyModel from crc.models.user import UserModel -from crc.models.workflow import WorkflowSpecModel, WorkflowSpecModelSchema, WorkflowModel +from crc.models.workflow import WorkflowSpecModel, WorkflowSpecModelSchema, WorkflowModel, WorkflowSpecCategoryModel from crc.services.file_service import FileService from crc.services.study_service import StudyService from crc.services.workflow_service import WorkflowService @@ -164,14 +164,21 @@ class BaseTest(unittest.TestCase): self.assertGreater(len(file_data), 0) @staticmethod - def load_test_spec(dir_name, master_spec=False, category_id=None): + def load_test_spec(dir_name, display_name=None, master_spec=False, category_id=None): """Loads a spec into the database based on a directory in /tests/data""" + if category_id is None: + category = WorkflowSpecCategoryModel(name="test", display_name="Test Workflows", display_order=0) + db.session.add(category) + db.session.commit() + category_id = category.id if session.query(WorkflowSpecModel).filter_by(id=dir_name).count() > 0: return session.query(WorkflowSpecModel).filter_by(id=dir_name).first() filepath = os.path.join(app.root_path, '..', 'tests', 'data', dir_name, "*") + if display_name is None: + display_name = dir_name return ExampleDataLoader().create_spec(id=dir_name, name=dir_name, filepath=filepath, master_spec=master_spec, - category_id=category_id) + display_name=display_name, category_id=category_id) @staticmethod def protocol_builder_response(file_name): @@ -263,11 +270,13 @@ class BaseTest(unittest.TestCase): return full_study - def create_workflow(self, workflow_name, study=None, category_id=None, as_user="dhf8r"): + def create_workflow(self, workflow_name, display_name=None, study=None, category_id=None, as_user="dhf8r"): db.session.flush() spec = db.session.query(WorkflowSpecModel).filter(WorkflowSpecModel.name == workflow_name).first() if spec is None: - spec = self.load_test_spec(workflow_name, category_id=category_id) + if display_name is None: + display_name = workflow_name + spec = self.load_test_spec(workflow_name, display_name, category_id=category_id) if study is None: study = self.create_study(uid=as_user) workflow_model = StudyService._create_workflow_model(study, spec) diff --git a/tests/data/docx/docx.bpmn b/tests/data/docx/docx.bpmn index a95feb07..8c741114 100644 --- a/tests/data/docx/docx.bpmn +++ b/tests/data/docx/docx.bpmn @@ -1,5 +1,5 @@ - + SequenceFlow_0637d8i @@ -27,7 +27,7 @@ SequenceFlow_1i7hk1a SequenceFlow_11c35oq - CompleteTemplate Letter.docx AD_CoCApp + #! CompleteTemplate Letter.docx AD_CoCApp SequenceFlow_11c35oq diff --git a/tests/data/email/email.bpmn b/tests/data/email/email.bpmn index 1b8d5252..11ecec2e 100644 --- a/tests/data/email/email.bpmn +++ b/tests/data/email/email.bpmn @@ -1,5 +1,5 @@ - + Flow_1synsig @@ -20,7 +20,7 @@ Email content to be delivered to {{ ApprvlApprvr1 }} --- Flow_08n2npe Flow_1xlrgne - Email "Camunda Email Subject" ApprvlApprvr1 PIComputingID + #! Email "Camunda Email Subject" ApprvlApprvr1 PIComputingID diff --git a/tests/data/invalid_script/invalid_script.bpmn b/tests/data/invalid_script/invalid_script.bpmn index 80417e90..b85e2bc4 100644 --- a/tests/data/invalid_script/invalid_script.bpmn +++ b/tests/data/invalid_script/invalid_script.bpmn @@ -1,5 +1,5 @@ - + SequenceFlow_1pnq3kg @@ -11,7 +11,7 @@ SequenceFlow_1pnq3kg SequenceFlow_12pf6um - NoSuchScript withArg1 + #! NoSuchScript withArg1 diff --git a/tests/data/invalid_script2/invalid_script2.bpmn b/tests/data/invalid_script2/invalid_script2.bpmn new file mode 100644 index 00000000..b061e76c --- /dev/null +++ b/tests/data/invalid_script2/invalid_script2.bpmn @@ -0,0 +1,39 @@ + + + + + SequenceFlow_1pnq3kg + + + + SequenceFlow_12pf6um + + + SequenceFlow_1pnq3kg + SequenceFlow_12pf6um + a really bad error that should fail + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/data/multi_instance/multi_instance.bpmn b/tests/data/multi_instance/multi_instance.bpmn index 28bda546..c1e610a5 100644 --- a/tests/data/multi_instance/multi_instance.bpmn +++ b/tests/data/multi_instance/multi_instance.bpmn @@ -1,5 +1,5 @@ - + Flow_0t6p1sb @@ -29,7 +29,7 @@ Flow_0t6p1sb SequenceFlow_1p568pp - StudyInfo investigators + #! StudyInfo investigators diff --git a/tests/data/multi_instance_parallel/multi_instance_parallel.bpmn b/tests/data/multi_instance_parallel/multi_instance_parallel.bpmn index 9e53323f..d20c8499 100644 --- a/tests/data/multi_instance_parallel/multi_instance_parallel.bpmn +++ b/tests/data/multi_instance_parallel/multi_instance_parallel.bpmn @@ -1,5 +1,5 @@ - + Flow_0t6p1sb @@ -29,7 +29,7 @@ Flow_0t6p1sb SequenceFlow_1p568pp - StudyInfo investigators + #! StudyInfo investigators diff --git a/tests/data/random_fact/random_fact.bpmn b/tests/data/random_fact/random_fact.bpmn index fc5e41bb..d5ffcbed 100644 --- a/tests/data/random_fact/random_fact.bpmn +++ b/tests/data/random_fact/random_fact.bpmn @@ -1,5 +1,5 @@ - + SequenceFlow_0c7wlth @@ -132,7 +132,7 @@ Autoconverted link https://github.com/nodeca/pica (enable linkify to see) SequenceFlow_0641sh6 SequenceFlow_0t29gjo - FactService + #! FactService # Great Job! diff --git a/tests/data/study_details/study_details.bpmn b/tests/data/study_details/study_details.bpmn index b9aead94..2b46f935 100644 --- a/tests/data/study_details/study_details.bpmn +++ b/tests/data/study_details/study_details.bpmn @@ -1,5 +1,5 @@ - + SequenceFlow_1nfe5m9 @@ -8,7 +8,7 @@ SequenceFlow_1nfe5m9 SequenceFlow_1bqiin0 - StudyInfo info + #! StudyInfo info diff --git a/tests/data/top_level_workflow/top_level_workflow.bpmn b/tests/data/top_level_workflow/top_level_workflow.bpmn index cc6e1c57..8b1bb888 100644 --- a/tests/data/top_level_workflow/top_level_workflow.bpmn +++ b/tests/data/top_level_workflow/top_level_workflow.bpmn @@ -1,5 +1,5 @@ - + SequenceFlow_1ees8ka @@ -11,7 +11,7 @@ SequenceFlow_1ees8ka SequenceFlow_17ct47v - StudyInfo documents + #! StudyInfo documents Flow_1m8285h diff --git a/tests/study/test_study_service.py b/tests/study/test_study_service.py index b436835f..f1e43c8a 100644 --- a/tests/study/test_study_service.py +++ b/tests/study/test_study_service.py @@ -27,7 +27,10 @@ class TestStudyService(BaseTest): # Assure some basic models are in place, This is a damn mess. Our database models need an overhaul to make # this easier - better relationship modeling is now critical. - self.load_test_spec("top_level_workflow", master_spec=True) + cat = WorkflowSpecCategoryModel(name="approvals", display_name="Approvals", display_order=0) + db.session.add(cat) + db.session.commit() + self.load_test_spec("top_level_workflow", master_spec=True, category_id=cat.id) user = db.session.query(UserModel).filter(UserModel.uid == "dhf8r").first() if not user: user = UserModel(uid="dhf8r", email_address="whatever@stuff.com", display_name="Stayathome Smellalots") @@ -39,11 +42,7 @@ class TestStudyService(BaseTest): study = StudyModel(title="My title", protocol_builder_status=ProtocolBuilderStatus.ACTIVE, user_uid=user.uid) db.session.add(study) - cat = WorkflowSpecCategoryModel(name="approvals", display_name="Approvals", display_order=0) - db.session.add(cat) - db.session.commit() - self.assertIsNotNone(cat.id) self.load_test_spec("random_fact", category_id=cat.id) self.assertIsNotNone(study.id) diff --git a/tests/test_user_roles.py b/tests/test_user_roles.py index 6104641c..cc7ff613 100644 --- a/tests/test_user_roles.py +++ b/tests/test_user_roles.py @@ -68,7 +68,7 @@ class TestTasksApi(BaseTest): def test_get_outstanding_tasks_awaiting_current_user(self): submitter = self.create_user(uid='lje5u') supervisor = self.create_user(uid='lb3dp') - workflow = self.create_workflow('roles', as_user=submitter.uid) + workflow = self.create_workflow('roles', display_name="Roles", as_user=submitter.uid) workflow_api = self.get_workflow_api(workflow, user_uid=submitter.uid) # User lje5u can complete the first task, and set her supervisor @@ -94,6 +94,7 @@ class TestTasksApi(BaseTest): self.assertEquals(1, len(tasks)) self.assertEquals(workflow.id, tasks[0]['workflow']['id']) self.assertEquals(workflow.study.id, tasks[0]['study']['id']) + self.assertEquals("Test Workflows", tasks[0]['workflow']['category_display_name']) # Assure we can say something sensible like: # You have a task called "Approval" to be completed in the "Supervisor Approval" workflow diff --git a/tests/workflow/test_workflow_service.py b/tests/workflow/test_workflow_service.py index f208eecb..9ae49b5a 100644 --- a/tests/workflow/test_workflow_service.py +++ b/tests/workflow/test_workflow_service.py @@ -89,51 +89,7 @@ class TestWorkflowService(BaseTest): WorkflowService.populate_form_with_random_data(task, task_api, required_only=False) self.assertTrue(isinstance(task.data["sponsor"], dict)) - @unittest.skip("RRT no longer needs to be supported") - def test_fix_legacy_data_model_for_rrt(self): - ExampleDataLoader().load_rrt() # Make sure the research_rampup is loaded, as it's not a test spec. - workflow = self.create_workflow('research_rampup') - processor = WorkflowProcessor(workflow, validate_only=True) - - # Use the test spec code to complete the workflow of research rampup. - while not processor.bpmn_workflow.is_completed(): - processor.bpmn_workflow.do_engine_steps() - tasks = processor.bpmn_workflow.get_tasks(SpiffTask.READY) - for task in tasks: - task_api = WorkflowService.spiff_task_to_api_task(task, add_docs_and_forms=True) - WorkflowService.populate_form_with_random_data(task, task_api, False) - task.complete() - # create the task events - WorkflowService.log_task_action('dhf8r', processor, task, - WorkflowService.TASK_ACTION_COMPLETE) - processor.save() - db.session.commit() - - WorkflowService.fix_legacy_data_model_for_rrt() - - # All tasks should now have data associated with them. - task_logs = db.session.query(TaskEventModel) \ - .filter(TaskEventModel.workflow_id == workflow.id) \ - .filter(TaskEventModel.action == WorkflowService.TASK_ACTION_COMPLETE) \ - .order_by(TaskEventModel.date).all() # Get them back in order. - - self.assertEqual(17, len(task_logs)) - for log in task_logs: - task = processor.bpmn_workflow.get_tasks_from_spec_name(log.task_name)[0] - self.assertIsNotNone(log.form_data) - # Each task should have the data in the form for that task in the task event. - if hasattr(task.task_spec, 'form'): - for field in task.task_spec.form.fields: - if field.has_property(Task.PROP_OPTIONS_REPEAT): - self.assertIn(field.get_property(Task.PROP_OPTIONS_REPEAT), log.form_data) - else: - self.assertIn(field.id, log.form_data) - - # Some spot checks: - # The first task should be empty, with all the data removed. - self.assertEqual({}, task_logs[0].form_data) - def test_dmn_evaluation_errors_in_oncomplete_raise_api_errors_during_validation(self): workflow_spec_model = self.load_test_spec("decision_table_invalid") with self.assertRaises(ApiError): - WorkflowService.test_spec(workflow_spec_model.id) \ No newline at end of file + WorkflowService.test_spec(workflow_spec_model.id) diff --git a/tests/workflow/test_workflow_spec_validation_api.py b/tests/workflow/test_workflow_spec_validation_api.py index 2a5b5455..0c17892e 100644 --- a/tests/workflow/test_workflow_spec_validation_api.py +++ b/tests/workflow/test_workflow_spec_validation_api.py @@ -52,10 +52,6 @@ class TestWorkflowSpecValidation(BaseTest): app.config['PB_ENABLED'] = True self.validate_all_loaded_workflows() - @unittest.skip("RRT no longer needs to be supported") - def test_successful_validation_of_rrt_workflows(self): - self.load_example_data(use_rrt_data=True) - self.validate_all_loaded_workflows() def validate_all_loaded_workflows(self): workflows = session.query(WorkflowSpecModel).all() @@ -68,7 +64,6 @@ class TestWorkflowSpecValidation(BaseTest): errors.extend(ApiErrorSchema(many=True).load(json_data)) self.assertEqual(0, len(errors), json.dumps(errors)) - def test_invalid_expression(self): self.load_example_data() errors = self.validate_workflow("invalid_expression") @@ -100,6 +95,15 @@ class TestWorkflowSpecValidation(BaseTest): self.assertEqual("An Invalid Script Reference", errors[0]['task_name']) self.assertEqual("invalid_script.bpmn", errors[0]['file_name']) + def test_invalid_script2(self): + self.load_example_data() + errors = self.validate_workflow("invalid_script2") + self.assertEqual(2, len(errors)) + self.assertEqual("error_loading_workflow", errors[0]['code']) + self.assertEqual("Invalid_Script_Task", errors[0]['task_id']) + self.assertEqual("An Invalid Script Reference", errors[0]['task_name']) + self.assertEqual("invalid_script2.bpmn", errors[0]['file_name']) + def test_repeating_sections_correctly_populated(self): self.load_example_data() spec_model = self.load_test_spec('repeat_form')