Merge branch 'dev' of https://github.com/sartography/cr-connect-workflow into partial_testing

This commit is contained in:
NWalker4483 2021-06-17 15:12:28 -04:00
commit 4ca94b39ce
7 changed files with 319 additions and 65 deletions

View File

@ -87,10 +87,7 @@ def evaluate_python_expression(body):
result = script_engine.eval(body['expression'], body['data'])
return {"result": result, "expression": body['expression'], "data": body['data']}
except Exception as e:
raise ApiError("expression_error", f"Failed to evaluate the expression '%s'. %s" %
(body['expression'], str(e)),
task_data = body["data"])
return {"result": False, "expression": body['expression'], "data": body['data'], "error": str(e)}
def send_test_email(subject, address, message, data=None):
rendered, wrapped = EmailService().get_rendered_content(message, data)

View File

@ -0,0 +1,41 @@
from crc import session
from crc.api.common import ApiError
from crc.models.workflow import WorkflowModel, WorkflowSpecModel
from crc.scripts.script import Script
from crc.services.workflow_processor import WorkflowProcessor
class ResetWorkflow(Script):
def get_description(self):
return """Reset a workflow. Run by master workflow.
Designed for completed workflows where we need to force rerunning the workflow.
I.e., a new PI"""
def do_task_validate_only(self, task, study_id, workflow_id, *args, **kwargs):
return hasattr(kwargs, 'workflow_name')
def do_task(self, task, study_id, workflow_id, *args, **kwargs):
if 'workflow_name' in kwargs.keys():
workflow_name = kwargs['workflow_name']
workflow_spec: WorkflowSpecModel = session.query(WorkflowSpecModel).filter_by(name=workflow_name).first()
if workflow_spec:
workflow_model: WorkflowModel = session.query(WorkflowModel).filter_by(
workflow_spec_id=workflow_spec.id,
study_id=study_id).first()
if workflow_model:
workflow_processor = WorkflowProcessor.reset(workflow_model, clear_data=False, delete_files=False)
return workflow_processor
else:
raise ApiError(code='missing_workflow_model',
message=f'No WorkflowModel returned. \
workflow_spec_id: {workflow_spec.id} \
study_id: {study_id}')
else:
raise ApiError(code='missing_workflow_spec',
message=f'No WorkflowSpecModel returned. \
name: {workflow_name}')
else:
raise ApiError(code='missing_workflow_name',
message='Reset workflow requires a workflow name')

View File

@ -7,6 +7,7 @@ from typing import List
import jinja2
from SpiffWorkflow import Task as SpiffTask, WorkflowException, NavItem
from SpiffWorkflow.bpmn.PythonScriptEngine import Box
from SpiffWorkflow.bpmn.specs.EndEvent import EndEvent
from SpiffWorkflow.bpmn.specs.ManualTask import ManualTask
from SpiffWorkflow.bpmn.specs.MultiInstanceTask import MultiInstanceTask
@ -81,11 +82,12 @@ class WorkflowService(object):
return workflow_model
@staticmethod
def delete_test_data():
def delete_test_data(workflow: WorkflowModel):
db.session.delete(workflow)
# Also, delete any test study or user models that may have been created.
for study in db.session.query(StudyModel).filter(StudyModel.user_uid == "test"):
StudyService.delete_study(study.id)
db.session.commit()
user = db.session.query(UserModel).filter_by(uid="test").first()
if user:
db.session.delete(user)
@ -103,60 +105,56 @@ class WorkflowService(object):
"""
workflow_model = WorkflowService.make_test_workflow(spec_id, validate_study_id)
try:
processor = WorkflowProcessor(workflow_model, validate_only=True)
except WorkflowException as we:
count = 0
while not processor.bpmn_workflow.is_completed():
if count < 100: # check for infinite loop
try:
processor.bpmn_workflow.get_deep_nav_list() # Assure no errors with navigation.
exit_task = processor.bpmn_workflow.do_engine_steps(exit_at=test_until)
if (exit_task != None):
WorkflowService.delete_test_data()
raise ApiError.from_task("validation_break",
f"The validation has been exited early on task '{exit_task.task_spec.name}' and was parented by ",
exit_task.parent)
tasks = processor.bpmn_workflow.get_tasks(SpiffTask.READY)
for task in tasks:
if task.task_spec.lane is not None and task.task_spec.lane not in task.data:
raise ApiError.from_task("invalid_role",
f"This task is in a lane called '{task.task_spec.lane}', The "
f" current task data must have information mapping this role to "
f" a unique user id.", task)
task_api = WorkflowService.spiff_task_to_api_task(
task,
add_docs_and_forms=True) # Assure we try to process the documentation, and raise those errors.
# make sure forms have a form key
if hasattr(task_api, 'form') and task_api.form is not None and task_api.form.key == '':
raise ApiError(code='missing_form_key',
message='Forms must include a Form Key.',
task_id=task.id,
task_name=task.get_name())
WorkflowService.populate_form_with_random_data(task, task_api, required_only)
processor.complete_task(task)
if test_until == task.task_spec.name:
WorkflowService.delete_test_data()
raise ApiError.from_task("validation_break",
f"The validation has been exited early on task '{task.task_spec.name}' and was parented by ",
task.parent)
count += 1
except WorkflowException as we:
WorkflowService.delete_test_data()
raise ApiError.from_workflow_exception("workflow_validation_exception", str(we), we)
else:
raise ApiError.from_task(code='validation_loop',
message=f'There appears to be an infinite loop in the validation. Task is {task.task_spec.description}',
task=task)
WorkflowService.delete_test_data()
raise ApiError.from_workflow_exception("workflow_validation_exception", str(we), we)
count = 0
escaped = False
while not processor.bpmn_workflow.is_completed() and not escaped:
if count < 100: # check for infinite loop
try:
processor.bpmn_workflow.get_deep_nav_list() # Assure no errors with navigation.
exit_task = processor.bpmn_workflow.do_engine_steps(exit_at=test_until)
if (exit_task != None):
WorkflowService.delete_test_data()
raise ApiError.from_task("validation_break",
f"The validation has been exited early on task '{exit_task.task_spec.name}' and was parented by ",
exit_task.parent)
tasks = processor.bpmn_workflow.get_tasks(SpiffTask.READY)
for task in tasks:
if task.task_spec.lane is not None and task.task_spec.lane not in task.data:
raise ApiError.from_task("invalid_role",
f"This task is in a lane called '{task.task_spec.lane}', The "
f" current task data must have information mapping this role to "
f" a unique user id.", task)
task_api = WorkflowService.spiff_task_to_api_task(
task,
add_docs_and_forms=True) # Assure we try to process the documentation, and raise those errors.
# make sure forms have a form key
if hasattr(task_api, 'form') and task_api.form is not None and task_api.form.key == '':
raise ApiError(code='missing_form_key',
message='Forms must include a Form Key.',
task_id=task.id,
task_name=task.get_name())
WorkflowService.populate_form_with_random_data(task, task_api, required_only)
processor.complete_task(task)
if test_until == task.task_spec.name:
WorkflowService.delete_test_data()
raise ApiError.from_task("validation_break",
f"The validation has been exited early on task '{task.task_spec.name}' and was parented by ",
task.parent)
count += 1
except WorkflowException as we:
WorkflowService.delete_test_data()
raise ApiError.from_workflow_exception("workflow_validation_exception", str(we), we)
else:
raise ApiError.from_task(code='validation_loop',
message=f'There appears to be an infinite loop in the validation. Task is {task.task_spec.description}',
task=task)
WorkflowService.delete_test_data()
WorkflowService._process_documentation(processor.bpmn_workflow.last_task.parent.parent)
WorkflowService._process_documentation(processor.bpmn_workflow.last_task.parent.parent)
finally:
WorkflowService.delete_test_data(workflow_model)
return processor.bpmn_workflow.last_task.data
@staticmethod
@ -305,8 +303,14 @@ class WorkflowService(object):
@staticmethod
def evaluate_property(property_name, field, task):
expression = field.get_property(property_name)
data = task.data
if field.has_property(Task.FIELD_PROP_REPEAT):
# Then you must evaluate the expression based on the data within the group only.
group = field.get_property(Task.FIELD_PROP_REPEAT)
if group in task.data:
data = task.data[group][0]
try:
return task.workflow.script_engine.evaluate_expression(task, expression)
return task.workflow.script_engine.eval(expression, data)
except Exception as e:
message = f"The field {field.id} contains an invalid expression. {e}"
raise ApiError.from_task(f'invalid_{property_name}', message, task=task)
@ -387,7 +391,7 @@ class WorkflowService(object):
if len(field.options) > 0:
random_choice = random.choice(field.options)
if isinstance(random_choice, dict):
return {'value': random_choice['id'], 'label': random_choice['name']}
return {'value': random_choice['id'], 'label': random_choice['name'], 'data': random_choice['data']}
else:
# fixme: why it is sometimes an EnumFormFieldOption, and other times not?
return {'value': random_choice.id, 'label': random_choice.name}
@ -708,7 +712,7 @@ class WorkflowService(object):
raise ApiError.from_task("invalid_enum", f"The label column '{label_column}' does not exist for item {item}",
task=spiff_task)
options.append({"id": item[value_column], "name": item[label_column], "data": item})
options.append(Box({"id": item[value_column], "name": item[label_column], "data": item}))
return options
@staticmethod

View File

@ -0,0 +1,82 @@
<?xml version="1.0" encoding="UTF-8"?>
<bpmn:definitions xmlns:bpmn="http://www.omg.org/spec/BPMN/20100524/MODEL" xmlns:bpmndi="http://www.omg.org/spec/BPMN/20100524/DI" xmlns:dc="http://www.omg.org/spec/DD/20100524/DC" xmlns:camunda="http://camunda.org/schema/1.0/bpmn" xmlns:di="http://www.omg.org/spec/DD/20100524/DI" id="Definitions_0vny0hv" targetNamespace="http://bpmn.io/schema/bpmn" exporter="Camunda Modeler" exporterVersion="3.5.0">
<bpmn:process id="Process_ResetWorkflow" name="Reset Workflow" isExecutable="true">
<bpmn:documentation>Use this process to reset a workflow for the current study. You must enter the name of the workflow. I.e., lower case with underscores.</bpmn:documentation>
<bpmn:startEvent id="StartEvent_1">
<bpmn:outgoing>SequenceFlow_0i872g2</bpmn:outgoing>
</bpmn:startEvent>
<bpmn:sequenceFlow id="SequenceFlow_0i872g2" sourceRef="StartEvent_1" targetRef="Task_GetWorkflow" />
<bpmn:sequenceFlow id="SequenceFlow_1q2ton3" sourceRef="Task_GetWorkflow" targetRef="Task_ResetWorkflow" />
<bpmn:sequenceFlow id="SequenceFlow_0x127gc" sourceRef="Task_ResetWorkflow" targetRef="Task_DisplayWorkflow" />
<bpmn:endEvent id="EndEvent_0fdym05">
<bpmn:incoming>SequenceFlow_0yy50p2</bpmn:incoming>
</bpmn:endEvent>
<bpmn:sequenceFlow id="SequenceFlow_0yy50p2" sourceRef="Task_DisplayWorkflow" targetRef="EndEvent_0fdym05" />
<bpmn:userTask id="Task_GetWorkflow" name="Get Workflow" camunda:formKey="WorkflowForm">
<bpmn:extensionElements>
<camunda:formData>
<camunda:formField id="workflow_name" label="Workflow Name" type="string">
<camunda:validation>
<camunda:constraint name="required" config="True" />
</camunda:validation>
</camunda:formField>
</camunda:formData>
</bpmn:extensionElements>
<bpmn:incoming>SequenceFlow_0i872g2</bpmn:incoming>
<bpmn:outgoing>SequenceFlow_1q2ton3</bpmn:outgoing>
</bpmn:userTask>
<bpmn:scriptTask id="Task_ResetWorkflow" name="Reset Workflow">
<bpmn:incoming>SequenceFlow_1q2ton3</bpmn:incoming>
<bpmn:outgoing>SequenceFlow_0x127gc</bpmn:outgoing>
<bpmn:script>value = reset_workflow(workflow_name=workflow_name)</bpmn:script>
</bpmn:scriptTask>
<bpmn:manualTask id="Task_DisplayWorkflow" name="Display Workflow">
<bpmn:documentation># Reset Workflow
&lt;div&gt;
{% if value %}
&lt;span&gt;Workflow {{workflow_name}} was reset.&lt;/span&gt;
{% else %}
&lt;span&gt;There was a problem resetting workflow {{workflow_name}}.&lt;/span&gt;
{% endif %}
&lt;/div&gt;
</bpmn:documentation>
<bpmn:incoming>SequenceFlow_0x127gc</bpmn:incoming>
<bpmn:outgoing>SequenceFlow_0yy50p2</bpmn:outgoing>
</bpmn:manualTask>
</bpmn:process>
<bpmndi:BPMNDiagram id="BPMNDiagram_1">
<bpmndi:BPMNPlane id="BPMNPlane_1" bpmnElement="Process_ResetWorkflow">
<bpmndi:BPMNShape id="_BPMNShape_StartEvent_2" bpmnElement="StartEvent_1">
<dc:Bounds x="179" y="99" width="36" height="36" />
</bpmndi:BPMNShape>
<bpmndi:BPMNEdge id="SequenceFlow_0i872g2_di" bpmnElement="SequenceFlow_0i872g2">
<di:waypoint x="215" y="117" />
<di:waypoint x="270" y="117" />
</bpmndi:BPMNEdge>
<bpmndi:BPMNEdge id="SequenceFlow_1q2ton3_di" bpmnElement="SequenceFlow_1q2ton3">
<di:waypoint x="370" y="117" />
<di:waypoint x="430" y="117" />
</bpmndi:BPMNEdge>
<bpmndi:BPMNEdge id="SequenceFlow_0x127gc_di" bpmnElement="SequenceFlow_0x127gc">
<di:waypoint x="530" y="117" />
<di:waypoint x="590" y="117" />
</bpmndi:BPMNEdge>
<bpmndi:BPMNShape id="EndEvent_0fdym05_di" bpmnElement="EndEvent_0fdym05">
<dc:Bounds x="752" y="99" width="36" height="36" />
</bpmndi:BPMNShape>
<bpmndi:BPMNEdge id="SequenceFlow_0yy50p2_di" bpmnElement="SequenceFlow_0yy50p2">
<di:waypoint x="690" y="117" />
<di:waypoint x="752" y="117" />
</bpmndi:BPMNEdge>
<bpmndi:BPMNShape id="UserTask_0li5ksb_di" bpmnElement="Task_GetWorkflow">
<dc:Bounds x="270" y="77" width="100" height="80" />
</bpmndi:BPMNShape>
<bpmndi:BPMNShape id="ScriptTask_07qq4pl_di" bpmnElement="Task_ResetWorkflow">
<dc:Bounds x="430" y="77" width="100" height="80" />
</bpmndi:BPMNShape>
<bpmndi:BPMNShape id="ManualTask_0ianu3f_di" bpmnElement="Task_DisplayWorkflow">
<dc:Bounds x="590" y="77" width="100" height="80" />
</bpmndi:BPMNShape>
</bpmndi:BPMNPlane>
</bpmndi:BPMNDiagram>
</bpmn:definitions>

View File

@ -0,0 +1,82 @@
<?xml version="1.0" encoding="UTF-8"?>
<bpmn:definitions xmlns:bpmn="http://www.omg.org/spec/BPMN/20100524/MODEL" xmlns:bpmndi="http://www.omg.org/spec/BPMN/20100524/DI" xmlns:dc="http://www.omg.org/spec/DD/20100524/DC" xmlns:camunda="http://camunda.org/schema/1.0/bpmn" xmlns:di="http://www.omg.org/spec/DD/20100524/DI" id="Definitions_04zta39" targetNamespace="http://bpmn.io/schema/bpmn" exporter="Camunda Modeler" exporterVersion="3.5.0">
<bpmn:process id="Process_NameAge" name="Name Age" isExecutable="true">
<bpmn:startEvent id="StartEvent_1">
<bpmn:outgoing>SequenceFlow_1oykjju</bpmn:outgoing>
</bpmn:startEvent>
<bpmn:sequenceFlow id="SequenceFlow_1oykjju" sourceRef="StartEvent_1" targetRef="Task_GetName" />
<bpmn:sequenceFlow id="SequenceFlow_0z8c3ob" sourceRef="Task_GetName" targetRef="Task_GetAge" />
<bpmn:sequenceFlow id="SequenceFlow_1jfrd7w" sourceRef="Task_GetAge" targetRef="Task_PrintData" />
<bpmn:userTask id="Task_GetName" name="Get Name" camunda:formKey="NameForm">
<bpmn:extensionElements>
<camunda:formData>
<camunda:formField id="name" label="Name" type="string">
<camunda:validation>
<camunda:constraint name="required" config="True" />
</camunda:validation>
</camunda:formField>
</camunda:formData>
</bpmn:extensionElements>
<bpmn:incoming>SequenceFlow_1oykjju</bpmn:incoming>
<bpmn:outgoing>SequenceFlow_0z8c3ob</bpmn:outgoing>
</bpmn:userTask>
<bpmn:userTask id="Task_GetAge" name="Get Age" camunda:formKey="AgeForm">
<bpmn:extensionElements>
<camunda:formData>
<camunda:formField id="age" label="Age" type="long">
<camunda:validation>
<camunda:constraint name="required" config="True" />
</camunda:validation>
</camunda:formField>
</camunda:formData>
</bpmn:extensionElements>
<bpmn:incoming>SequenceFlow_0z8c3ob</bpmn:incoming>
<bpmn:outgoing>SequenceFlow_1jfrd7w</bpmn:outgoing>
</bpmn:userTask>
<bpmn:manualTask id="Task_PrintData" name="Print Data">
<bpmn:documentation># Data
{{name}} is {{age}} years old.</bpmn:documentation>
<bpmn:incoming>SequenceFlow_1jfrd7w</bpmn:incoming>
<bpmn:outgoing>SequenceFlow_0yjk26l</bpmn:outgoing>
</bpmn:manualTask>
<bpmn:endEvent id="EndEvent_125fqq9">
<bpmn:incoming>SequenceFlow_0yjk26l</bpmn:incoming>
</bpmn:endEvent>
<bpmn:sequenceFlow id="SequenceFlow_0yjk26l" sourceRef="Task_PrintData" targetRef="EndEvent_125fqq9" />
</bpmn:process>
<bpmndi:BPMNDiagram id="BPMNDiagram_1">
<bpmndi:BPMNPlane id="BPMNPlane_1" bpmnElement="Process_NameAge">
<bpmndi:BPMNShape id="_BPMNShape_StartEvent_2" bpmnElement="StartEvent_1">
<dc:Bounds x="179" y="99" width="36" height="36" />
</bpmndi:BPMNShape>
<bpmndi:BPMNEdge id="SequenceFlow_1oykjju_di" bpmnElement="SequenceFlow_1oykjju">
<di:waypoint x="215" y="117" />
<di:waypoint x="270" y="117" />
</bpmndi:BPMNEdge>
<bpmndi:BPMNEdge id="SequenceFlow_0z8c3ob_di" bpmnElement="SequenceFlow_0z8c3ob">
<di:waypoint x="370" y="117" />
<di:waypoint x="430" y="117" />
</bpmndi:BPMNEdge>
<bpmndi:BPMNEdge id="SequenceFlow_1jfrd7w_di" bpmnElement="SequenceFlow_1jfrd7w">
<di:waypoint x="530" y="117" />
<di:waypoint x="590" y="117" />
</bpmndi:BPMNEdge>
<bpmndi:BPMNShape id="UserTask_1jrkk5z_di" bpmnElement="Task_GetName">
<dc:Bounds x="270" y="77" width="100" height="80" />
</bpmndi:BPMNShape>
<bpmndi:BPMNShape id="UserTask_080wksg_di" bpmnElement="Task_GetAge">
<dc:Bounds x="430" y="77" width="100" height="80" />
</bpmndi:BPMNShape>
<bpmndi:BPMNShape id="ManualTask_1k7rizm_di" bpmnElement="Task_PrintData">
<dc:Bounds x="590" y="77" width="100" height="80" />
</bpmndi:BPMNShape>
<bpmndi:BPMNShape id="EndEvent_125fqq9_di" bpmnElement="EndEvent_125fqq9">
<dc:Bounds x="752" y="99" width="36" height="36" />
</bpmndi:BPMNShape>
<bpmndi:BPMNEdge id="SequenceFlow_0yjk26l_di" bpmnElement="SequenceFlow_0yjk26l">
<di:waypoint x="690" y="117" />
<di:waypoint x="752" y="117" />
</bpmndi:BPMNEdge>
</bpmndi:BPMNPlane>
</bpmndi:BPMNDiagram>
</bpmn:definitions>

View File

@ -0,0 +1,45 @@
from tests.base_test import BaseTest
from crc.scripts.reset_workflow import ResetWorkflow
from crc.api.common import ApiError
class TestWorkflowReset(BaseTest):
def test_workflow_reset_validation(self):
self.load_example_data()
spec_model = self.load_test_spec('reset_workflow')
rv = self.app.get('/v1.0/workflow-specification/%s/validate' % spec_model.id, headers=self.logged_in_headers())
self.assertEqual([], rv.json)
def test_workflow_reset(self):
workflow = self.create_workflow('two_user_tasks')
workflow_api = self.get_workflow_api(workflow)
first_task = workflow_api.next_task
self.assertEqual('Task_GetName', first_task.name)
self.complete_form(workflow, first_task, {'name': 'Mona'})
workflow_api = self.get_workflow_api(workflow)
second_task = workflow_api.next_task
self.assertEqual('Task_GetAge', second_task.name)
ResetWorkflow().do_task(second_task, workflow.study_id, workflow.id, workflow_name='two_user_tasks')
workflow_api = self.get_workflow_api(workflow)
task = workflow_api.next_task
self.assertEqual('Task_GetName', task.name)
def test_workflow_reset_missing_name(self):
workflow = self.create_workflow('two_user_tasks')
workflow_api = self.get_workflow_api(workflow)
first_task = workflow_api.next_task
with self.assertRaises(ApiError):
ResetWorkflow().do_task(first_task, workflow.study_id, workflow.id)
def test_workflow_reset_bad_name(self):
workflow = self.create_workflow('two_user_tasks')
workflow_api = self.get_workflow_api(workflow)
first_task = workflow_api.next_task
with self.assertRaises(ApiError):
ResetWorkflow().do_task(first_task, workflow.study_id, workflow.id, workflow_name='bad_workflow_name')

View File

@ -2,12 +2,14 @@ import json
import unittest
from unittest.mock import patch
from sqlalchemy import func
from tests.base_test import BaseTest
from crc import session, app
from crc.api.common import ApiErrorSchema
from crc.models.protocol_builder import ProtocolBuilderStudySchema
from crc.models.workflow import WorkflowSpecModel
from crc.models.workflow import WorkflowSpecModel, WorkflowModel
from crc.services.workflow_service import WorkflowService
@ -15,8 +17,11 @@ class TestWorkflowSpecValidation(BaseTest):
def validate_workflow(self, workflow_name):
spec_model = self.load_test_spec(workflow_name)
total_workflows = session.query(WorkflowModel).count()
rv = self.app.get('/v1.0/workflow-specification/%s/validate' % spec_model.id, headers=self.logged_in_headers())
self.assert_success(rv)
total_workflows_after = session.query(WorkflowModel).count()
self.assertEqual(total_workflows, total_workflows_after, "No rogue workflow exists after validation.")
json_data = json.loads(rv.get_data(as_text=True))
return ApiErrorSchema(many=True).load(json_data)
@ -59,10 +64,7 @@ class TestWorkflowSpecValidation(BaseTest):
workflows = session.query(WorkflowSpecModel).all()
errors = []
for w in workflows:
rv = self.app.get('/v1.0/workflow-specification/%s/validate' % w.id,
headers=self.logged_in_headers())
self.assert_success(rv)
json_data = json.loads(rv.get_data(as_text=True))
json_data = self.validate_workflow(w.name)
errors.extend(ApiErrorSchema(many=True).load(json_data))
self.assertEqual(0, len(errors), json.dumps(errors))
@ -87,6 +89,7 @@ class TestWorkflowSpecValidation(BaseTest):
self.assertEqual("StartEvent_1", errors[0]['task_id'])
self.assertEqual("invalid_spec.bpmn", errors[0]['file_name'])
def test_invalid_script(self):
self.load_example_data()
errors = self.validate_workflow("invalid_script")