Fill out repeating sections during validation process.

Also, when returning error messages, attempt to include the task data for the task that caused the error.
Also, when attempting to delete any file, respond with an API error explaining the issue, and log the details.

crc/api/common.py

@@ -3,7 +3,7 @@ from crc import ma, app
 
 class ApiError(Exception):
     def __init__(self, code, message, status_code=400,
-                 file_name="", task_id="", task_name="", tag=""):
+                 file_name="", task_id="", task_name="", tag="", task_data = {}):
         self.status_code = status_code
         self.code = code  # a short consistent string describing the error.
         self.message = message  # A detailed message that provides more information.
@@ -11,6 +11,7 @@ class ApiError(Exception):
         self.task_name = task_name or ""  # OPTIONAL: The name of the task in the BPMN Diagram.
         self.file_name = file_name or ""  # OPTIONAL: The file that caused the error.
         self.tag = tag or ""  # OPTIONAL: The XML Tag that caused the issue.
+        self.task_data = task_data or ""  # OPTIONAL: A snapshot of data connected to the task when error ocurred.
         Exception.__init__(self, self.message)
 
     @classmethod
@@ -20,6 +21,7 @@ class ApiError(Exception):
         instance.task_id = task.task_spec.name or ""
         instance.task_name = task.task_spec.description or ""
         instance.file_name = task.workflow.spec.file or ""
+        instance.task_data = task.data
         return instance
 
     @classmethod
@@ -35,7 +37,8 @@ class ApiError(Exception):
 
 class ApiErrorSchema(ma.Schema):
     class Meta:
-        fields = ("code", "message", "workflow_name", "file_name", "task_name", "task_id")
+        fields = ("code", "message", "workflow_name", "file_name", "task_name", "task_id",
+                  "task_data")
 
 
 @app.errorhandler(ApiError)

crc/models/api_models.py

@@ -31,6 +31,7 @@ class NavigationItem(object):
 
 class Task(object):
 
+    PROP_OPTIONS_REPEAT = "repeat"
     PROP_OPTIONS_FILE = "spreadsheet.name"
     PROP_OPTIONS_VALUE_COLUMN = "spreadsheet.value.column"
     PROP_OPTIONS_LABEL_COL = "spreadsheet.label.column"

crc/services/file_service.py

@@ -5,11 +5,13 @@ from datetime import datetime
 from uuid import UUID
 from xml.etree import ElementTree
 
+import flask
 from SpiffWorkflow.bpmn.parser.ValidationException import ValidationException
 from pandas import ExcelFile
 from sqlalchemy import desc
+from sqlalchemy.exc import IntegrityError
 
-from crc import session
+from crc import session, app
 from crc.api.common import ApiError
 from crc.models.file import FileType, FileDataModel, FileModel, LookupFileModel, LookupDataModel
 from crc.models.workflow import WorkflowSpecModel, WorkflowModel, WorkflowSpecDependencyFile
@@ -295,12 +297,17 @@ class FileService(object):
 
     @staticmethod
     def delete_file(file_id):
-        data_models = session.query(FileDataModel).filter_by(file_model_id=file_id).all()
+        try:
-        for dm in data_models:
+            data_models = session.query(FileDataModel).filter_by(file_model_id=file_id).all()
-            lookup_files = session.query(LookupFileModel).filter_by(file_data_model_id=dm.id).all()
+            for dm in data_models:
-            for lf in lookup_files:
+                lookup_files = session.query(LookupFileModel).filter_by(file_data_model_id=dm.id).all()
-                session.query(LookupDataModel).filter_by(lookup_file_model_id=lf.id).delete()
+                for lf in lookup_files:
-                session.query(LookupFileModel).filter_by(id=lf.id).delete()
+                    session.query(LookupDataModel).filter_by(lookup_file_model_id=lf.id).delete()
-        session.query(FileDataModel).filter_by(file_model_id=file_id).delete()
+                    session.query(LookupFileModel).filter_by(id=lf.id).delete()
-        session.query(FileModel).filter_by(id=file_id).delete()
+            session.query(FileDataModel).filter_by(file_model_id=file_id).delete()
-        session.commit()
+            session.query(FileModel).filter_by(id=file_id).delete()
+            session.commit()
+        except IntegrityError as ie:
+            app.logger.error("Failed to delete file: %i, due to %s" % (file_id, str(ie)))
+            raise ApiError('file_integrity_error', "You are attempting to delete a file that is "
+                                                   "required by other records in the system.")

crc/services/study_service.py

@@ -4,6 +4,7 @@ from typing import List
 
 import requests
 from SpiffWorkflow import WorkflowException
+from SpiffWorkflow.exceptions import WorkflowTaskExecException
 from ldap3.core.exceptions import LDAPSocketOpenError
 
 from crc import db, session, app
@@ -309,6 +310,8 @@ class StudyService(object):
         for workflow_spec in new_specs:
             try:
                 StudyService._create_workflow_model(study_model, workflow_spec)
+            except WorkflowTaskExecException as wtee:
+                errors.append(ApiError.from_task("workflow_execution_exception", str(wtee), wtee.task))
             except WorkflowException as we:
                 errors.append(ApiError.from_task_spec("workflow_execution_exception", str(we), we.sender))
         return errors

crc/services/workflow_service.py

@@ -7,8 +7,9 @@ from SpiffWorkflow import Task as SpiffTask, WorkflowException
 from SpiffWorkflow.bpmn.specs.ManualTask import ManualTask
 from SpiffWorkflow.bpmn.specs.ScriptTask import ScriptTask
 from SpiffWorkflow.bpmn.specs.UserTask import UserTask
-from SpiffWorkflow.bpmn.workflow import BpmnWorkflow
+from SpiffWorkflow.camunda.specs.UserTask import EnumFormField
 from SpiffWorkflow.dmn.specs.BusinessRuleTask import BusinessRuleTask
+from SpiffWorkflow.exceptions import WorkflowTaskExecException
 from SpiffWorkflow.specs import CancelTask, StartTask
 from flask import g
 from jinja2 import Template
@@ -17,7 +18,6 @@ from crc import db, app
 from crc.api.common import ApiError
 from crc.models.api_models import Task, MultiInstanceType
 from crc.models.file import LookupDataModel
-from crc.models.protocol_builder import ProtocolBuilderStatus
 from crc.models.stats import TaskEventModel
 from crc.models.study import StudyModel
 from crc.models.user import UserModel
@@ -39,7 +39,9 @@ class WorkflowService(object):
      the workflow Processor should be hidden behind this service.
      This will help maintain a structure that avoids circular dependencies.
      But for now, this contains tools for converting spiff-workflow models into our
-     own API models with additional information and capabilities."""
+     own API models with additional information and capabilities and 
+     handles the testing of a workflow specification by completing it with
+     random selections, attempting to mimic a front end as much as possible. """
 
     @staticmethod
     def make_test_workflow(spec_id):
@@ -61,17 +63,25 @@ class WorkflowService(object):
         for study in db.session.query(StudyModel).filter(StudyModel.user_uid=="test"):
             StudyService.delete_study(study.id)
             db.session.commit()
-        db.session.query(UserModel).filter_by(uid="test").delete()
+
+        user = db.session.query(UserModel).filter_by(uid="test").first()
+        if user:
+            db.session.delete(user)
 
     @staticmethod
     def test_spec(spec_id):
-        """Runs a spec through it's paces to see if it results in any errors.  Not fool-proof, but a good
+        """Runs a spec through it's paces to see if it results in any errors.
-        sanity check."""
+          Not fool-proof, but a good sanity check.  Returns the final data
+          output form the last task if successful. """
 
         workflow_model = WorkflowService.make_test_workflow(spec_id)
 
         try:
             processor = WorkflowProcessor(workflow_model, validate_only=True)
+        except WorkflowTaskExecException as wtee:
+            WorkflowService.delete_test_data()
+            raise ApiError.from_task("workflow_execution_exception", str(wtee),
+                                          wtee.task)
         except WorkflowException as we:
             WorkflowService.delete_test_data()
             raise ApiError.from_task_spec("workflow_execution_exception", str(we),
@@ -87,11 +97,17 @@ class WorkflowService(object):
                         add_docs_and_forms=True)  # Assure we try to process the documenation, and raise those errors.
                     WorkflowService.populate_form_with_random_data(task, task_api)
                     task.complete()
+            except WorkflowTaskExecException as wtee:
+                WorkflowService.delete_test_data()
+                raise ApiError.from_task("workflow_execution_exception", str(wtee),
+                                         wtee.task)
             except WorkflowException as we:
                 WorkflowService.delete_test_data()
                 raise ApiError.from_task_spec("workflow_execution_exception", str(we),
                                               we.sender)
+
         WorkflowService.delete_test_data()
+        return processor.bpmn_workflow.last_task.data
 
     @staticmethod
     def populate_form_with_random_data(task, task_api):
@@ -101,22 +117,35 @@ class WorkflowService(object):
 
         form_data = {}
         for field in task_api.form.fields:
-            if field.type == "enum":
+            if field.has_property(Task.PROP_OPTIONS_REPEAT):
-                if len(field.options) > 0:
+                group = field.get_property(Task.PROP_OPTIONS_REPEAT)
-                    random_choice = random.choice(field.options)
+                if group not in form_data:
-                    if isinstance(random_choice, dict):
+                    form_data[group] = [{},{},{}]
-                        form_data[field.id] = random.choice(field.options)['id']
+                for i in range(3):
-                    else:
+                    form_data[group][i][field.id] = WorkflowService.get_random_data_for_field(field, task)
-                        # fixme: why it is sometimes an EnumFormFieldOption, and other times not?
+            else:
-                        form_data[field.id] = random_choice.id ## Assume it is an EnumFormFieldOption
+                form_data[field.id] = WorkflowService.get_random_data_for_field(field, task)
+        if task.data is None:
+            task.data = {}
+        task.data.update(form_data)
+
+    @staticmethod
+    def get_random_data_for_field(field, task):
+        if field.type == "enum":
+            if len(field.options) > 0:
+                random_choice = random.choice(field.options)
+                if isinstance(random_choice, dict):
+                    return random.choice(field.options)['id']
                 else:
-                    raise ApiError.from_task("invalid_enum", "You specified an enumeration field (%s),"
+                    # fixme: why it is sometimes an EnumFormFieldOption, and other times not?
-                                                             " with no options" % field.id,
+                    return random_choice.id  ## Assume it is an EnumFormFieldOption
-                                             task)
+            else:
-            elif field.type == "autocomplete":
+                raise ApiError.from_task("invalid_enum", "You specified an enumeration field (%s),"
-                lookup_model = LookupService.get_lookup_model(task, field)
+                                                         " with no options" % field.id, task)
-                if field.has_property(Task.PROP_LDAP_LOOKUP):
+        elif field.type == "autocomplete":
-                    form_data[field.id] = {
+            lookup_model = LookupService.get_lookup_model(task, field)
+            if field.has_property(Task.PROP_LDAP_LOOKUP):  # All ldap records get the same person.
+                return {
                         "label": "dhf8r",
                         "value": "Dan Funk",
                         "data": {
@@ -126,32 +155,30 @@ class WorkflowService(object):
                             "email_address": "dhf8r@virginia.edu",
                             "department": "Depertment of Psychocosmographictology",
                             "affiliation": "Rousabout",
-                            "sponsor_type": "Staff"
+                            "sponsor_type": "Staff"}
                         }
-                    }
+            elif lookup_model:
-                elif lookup_model:
+                data = db.session.query(LookupDataModel).filter(
-                    data = db.session.query(LookupDataModel).filter(
+                    LookupDataModel.lookup_file_model == lookup_model).limit(10).all()
-                        LookupDataModel.lookup_file_model == lookup_model).limit(10).all()
+                options = []
-                    options = []
+                for d in data:
-                    for d in data:
+                    options.append({"id": d.value, "name": d.label})
-                        options.append({"id": d.value, "name": d.label})
+                return random.choice(options)
-                    form_data[field.id] = random.choice(options)
-                else:
-                    raise ApiError.from_task("invalid_autocomplete", "The settings for this auto complete field "
-                                                                     "are incorrect: %s " % field.id, task)
-            elif field.type == "long":
-                form_data[field.id] = random.randint(1, 1000)
-            elif field.type == 'boolean':
-                form_data[field.id] = random.choice([True, False])
-            elif field.type == 'file':
-                form_data[field.id] = random.randint(1, 100)
-            elif field.type == 'files':
-                form_data[field.id] = random.randrange(1, 100)
             else:
-                form_data[field.id] = WorkflowService._random_string()
+                raise ApiError.from_task("invalid_autocomplete", "The settings for this auto complete field "
-        if task.data is None:
+                                                                 "are incorrect: %s " % field.id, task)
-            task.data = {}
+        elif field.type == "long":
-        task.data.update(form_data)
+            return random.randint(1, 1000)
+        elif field.type == 'boolean':
+            return random.choice([True, False])
+        elif field.type == 'file':
+            # fixme: produce some something sensible for files.
+            return random.randint(1, 100)
+            # fixme: produce some something sensible for files.
+        elif field.type == 'files':
+            return random.randrange(1, 100)
+        else:
+            return WorkflowService._random_string()
 
     def __get_options(self):
         pass

tests/data/repeat_form/repeat_form.bpmn

@@ -0,0 +1,47 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<bpmn:definitions xmlns:bpmn="http://www.omg.org/spec/BPMN/20100524/MODEL" xmlns:bpmndi="http://www.omg.org/spec/BPMN/20100524/DI" xmlns:dc="http://www.omg.org/spec/DD/20100524/DC" xmlns:camunda="http://camunda.org/schema/1.0/bpmn" xmlns:di="http://www.omg.org/spec/DD/20100524/DI" id="Definitions_1v1rp1q" targetNamespace="http://bpmn.io/schema/bpmn" exporter="Camunda Modeler" exporterVersion="3.7.3">
+  <bpmn:process id="Repeat" isExecutable="true">
+    <bpmn:startEvent id="StartEvent_1">
+      <bpmn:outgoing>SequenceFlow_0lvudp8</bpmn:outgoing>
+    </bpmn:startEvent>
+    <bpmn:sequenceFlow id="SequenceFlow_0lvudp8" sourceRef="StartEvent_1" targetRef="Task_14svgcu" />
+    <bpmn:endEvent id="EndEvent_0q4qzl9">
+      <bpmn:incoming>SequenceFlow_02vev7n</bpmn:incoming>
+    </bpmn:endEvent>
+    <bpmn:sequenceFlow id="SequenceFlow_02vev7n" sourceRef="Task_14svgcu" targetRef="EndEvent_0q4qzl9" />
+    <bpmn:userTask id="Task_14svgcu" name="Repeating Form" camunda:formKey="RepeatForm">
+      <bpmn:extensionElements>
+        <camunda:formData>
+          <camunda:formField id="name" label="Add a cat name" type="string" defaultValue="couger buttons">
+            <camunda:properties>
+              <camunda:property id="repeat" value="cats" />
+            </camunda:properties>
+          </camunda:formField>
+        </camunda:formData>
+      </bpmn:extensionElements>
+      <bpmn:incoming>SequenceFlow_0lvudp8</bpmn:incoming>
+      <bpmn:outgoing>SequenceFlow_02vev7n</bpmn:outgoing>
+    </bpmn:userTask>
+  </bpmn:process>
+  <bpmndi:BPMNDiagram id="BPMNDiagram_1">
+    <bpmndi:BPMNPlane id="BPMNPlane_1" bpmnElement="Repeat">
+      <bpmndi:BPMNEdge id="SequenceFlow_02vev7n_di" bpmnElement="SequenceFlow_02vev7n">
+        <di:waypoint x="370" y="117" />
+        <di:waypoint x="432" y="117" />
+      </bpmndi:BPMNEdge>
+      <bpmndi:BPMNEdge id="SequenceFlow_0lvudp8_di" bpmnElement="SequenceFlow_0lvudp8">
+        <di:waypoint x="215" y="117" />
+        <di:waypoint x="270" y="117" />
+      </bpmndi:BPMNEdge>
+      <bpmndi:BPMNShape id="_BPMNShape_StartEvent_2" bpmnElement="StartEvent_1">
+        <dc:Bounds x="179" y="99" width="36" height="36" />
+      </bpmndi:BPMNShape>
+      <bpmndi:BPMNShape id="EndEvent_0q4qzl9_di" bpmnElement="EndEvent_0q4qzl9">
+        <dc:Bounds x="432" y="99" width="36" height="36" />
+      </bpmndi:BPMNShape>
+      <bpmndi:BPMNShape id="UserTask_18ly1yq_di" bpmnElement="Task_14svgcu">
+        <dc:Bounds x="270" y="77" width="100" height="80" />
+      </bpmndi:BPMNShape>
+    </bpmndi:BPMNPlane>
+  </bpmndi:BPMNDiagram>
+</bpmn:definitions>

tests/test_workflow_spec_validation_api.py

@@ -3,17 +3,16 @@ from unittest.mock import patch
 
 from tests.base_test import BaseTest
 
-from crc.services.protocol_builder import ProtocolBuilderService
 from crc import session, app
 from crc.api.common import ApiErrorSchema
 from crc.models.protocol_builder import ProtocolBuilderStudySchema
 from crc.models.workflow import WorkflowSpecModel
+from crc.services.workflow_service import WorkflowService
 
 
 class TestWorkflowSpecValidation(BaseTest):
 
     def validate_workflow(self, workflow_name):
-        self.load_example_data()
         spec_model = self.load_test_spec(workflow_name)
         rv = self.app.get('/v1.0/workflow-specification/%s/validate' % spec_model.id, headers=self.logged_in_headers())
         self.assert_success(rv)
@@ -22,6 +21,7 @@ class TestWorkflowSpecValidation(BaseTest):
 
     def test_successful_validation_of_test_workflows(self):
         app.config['PB_ENABLED'] = False  # Assure this is disabled.
+        self.load_example_data()
         self.assertEqual(0, len(self.validate_workflow("parallel_tasks")))
         self.assertEqual(0, len(self.validate_workflow("decision_table")))
         self.assertEqual(0, len(self.validate_workflow("docx")))
@@ -60,6 +60,7 @@ class TestWorkflowSpecValidation(BaseTest):
         self.assertEqual(0, len(errors), json.dumps(errors))
 
     def test_invalid_expression(self):
+        self.load_example_data()
         errors = self.validate_workflow("invalid_expression")
         self.assertEqual(1, len(errors))
         self.assertEqual("workflow_execution_exception", errors[0]['code'])
@@ -68,8 +69,11 @@ class TestWorkflowSpecValidation(BaseTest):
         self.assertEqual("invalid_expression.bpmn", errors[0]['file_name'])
         self.assertEqual('ExclusiveGateway_003amsm: Error evaluating expression \'this_value_does_not_exist==true\', '
                          'name \'this_value_does_not_exist\' is not defined', errors[0]["message"])
+        self.assertIsNotNone(errors[0]['task_data'])
+        self.assertIn("has_bananas", errors[0]['task_data'])
 
     def test_validation_error(self):
+        self.load_example_data()
         errors = self.validate_workflow("invalid_spec")
         self.assertEqual(1, len(errors))
         self.assertEqual("workflow_validation_error", errors[0]['code'])
@@ -77,6 +81,7 @@ class TestWorkflowSpecValidation(BaseTest):
         self.assertEqual("invalid_spec.bpmn", errors[0]['file_name'])
 
     def test_invalid_script(self):
+        self.load_example_data()
         errors = self.validate_workflow("invalid_script")
         self.assertEqual(1, len(errors))
         self.assertEqual("workflow_execution_exception", errors[0]['code'])
@@ -84,3 +89,10 @@ class TestWorkflowSpecValidation(BaseTest):
         self.assertEqual("Invalid_Script_Task", errors[0]['task_id'])
         self.assertEqual("An Invalid Script Reference", errors[0]['task_name'])
         self.assertEqual("invalid_script.bpmn", errors[0]['file_name'])
+
+    def test_repeating_sections_correctly_populated(self):
+        self.load_example_data()
+        spec_model = self.load_test_spec('repeat_form')
+        final_data = WorkflowService.test_spec(spec_model.id)
+        self.assertIsNotNone(final_data)
+        self.assertIn('cats', final_data)