Feature/better subworkflow management (#331)

* WIP: some updates to support new spiff w/ burnettk

* unit tests are passing

* all tests except message tests are passing

* fixed usage of catch message event w/ burnettk

* messages are working again w/ burnettk

* uncommented remaining message tests w/ burnettk

* fixed cypress tests w/ burnettk

* use main for spiffworkflow

---------

Co-authored-by: jasquat <jasquat@users.noreply.github.com>

spiffworkflow-backend/bin/last_pi_milestone_query

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+
+function error_handler() {
+  >&2 echo "Exited with BAD EXIT CODE '${2}' in ${0} script at line: ${1}."
+  exit "$2"
+}
+trap 'error_handler ${LINENO} $?' ERR
+set -o errtrace -o errexit -o nounset -o pipefail
+
+mysql -uroot spiffworkflow_backend_local_development -e '
+  SELECT td.bpmn_identifier FROM process_instance p
+
+  JOIN process_instance_event pie ON pie.process_instance_id = p.id
+  JOIN task t ON t.guid = pie.task_guid
+  JOIN task_definition td ON td.id = t.task_definition_id
+
+  JOIN (
+    SELECT max(pie.id) as max_pie_id
+    FROM process_instance_event pie
+
+    JOIN task t ON t.guid = pie.task_guid
+    JOIN task_definition td ON td.id = t.task_definition_id
+    JOIN bpmn_process bp ON bp.id = t.bpmn_process_id
+
+    WHERE td.typename = "IntermediateThrowEvent" OR (bp.direct_parent_process_id is NULL AND td.typename IN ("SimpleBpmnTask", "BpmnStartTask"))
+
+    GROUP BY pie.process_instance_id
+  ) AS max_pie ON max_pie.max_pie_id = pie.id
+
+  WHERE pie.process_instance_id = 27
+  ;
+'

spiffworkflow-backend/poetry.lock

@@ -2385,7 +2385,7 @@ lxml = "*"
 type = "git"
 url = "https://github.com/sartography/SpiffWorkflow"
 reference = "main"
-resolved_reference = "efcdcf545c861d58cfae92ad070b3208ef6028db"
+resolved_reference = "dbe41c8e3e082b7a4e6e207a2c82bd55685a92fc"
 
 [[package]]
 name = "sqlalchemy"

spiffworkflow-backend/pyproject.toml

@@ -30,7 +30,6 @@ flask-restful = "*"
 flask-simple-crypt = "^0.3.3"
 werkzeug = "*"
 SpiffWorkflow = {git = "https://github.com/sartography/SpiffWorkflow", rev = "main"}
-# SpiffWorkflow = {git = "https://github.com/sartography/SpiffWorkflow", rev = "6cad2981712bb61eca23af1adfafce02d3277cb9"}
 # SpiffWorkflow = {develop = true, path = "../../spiffworkflow/" }
 sentry-sdk = "^1.10"
 # sphinx-autoapi = "^2.0"

spiffworkflow-backend/src/spiffworkflow_backend/services/message_service.py

@@ -1,3 +1,4 @@
+from SpiffWorkflow.bpmn.workflow import BpmnMessage  # type: ignore
 from spiffworkflow_backend.models.db import db
 from spiffworkflow_backend.models.message_instance import MessageInstanceModel
 from spiffworkflow_backend.models.message_instance import MessageStatuses
@@ -143,8 +144,13 @@ class MessageService:
         message_model_name: str,
         message_payload: dict,
     ) -> None:
+        bpmn_message = BpmnMessage(
+            None,
+            message_model_name,
+            message_payload,
+        )
         processor_receive = ProcessInstanceProcessor(process_instance_receive)
-        processor_receive.bpmn_process_instance.catch_bpmn_message(message_model_name, message_payload)
+        processor_receive.bpmn_process_instance.catch_bpmn_message(bpmn_message)
         processor_receive.do_engine_steps(save=True)
         message_instance_receive.status = MessageStatuses.completed.value
         db.session.add(message_instance_receive)

spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py

@@ -1018,7 +1018,7 @@ class ProcessInstanceProcessor:
 
                 # in the xml, it's the id attribute. this identifies the process where the activity lives.
                 # if it's in a subprocess, it's the inner process.
-                bpmn_process_identifier = ready_or_waiting_task.workflow.name
+                bpmn_process_identifier = ready_or_waiting_task.workflow.spec.name
 
                 form_file_name = None
                 ui_form_file_name = None
@@ -1081,9 +1081,6 @@ class ProcessInstanceProcessor:
         event_definition = self._event_serializer.registry.restore(event_data)
         if payload is not None:
             event_definition.payload = payload
-        current_app.logger.info(
-            f"Event of type {event_definition.event_type} sent to process instance {self.process_instance_model.id}"
-        )
         try:
             self.bpmn_process_instance.catch(event_definition)
         except Exception as e:

spiffworkflow-backend/src/spiffworkflow_backend/services/task_service.py

@@ -157,7 +157,7 @@ class TaskService:
         """
         (parent_subprocess_guid, _parent_subprocess) = self.__class__._task_subprocess(spiff_task)
         if parent_subprocess_guid is not None:
-            spiff_task_of_parent_subprocess = spiff_task.workflow._get_outermost_workflow().get_task_from_id(
+            spiff_task_of_parent_subprocess = spiff_task.workflow.top_workflow.get_task_from_id(
                 UUID(parent_subprocess_guid)
             )
 
@@ -242,11 +242,11 @@ class TaskService:
 
         self.bpmn_processes[bpmn_process.guid or "top_level"] = bpmn_process
 
-        if spiff_workflow.outer_workflow != spiff_workflow:
+        if spiff_workflow.parent_task_id:
             direct_parent_bpmn_process = BpmnProcessModel.query.filter_by(
                 id=bpmn_process.direct_parent_process_id
             ).first()
-            self.update_bpmn_process(spiff_workflow.outer_workflow, direct_parent_bpmn_process)
+            self.update_bpmn_process(spiff_workflow.parent_workflow, direct_parent_bpmn_process)
 
     def update_task_model(
         self,
@@ -311,7 +311,7 @@ class TaskService:
             # This is the top level workflow, which has no guid
             # check for bpmn_process_id because mypy doesn't realize bpmn_process can be None
             if self.process_instance.bpmn_process_id is None:
-                spiff_workflow = spiff_task.workflow._get_outermost_workflow()
+                spiff_workflow = spiff_task.workflow.top_workflow
                 bpmn_process = self.add_bpmn_process(
                     bpmn_process_dict=self.serializer.workflow_to_dict(spiff_workflow),
                     spiff_workflow=spiff_workflow,
@@ -321,7 +321,7 @@ class TaskService:
             if bpmn_process is None:
                 spiff_workflow = spiff_task.workflow
                 bpmn_process = self.add_bpmn_process(
-                    bpmn_process_dict=self.serializer.workflow_to_dict(subprocess),
+                    bpmn_process_dict=self.serializer.subworkflow_to_dict(subprocess),
                     top_level_process=self.process_instance.bpmn_process,
                     bpmn_process_guid=subprocess_guid,
                     spiff_workflow=spiff_workflow,
@@ -369,10 +369,10 @@ class TaskService:
             bpmn_process.bpmn_process_definition = bpmn_process_definition
 
             if top_level_process is not None:
-                subprocesses = spiff_workflow._get_outermost_workflow().subprocesses
+                subprocesses = spiff_workflow.top_workflow.subprocesses
                 direct_bpmn_process_parent = top_level_process
                 for subprocess_guid, subprocess in subprocesses.items():
-                    if subprocess == spiff_workflow.outer_workflow:
+                    if subprocess == spiff_workflow.parent_workflow:
                         direct_bpmn_process_parent = BpmnProcessModel.query.filter_by(
                             guid=str(subprocess_guid)
                         ).first()
@@ -674,7 +674,7 @@ class TaskService:
 
     @classmethod
     def _task_subprocess(cls, spiff_task: SpiffTask) -> tuple[str | None, BpmnWorkflow | None]:
-        top_level_workflow = spiff_task.workflow._get_outermost_workflow()
+        top_level_workflow = spiff_task.workflow.top_workflow
         my_wf = spiff_task.workflow  # This is the workflow the spiff_task is part of
         my_sp = None
         my_sp_id = None

spiffworkflow-backend/src/spiffworkflow_backend/services/workflow_execution_service.py

@@ -451,7 +451,6 @@ class WorkflowExecutionService:
     def queue_waiting_receive_messages(self) -> None:
         waiting_events = self.bpmn_process_instance.waiting_events()
         waiting_message_events = filter(lambda e: e["event_type"] == "MessageEventDefinition", waiting_events)
-
         for event in waiting_message_events:
             # Ensure we are only creating one message instance for each waiting message
             if (

spiffworkflow-backend/tests/data/message_send_one_conversation/message_receiver.bpmn

@@ -12,18 +12,18 @@
   </bpmn:collaboration>
   <bpmn:correlationProperty id="customer_id" name="Customer Id">
     <bpmn:correlationPropertyRetrievalExpression messageRef="request_approval">
-      <bpmn:formalExpression>customer_id</bpmn:formalExpression>
+      <bpmn:messagePath>customer_id</bpmn:messagePath>
     </bpmn:correlationPropertyRetrievalExpression>
     <bpmn:correlationPropertyRetrievalExpression messageRef="approval_result">
-      <bpmn:formalExpression>customer_id</bpmn:formalExpression>
+      <bpmn:messagePath>customer_id</bpmn:messagePath>
     </bpmn:correlationPropertyRetrievalExpression>
   </bpmn:correlationProperty>
   <bpmn:correlationProperty id="po_number" name="Purchase Order Number">
     <bpmn:correlationPropertyRetrievalExpression messageRef="request_approval">
-      <bpmn:formalExpression>po_number</bpmn:formalExpression>
+      <bpmn:messagePath>po_number</bpmn:messagePath>
     </bpmn:correlationPropertyRetrievalExpression>
     <bpmn:correlationPropertyRetrievalExpression messageRef="approval_result">
-      <bpmn:formalExpression>po_number</bpmn:formalExpression>
+      <bpmn:messagePath>po_number</bpmn:messagePath>
     </bpmn:correlationPropertyRetrievalExpression>
   </bpmn:correlationProperty>
   <bpmn:message id="request_approval" name="Request Approval">

spiffworkflow-backend/tests/data/message_send_one_conversation/message_sender.bpmn

@@ -18,18 +18,18 @@ It will fire a message connected to the invoice keys above, starting another pro
   </bpmn:collaboration>
   <bpmn:correlationProperty id="po_number" name="Purchase Order Number">
     <bpmn:correlationPropertyRetrievalExpression messageRef="request_approval">
-      <bpmn:formalExpression>po_number</bpmn:formalExpression>
+      <bpmn:messagePath>po_number</bpmn:messagePath>
     </bpmn:correlationPropertyRetrievalExpression>
     <bpmn:correlationPropertyRetrievalExpression messageRef="approval_result">
-      <bpmn:formalExpression>po_number</bpmn:formalExpression>
+      <bpmn:messagePath>po_number</bpmn:messagePath>
     </bpmn:correlationPropertyRetrievalExpression>
   </bpmn:correlationProperty>
   <bpmn:correlationProperty id="customer_id" name="Customer ID">
     <bpmn:correlationPropertyRetrievalExpression messageRef="request_approval">
-      <bpmn:formalExpression>customer_id</bpmn:formalExpression>
+      <bpmn:messagePath>customer_id</bpmn:messagePath>
     </bpmn:correlationPropertyRetrievalExpression>
     <bpmn:correlationPropertyRetrievalExpression messageRef="approval_result">
-      <bpmn:formalExpression>customer_id</bpmn:formalExpression>
+      <bpmn:messagePath>customer_id</bpmn:messagePath>
     </bpmn:correlationPropertyRetrievalExpression>
   </bpmn:correlationProperty>
   <bpmn:process id="message_send_process" name="Message Send Process" isExecutable="true">

spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_error_handling_service.py

@@ -1,7 +1,7 @@
 import pytest
 from flask import Flask
 from flask.testing import FlaskClient
-from spiffworkflow_backend import db
+from spiffworkflow_backend.models.db import db
 from spiffworkflow_backend.models.message_instance import MessageInstanceModel
 from spiffworkflow_backend.models.process_instance import ProcessInstanceModel
 from spiffworkflow_backend.models.process_instance import ProcessInstanceStatus

spiffworkflow-frontend/cypress/e2e/process_models.cy.js

@@ -74,6 +74,7 @@ describe('process-models', () => {
     cy.contains(`${jsonFileName}.json`).should('not.exist');
 
     // add new bpmn file
+    cy.getBySel('process-model-add-file').click();
     cy.contains('New BPMN File').click();
     cy.contains(/^Process Model File$/);
     cy.get('g[data-element-id=StartEvent_1]').click();
@@ -92,6 +93,7 @@ describe('process-models', () => {
     cy.contains(`${bpmnFileName}.bpmn`).should('exist');
 
     // add new dmn file
+    cy.getBySel('process-model-add-file').click();
     cy.contains('New DMN File').click();
     cy.contains(/^Process Model File$/);
     cy.get('g[data-element-id=decision_1]').click();
@@ -109,6 +111,7 @@ describe('process-models', () => {
     cy.contains(`${dmnFileName}.dmn`).should('exist');
 
     // add new json file
+    cy.getBySel('process-model-add-file').click();
     cy.contains('New JSON File').click();
     cy.contains(/^Process Model File$/);
     // Some reason, cypress evals json strings so we have to escape it it with '{{}'

spiffworkflow-frontend/src/routes/ProcessModelShow.tsx

@@ -526,6 +526,7 @@ export default function ProcessModelShow() {
         size="lg"
         label="Add File"
         type="inline"
+        data-qa="process-model-add-file"
         onChange={(a: any) => {
           if (a.selectedItem.text === 'New BPMN File') {
             navigate(