Pi fix pm gone (#1648)

* some better error logging when the background processor fails w/ burnettk

* handle process model not found issues better and do not raise in handle exception if it is missing w/ burnettk

---------

Co-authored-by: jasquat <jasquat@users.noreply.github.com>

spiffworkflow-backend/src/spiffworkflow_backend/models/reference_cache.py

@@ -18,7 +18,6 @@ from spiffworkflow_backend.models.db import db
 from spiffworkflow_backend.models.process_caller_relationship import ProcessCallerRelationshipModel
 
 
-# SpecReferenceNotFoundError
 class ReferenceNotFoundError(Exception):
     pass
 
@@ -29,7 +28,6 @@ class ReferenceType(SpiffEnum):
     data_store = "data_store"
 
 
-# SpecReference
 @dataclass()
 class Reference:
     """File Reference Information.
@@ -43,7 +41,7 @@ class Reference:
     identifier: str  # The id of the process or decision.  "Process_1234"
     display_name: str  # The name of the process or decision. "Invoice Submission"
     relative_location: str
-    type: str  # can be 'process' or 'decision'
+    type: str
     file_name: str  # The name of the file where this process or decision is defined.
     messages: dict  # Any messages defined in the same file where this process is defined.
     correlations: dict  # Any correlations defined in the same file with this process.
@@ -80,9 +78,6 @@ class ReferenceCacheModel(SpiffworkflowBaseDBModel):
     relative_location: str = db.Column(db.String(255), index=True, nullable=False)
 
     properties: dict | None = db.Column(db.JSON)
-    # has_lanes = db.Column(db.Boolean())
-    # is_executable = db.Column(db.Boolean())
-    # is_primary = db.Column(db.Boolean())
 
     generation = relationship(CacheGenerationModel)
 

spiffworkflow-backend/src/spiffworkflow_backend/routes/process_instances_controller.py

@@ -633,9 +633,12 @@ def _get_process_instance(
         name_of_file_with_diagram = spec_reference.file_name
         process_instance.process_model_with_diagram_identifier = process_model_with_diagram.id
     else:
-        process_model_with_diagram = _get_process_model(process_model_identifier)
-        if process_model_with_diagram.primary_file_name:
-            name_of_file_with_diagram = process_model_with_diagram.primary_file_name
+        try:
+            process_model_with_diagram = _get_process_model(process_model_identifier)
+            if process_model_with_diagram.primary_file_name:
+                name_of_file_with_diagram = process_model_with_diagram.primary_file_name
+        except Exception as ex:
+            process_instance.bpmn_xml_file_contents_retrieval_error = str(ex)
 
     if process_model_with_diagram and name_of_file_with_diagram:
         bpmn_xml_file_contents = None

spiffworkflow-backend/src/spiffworkflow_backend/services/error_handling_service.py

@@ -1,10 +1,10 @@
 from flask import current_app
 from flask import g
 
+from spiffworkflow_backend.exceptions.process_entity_not_found_error import ProcessEntityNotFoundError
 from spiffworkflow_backend.models.db import db
 from spiffworkflow_backend.models.process_instance import ProcessInstanceModel
 from spiffworkflow_backend.models.process_instance import ProcessInstanceStatus
-from spiffworkflow_backend.models.process_model import ProcessModelInfo
 from spiffworkflow_backend.services.process_model_service import ProcessModelService
 
 
@@ -14,15 +14,23 @@ class ErrorHandlingService:
     @classmethod
     def handle_error(cls, process_instance: ProcessInstanceModel, error: Exception) -> None:
         """On unhandled exceptions, set instance.status based on model.fault_or_suspend_on_exception."""
-        process_model = ProcessModelService.get_process_model(process_instance.process_model_identifier)
-        cls._update_process_instance_in_database(process_instance, process_model.fault_or_suspend_on_exception)
+        fault_or_suspend_on_exception = "fault"
+        exception_notification_addresses = []
+        try:
+            process_model = ProcessModelService.get_process_model(process_instance.process_model_identifier)
+            fault_or_suspend_on_exception = process_model.fault_or_suspend_on_exception
+            exception_notification_addresses = process_model.exception_notification_addresses
+        except ProcessEntityNotFoundError:
+            pass
+
+        cls._update_process_instance_in_database(process_instance, fault_or_suspend_on_exception)
 
         # Second, send a bpmn message out, but only if an exception notification address is provided
         # This will create a new Send Message with correlation keys on the recipients and the message
         # body.
-        if len(process_model.exception_notification_addresses) > 0:
+        if len(exception_notification_addresses) > 0:
             try:
-                cls._handle_system_notification(error, process_model, process_instance)
+                cls._handle_system_notification(error, process_instance, exception_notification_addresses)
             except Exception as e:
                 # hmm... what to do if a notification method fails. Probably log, at least
                 current_app.logger.error(e)
@@ -52,8 +60,8 @@ class ErrorHandlingService:
     @staticmethod
     def _handle_system_notification(
         error: Exception,
-        process_model: ProcessModelInfo,
         process_instance: ProcessInstanceModel,
+        exception_notification_addresses: list,
     ) -> None:
         """Send a BPMN Message - which may kick off a waiting process."""
 
@@ -63,12 +71,12 @@ class ErrorHandlingService:
         from spiffworkflow_backend.services.message_service import MessageService
 
         message_text = (
-            f"There was an exception running process model {process_model.id} for instance"
+            f"There was an exception running process model {process_instance.process_model_identifier} for instance"
             f" {process_instance.id}.\nOriginal Error:\n{error.__repr__()}"
         )
         message_payload = {
             "message_text": message_text,
-            "recipients": process_model.exception_notification_addresses,
+            "recipients": exception_notification_addresses,
         }
         user_id = None
         if "user" in g:

spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py

@@ -912,7 +912,24 @@ class ProcessInstanceProcessor:
             "lane_assignment_id": lane_assignment_id,
         }
 
-    def extract_metadata(self, process_model_info: ProcessModelInfo) -> None:
+    def extract_metadata(self) -> None:
+        # we are currently not getting the metadata extraction paths based on the version in git from the process instance.
+        # it would make sense to do that if the shell-out-to-git performance cost was not too high.
+        # we also discussed caching this information in new database tables. something like:
+        #   process_model_version
+        #     id
+        #     process_model_identifier
+        #     git_hash
+        #     display_name
+        #     notification_type
+        #   metadata_extraction
+        #     id
+        #     extraction_key
+        #     extraction_path
+        #   metadata_extraction_process_model_version
+        #     process_model_version_id
+        #     metadata_extraction_id
+        process_model_info = ProcessModelService.get_process_model(self.process_instance_model.process_model_identifier)
         metadata_extraction_paths = process_model_info.metadata_extraction_paths
         if metadata_extraction_paths is None:
             return
@@ -1090,12 +1107,7 @@ class ProcessInstanceProcessor:
         human_tasks = HumanTaskModel.query.filter_by(process_instance_id=self.process_instance_model.id, completed=False).all()
         ready_or_waiting_tasks = self.get_all_ready_or_waiting_tasks()
 
-        process_model_display_name = ""
-        process_model_info = ProcessModelService.get_process_model(self.process_instance_model.process_model_identifier)
-        if process_model_info is not None:
-            process_model_display_name = process_model_info.display_name
-
-        self.extract_metadata(process_model_info)
+        self.extract_metadata()
 
         for ready_or_waiting_task in ready_or_waiting_tasks:
             # filter out non-usertasks
@@ -1131,7 +1143,7 @@ class ProcessInstanceProcessor:
 
                     human_task = HumanTaskModel(
                         process_instance_id=self.process_instance_model.id,
-                        process_model_display_name=process_model_display_name,
+                        process_model_display_name=self.process_instance_model.process_model_display_name,
                         bpmn_process_identifier=bpmn_process_identifier,
                         form_file_name=form_file_name,
                         ui_form_file_name=ui_form_file_name,

spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_service.py

@@ -264,14 +264,15 @@ class ProcessInstanceService:
                     process_instance, status_value=status_value, execution_strategy_name=execution_strategy_name
                 )
             except ProcessInstanceIsAlreadyLockedError:
+                # we will try again later
                 continue
-            except Exception as e:
+            except Exception as exception:
                 db.session.rollback()  # in case the above left the database with a bad transaction
-                error_message = (
+                new_exception = Exception(
                     f"Error running {status_value} task for process_instance {process_instance.id}"
-                    + f"({process_instance.process_model_identifier}). {str(e)}"
+                    + f"({process_instance.process_model_identifier}). {exception.__class__.__name__}: {str(exception)}"
                 )
-                current_app.logger.error(error_message)
+                current_app.logger.exception(new_exception, stack_info=True)
 
     @classmethod
     def run_process_instance_with_processor(

spiffworkflow-frontend/src/components/ProcessBreadcrumb.tsx

@@ -40,6 +40,10 @@ export default function ProcessBreadcrumb({ hotCrumbs }: OwnProps) {
             )}`,
             successCallback: setProcessEntity,
             onUnauthorized: () => {},
+            failureCallback: (error: any) =>
+              console.error(
+                `Failed to load process model for breadcrumb. Error was: ${error.message}`
+              ),
           });
         } else if (entityType === 'process-group-id') {
           HttpService.makeCallToBackend({
@@ -48,6 +52,10 @@ export default function ProcessBreadcrumb({ hotCrumbs }: OwnProps) {
             )}`,
             successCallback: setProcessEntity,
             onUnauthorized: () => {},
+            failureCallback: (error: any) =>
+              console.error(
+                `Failed to load process group for breadcrumb. Error was: ${error.message}`
+              ),
           });
         } else {
           setProcessEntity(entityToExplode as any);