Reference cache integrity fix (#1627)

* added test to make sure process caller relationship table is properly updated and refactored clear reference cache to hopefully fix oddities when attempting to run it w/ burnettk

* fixed pyl w/ burnettk

---------

Co-authored-by: jasquat <jasquat@users.noreply.github.com>

spiffworkflow-backend/src/spiffworkflow_backend/routes/process_groups_controller.py

@@ -53,7 +53,7 @@ def process_group_delete(modified_process_group_id: str) -> flask.wrappers.Respo
         ProcessModelService.process_group_delete(process_group_id)
 
         # can't do this in the ProcessModelService due to circular imports
-        SpecFileService.clear_caches_for_process_group(process_group_id)
+        SpecFileService.clear_caches_for_item(process_group_id=process_group_id)
         db.session.commit()
 
     except ProcessModelWithInstancesNotDeletableError as exception:

spiffworkflow-backend/src/spiffworkflow_backend/routes/process_models_controller.py

@@ -117,7 +117,7 @@ def process_model_delete(
         ProcessModelService.process_model_delete(process_model_identifier)
 
         # can't do this in the ProcessModelService due to circular imports
-        SpecFileService.clear_caches_for_process_model(process_model)
+        SpecFileService.clear_caches_for_item(process_model_info=process_model)
         db.session.commit()
     except ProcessModelWithInstancesNotDeletableError as exception:
         raise ApiError(

spiffworkflow-backend/src/spiffworkflow_backend/services/process_caller_service.py

@@ -20,6 +20,7 @@ class ProcessCallerService:
 
     @staticmethod
     def clear_cache_for_process_ids(reference_cache_ids: list[int]) -> None:
+        # query-invoked autoflush happens here
         ProcessCallerRelationshipModel.query.filter(
             or_(
                 ProcessCallerRelationshipModel.called_reference_cache_process_id.in_(reference_cache_ids),

spiffworkflow-backend/src/spiffworkflow_backend/services/spec_file_service.py

@@ -166,7 +166,7 @@ class SpecFileService(FileSystemService):
             (ref for ref in references if ref.prop_is_true("is_primary") and ref.prop_is_true("is_executable")), None
         )
 
-        SpecFileService.clear_caches_for_file(file_name, process_model_info)
+        cls.clear_caches_for_item(file_name=file_name, process_model_info=process_model_info)
         all_called_element_ids: set[str] = set()
         for ref in references:
             # If no valid primary process is defined, default to the first process in the
@@ -190,9 +190,9 @@ class SpecFileService(FileSystemService):
 
             all_called_element_ids = all_called_element_ids | set(ref.called_element_ids)
             if update_process_cache_only:
-                SpecFileService.update_process_cache(ref)
+                cls.update_process_cache(ref)
             else:
-                SpecFileService.update_all_caches(ref)
+                cls.update_all_caches(ref)
 
         if user is not None:
             called_element_refs = (
@@ -218,24 +218,24 @@ class SpecFileService(FileSystemService):
         db.session.commit()
 
         # make sure we save the file as the last thing we do to ensure validations have run
-        full_file_path = SpecFileService.full_file_path(process_model_info, file_name)
+        full_file_path = cls.full_file_path(process_model_info, file_name)
-        SpecFileService.write_file_data_to_system(full_file_path, binary_data)
+        cls.write_file_data_to_system(full_file_path, binary_data)
-        return (SpecFileService.to_file_object(file_name, full_file_path), references)
+        return (cls.to_file_object(file_name, full_file_path), references)
 
-    @staticmethod
+    @classmethod
-    def last_modified(process_model: ProcessModelInfo, file_name: str) -> datetime:
+    def last_modified(cls, process_model: ProcessModelInfo, file_name: str) -> datetime:
-        full_file_path = SpecFileService.full_file_path(process_model, file_name)
+        full_file_path = cls.full_file_path(process_model, file_name)
         return FileSystemService._last_modified(full_file_path)
 
-    @staticmethod
+    @classmethod
-    def timestamp(process_model: ProcessModelInfo, file_name: str) -> float:
+    def timestamp(cls, process_model: ProcessModelInfo, file_name: str) -> float:
-        full_file_path = SpecFileService.full_file_path(process_model, file_name)
+        full_file_path = cls.full_file_path(process_model, file_name)
         return FileSystemService._timestamp(full_file_path)
 
     @classmethod
     def delete_file(cls, process_model: ProcessModelInfo, file_name: str) -> None:
-        cls.clear_caches_for_file(file_name, process_model)
+        cls.clear_caches_for_item(file_name=file_name, process_model_info=process_model)
-        full_file_path = SpecFileService.full_file_path(process_model, file_name)
+        full_file_path = cls.full_file_path(process_model, file_name)
         os.remove(full_file_path)
 
     @staticmethod
@@ -258,53 +258,26 @@ class SpecFileService(FileSystemService):
         SpecFileService.update_correlation_cache(ref)
 
     @staticmethod
-    def clear_caches_for_file(file_name: str, process_model_info: ProcessModelInfo) -> None:
+    def clear_caches_for_item(
-        """Clear all caches related to a file."""
+        file_name: str | None = None, process_model_info: ProcessModelInfo | None = None, process_group_id: str | None = None
-        records = (
+    ) -> None:
-            db.session.query(ReferenceCacheModel)
+        reference_cache_query = ReferenceCacheModel.basic_query()
-            .filter(ReferenceCacheModel.file_name == file_name)
+
-            .filter(ReferenceCacheModel.relative_location == process_model_info.id)
+        if process_group_id is not None:
-            .all()
+            reference_cache_query = reference_cache_query.filter(
+                ReferenceCacheModel.relative_location.like(f"{process_group_id}/%")  # type: ignore
             )
+        if file_name is not None:
+            reference_cache_query = reference_cache_query.filter(ReferenceCacheModel.file_name == file_name)
+        if process_model_info is not None:
+            reference_cache_query = reference_cache_query.filter(ReferenceCacheModel.relative_location == process_model_info.id)
 
+        records = reference_cache_query.all()
         reference_cache_ids = []
-
         for record in records:
             reference_cache_ids.append(record.id)
             db.session.delete(record)
 
-        ProcessCallerService.clear_cache_for_process_ids(reference_cache_ids)
-
-    @staticmethod
-    def clear_caches_for_process_group(process_group_id: str) -> None:
-        records = (
-            db.session.query(ReferenceCacheModel)
-            .filter(ReferenceCacheModel.relative_location.like(f"{process_group_id}/%"))  # type: ignore
-            .all()
-        )
-
-        reference_cache_ids = []
-
-        for record in records:
-            reference_cache_ids.append(record.id)
-            db.session.delete(record)
-
-        ProcessCallerService.clear_cache_for_process_ids(reference_cache_ids)
-
-    @staticmethod
-    def clear_caches_for_process_model(process_model_info: ProcessModelInfo) -> None:
-        records = (
-            db.session.query(ReferenceCacheModel).filter(ReferenceCacheModel.relative_location == process_model_info.id).all()
-        )
-
-        reference_cache_ids = []
-
-        for record in records:
-            reference_cache_ids.append(record.id)
-            db.session.delete(record)
-
-        ProcessCallerService.clear_cache_for_process_ids(reference_cache_ids)
-
     @staticmethod
     def update_process_cache(ref: Reference) -> None:
         process_id_lookup = ReferenceCacheModel.basic_query().filter_by(identifier=ref.identifier, type=ref.type).first()

spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_spec_file_service.py

@@ -7,6 +7,7 @@ from flask.testing import FlaskClient
 from lxml import etree  # type: ignore
 from spiffworkflow_backend.models.cache_generation import CacheGenerationModel
 from spiffworkflow_backend.models.db import db
+from spiffworkflow_backend.models.process_caller_relationship import ProcessCallerRelationshipModel
 from spiffworkflow_backend.models.reference_cache import ReferenceCacheModel
 from spiffworkflow_backend.services.process_model_service import ProcessModelService
 from spiffworkflow_backend.services.reference_cache_service import ReferenceCacheService
@@ -272,6 +273,37 @@ class TestSpecFileService(BaseTest):
         assert bpmn_process_id_lookups[0].identifier == "Level1"
         assert bpmn_process_id_lookups[0].generation_id == current_cache_generation.id
 
+    def test_can_correctly_clear_caches_for_a_file(
+        self,
+        app: Flask,
+        client: FlaskClient,
+        with_db_and_bpmn_file_cleanup: None,
+    ) -> None:
+        load_test_spec(
+            process_model_id=self.process_model_id,
+            process_model_source_directory="call_activity_nested",
+        )
+        bpmn_process_id_lookups = ReferenceCacheService.get_reference_cache_entries_calling_process(["Level2"])
+        assert len(bpmn_process_id_lookups) == 1
+        reference = bpmn_process_id_lookups[0]
+        assert reference.identifier == "Level1"
+        assert reference.relative_path() == self.call_activity_nested_relative_file_path
+
+        # ensure we add and remove from this table
+        process_caller_relationships = ProcessCallerRelationshipModel.query.all()
+        assert len(process_caller_relationships) == 4
+
+        process_model = ProcessModelService.get_process_model(reference.relative_location)
+        assert process_model is not None
+        SpecFileService.clear_caches_for_item(file_name=reference.file_name, process_model_info=process_model)
+        db.session.commit()
+
+        bpmn_process_id_lookups = ReferenceCacheService.get_reference_cache_entries_calling_process(["Level2"])
+        assert len(bpmn_process_id_lookups) == 0
+
+        process_caller_relationships = ProcessCallerRelationshipModel.query.all()
+        assert len(process_caller_relationships) == 2
+
     @pytest.mark.skipif(
         sys.platform == "win32",
         reason="tmp file path is not valid xml for windows and it doesn't matter",