Upsearch for data stores (#520)

spiffworkflow-backend/src/spiffworkflow_backend/data_stores/json.py

@@ -7,6 +7,7 @@ from SpiffWorkflow.task import Task as SpiffTask  # type: ignore
 from spiffworkflow_backend.models.db import db
 from spiffworkflow_backend.models.json_data_store import JSONDataStoreModel
 from spiffworkflow_backend.services.file_system_service import FileSystemService
+from spiffworkflow_backend.services.reference_cache_service import ReferenceCacheService
 
 
 def _process_model_location_for_task(spiff_task: SpiffTask) -> str | None:
@@ -24,14 +25,26 @@ def _data_store_exists_at_location(location: str, name: str) -> bool:
     return FileSystemService.file_exists_at_relative_path(location, _data_store_filename(name))
 
 
+def _data_store_location_for_task(spiff_task: SpiffTask, name: str) -> str | None:
+    location = _process_model_location_for_task(spiff_task)
+    if location is None:
+        return None
+    if _data_store_exists_at_location(location, name):
+        return location
+    location = ReferenceCacheService.upsearch(location, name, "data_store")
+    if location is None or not _data_store_exists_at_location(location, name):
+        return None
+    return location
+
+
 class JSONDataStore(BpmnDataStoreSpecification):  # type: ignore
     """JSONDataStore."""
 
     def get(self, my_task: SpiffTask) -> None:
         """get."""
         model: JSONDataStoreModel | None = None
-        location = _process_model_location_for_task(my_task)
+        location = _data_store_location_for_task(my_task, self.bpmn_id)
-        if location is not None and _data_store_exists_at_location(location, self.bpmn_id):
+        if location is not None:
             model = db.session.query(JSONDataStoreModel).filter_by(name=self.bpmn_id, location=location).first()
         if model is None:
             raise Exception(f"Unable to read from data store '{self.bpmn_id}' using location '{location}'.")
@@ -39,8 +52,8 @@ class JSONDataStore(BpmnDataStoreSpecification):  # type: ignore
 
     def set(self, my_task: SpiffTask) -> None:
         """set."""
-        location = _process_model_location_for_task(my_task)
+        location = _data_store_location_for_task(my_task, self.bpmn_id)
-        if location is None or not _data_store_exists_at_location(location, self.bpmn_id):
+        if location is None:
             raise Exception(f"Unable to write to data store '{self.bpmn_id}' using location '{location}'.")
         data = my_task.data[self.bpmn_id]
         model = JSONDataStoreModel(
@@ -89,8 +102,8 @@ class JSONFileDataStore(BpmnDataStoreSpecification):  # type: ignore
 
     def get(self, my_task: SpiffTask) -> None:
         """get."""
-        location = _process_model_location_for_task(my_task)
+        location = _data_store_location_for_task(my_task, self.bpmn_id)
-        if location is None or not _data_store_exists_at_location(location, self.bpmn_id):
+        if location is None:
             raise Exception(f"Unable to read from data store '{self.bpmn_id}' using location '{location}'.")
         contents = FileSystemService.contents_of_json_file_at_relative_path(
             location, _data_store_filename(self.bpmn_id)
@@ -99,8 +112,8 @@ class JSONFileDataStore(BpmnDataStoreSpecification):  # type: ignore
 
     def set(self, my_task: SpiffTask) -> None:
         """set."""
-        location = _process_model_location_for_task(my_task)
+        location = _data_store_location_for_task(my_task, self.bpmn_id)
-        if location is None or not _data_store_exists_at_location(location, self.bpmn_id):
+        if location is None:
             raise Exception(f"Unable to write to data store '{self.bpmn_id}' using location '{location}'.")
         data = my_task.data[self.bpmn_id]
         FileSystemService.write_to_json_file_at_relative_path(location, _data_store_filename(self.bpmn_id), data)

spiffworkflow-backend/src/spiffworkflow_backend/services/data_setup_service.py

@@ -1,13 +1,12 @@
 import os
 
 from flask import current_app
-from spiffworkflow_backend.models.cache_generation import CacheGenerationModel
 from spiffworkflow_backend.models.db import db
 from spiffworkflow_backend.models.reference_cache import ReferenceCacheModel
 from spiffworkflow_backend.services.file_system_service import FileSystemService
 from spiffworkflow_backend.services.process_model_service import ProcessModelService
+from spiffworkflow_backend.services.reference_cache_service import ReferenceCacheService
 from spiffworkflow_backend.services.spec_file_service import SpecFileService
-from sqlalchemy import insert
 
 
 class DataSetupService:
@@ -15,15 +14,6 @@ class DataSetupService:
     def run_setup(cls) -> list:
         return cls.save_all_process_models()
 
-    @classmethod
-    def add_unique_reference_cache_object(
-        cls, reference_objects: dict[str, ReferenceCacheModel], reference_cache: ReferenceCacheModel
-    ) -> None:
-        reference_cache_unique = (
-            f"{reference_cache.identifier}{reference_cache.relative_location}{reference_cache.type}"
-        )
-        reference_objects[reference_cache_unique] = reference_cache
-
     @classmethod
     def save_all_process_models(cls) -> list:
         """Build a cache of all processes, messages, correlation keys, and start events.
@@ -47,7 +37,7 @@ class DataSetupService:
                     for ref in refs:
                         try:
                             reference_cache = ReferenceCacheModel.from_spec_reference(ref)
-                            cls.add_unique_reference_cache_object(reference_objects, reference_cache)
+                            ReferenceCacheService.add_unique_reference_cache_object(reference_objects, reference_cache)
                             SpecFileService.update_caches_except_process(ref)
                             db.session.commit()
                         except Exception as ex:
@@ -77,21 +67,10 @@ class DataSetupService:
                     None,
                     False,
                 )
-                cls.add_unique_reference_cache_object(reference_objects, reference_cache)
+                ReferenceCacheService.add_unique_reference_cache_object(reference_objects, reference_cache)
 
         current_app.logger.debug("DataSetupService.save_all_process_models() end")
 
-        # get inserted autoincrement primary key value back in a database agnostic way without committing the db session
+        ReferenceCacheService.add_new_generation(reference_objects)
-        ins = insert(CacheGenerationModel).values(cache_table="reference_cache")  # type: ignore
-        res = db.session.execute(ins)
-        cache_generation_id = res.inserted_primary_key[0]
 
-        # add primary key value to each element in reference objects list and store in new list
-        reference_object_list_with_cache_generation_id = []
-        for reference_object in reference_objects.values():
-            reference_object.generation_id = cache_generation_id
-            reference_object_list_with_cache_generation_id.append(reference_object)
-
-        db.session.bulk_save_objects(reference_object_list_with_cache_generation_id)
-        db.session.commit()
         return failing_process_models

spiffworkflow-backend/src/spiffworkflow_backend/services/reference_cache_service.py

@@ -0,0 +1,67 @@
+import os
+
+from spiffworkflow_backend.models.cache_generation import CacheGenerationModel
+from spiffworkflow_backend.models.db import db
+from spiffworkflow_backend.models.reference_cache import ReferenceCacheModel
+from sqlalchemy import insert
+
+
+class ReferenceCacheService:
+    @classmethod
+    def add_unique_reference_cache_object(
+        cls, reference_objects: dict[str, ReferenceCacheModel], reference_cache: ReferenceCacheModel
+    ) -> None:
+        reference_cache_unique = (
+            f"{reference_cache.identifier}{reference_cache.relative_location}{reference_cache.type}"
+        )
+        reference_objects[reference_cache_unique] = reference_cache
+
+    @classmethod
+    def add_new_generation(cls, reference_objects: dict[str, ReferenceCacheModel]) -> None:
+        # get inserted autoincrement primary key value back in a database agnostic way without committing the db session
+        ins = insert(CacheGenerationModel).values(cache_table="reference_cache")  # type: ignore
+        res = db.session.execute(ins)
+        cache_generation_id = res.inserted_primary_key[0]
+
+        # add primary key value to each element in reference objects list and store in new list
+        reference_object_list_with_cache_generation_id = []
+        for reference_object in reference_objects.values():
+            reference_object.generation_id = cache_generation_id
+            reference_object_list_with_cache_generation_id.append(reference_object)
+
+        db.session.bulk_save_objects(reference_object_list_with_cache_generation_id)
+        db.session.commit()
+
+    @classmethod
+    def upsearch(cls, location: str, identifier: str, type: str) -> str | None:
+        # really want to be able to join to this table on max(id)
+        cache_generation = CacheGenerationModel.newest_generation_for_table("reference_cache")
+        if cache_generation is None:
+            return None
+        locations = cls.upsearch_locations(location)
+        references = (
+            ReferenceCacheModel.query.filter_by(
+                identifier=identifier,
+                type=type,
+                generation=cache_generation,
+            )
+            .filter(ReferenceCacheModel.relative_location.in_(locations))  # type: ignore
+            .order_by(ReferenceCacheModel.relative_location.desc())  # type: ignore
+            .all()
+        )
+
+        for reference in references:
+            # TODO: permissions check
+            return reference.relative_location  # type: ignore
+
+        return None
+
+    @classmethod
+    def upsearch_locations(cls, location: str) -> list[str]:
+        locations = []
+
+        while location != "":
+            locations.append(location)
+            location = os.path.dirname(location)
+
+        return locations

spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_reference_cache_service.py

@@ -0,0 +1,69 @@
+from collections.abc import Generator
+
+import pytest
+from flask.app import Flask
+from spiffworkflow_backend.models.reference_cache import ReferenceCacheModel
+from spiffworkflow_backend.services.reference_cache_service import ReferenceCacheService
+
+from tests.spiffworkflow_backend.helpers.base_test import BaseTest
+
+
+@pytest.fixture()
+def with_loaded_reference_cache(app: Flask, with_db_and_bpmn_file_cleanup: None) -> Generator[None, None, None]:
+    reference_objects: dict[str, ReferenceCacheModel] = {}
+    ReferenceCacheService.add_unique_reference_cache_object(
+        reference_objects,
+        ReferenceCacheModel.from_params(
+            "contacts_datastore",
+            "contacts_datastore",
+            "data_store",
+            "contacts_datastore.bpmn",
+            "misc/jonjon",
+            None,
+            False,
+        ),
+    )
+    ReferenceCacheService.add_unique_reference_cache_object(
+        reference_objects,
+        ReferenceCacheModel.from_params(
+            "contacts_datastore",
+            "contacts_datastore",
+            "data_store",
+            "contacts_datastore.bpmn",
+            "misc/jonjon/generic-data-store-area/test-level-1",
+            None,
+            False,
+        ),
+    )
+
+    ReferenceCacheService.add_new_generation(reference_objects)
+    yield
+
+
+class TestReferenceCacheService(BaseTest):
+    def test_upsearch_locations(
+        self,
+    ) -> None:
+        locations = ReferenceCacheService.upsearch_locations("misc/jonjon/generic-data-store-area/test-level-2")
+        assert locations == [
+            "misc/jonjon/generic-data-store-area/test-level-2",
+            "misc/jonjon/generic-data-store-area",
+            "misc/jonjon",
+            "misc",
+        ]
+
+    def test_can_find_data_store_in_current_location(self, with_loaded_reference_cache: None) -> None:
+        location = ReferenceCacheService.upsearch(
+            "misc/jonjon/generic-data-store-area/test-level-1", "contacts_datastore", "data_store"
+        )
+        assert location == "misc/jonjon/generic-data-store-area/test-level-1"
+
+    def test_can_find_data_store_in_upsearched_location(self, with_loaded_reference_cache: None) -> None:
+        location = ReferenceCacheService.upsearch(
+            "misc/jonjon/generic-data-store-area/test-level-2", "contacts_datastore", "data_store"
+        )
+        assert location == "misc/jonjon"
+
+    def test_does_not_find_data_store_in_non_upsearched_location(self, with_loaded_reference_cache: None) -> None:
+        location = ReferenceCacheService.upsearch("some/other/place", "contacts_datastore", "data_store")
+        assert location is None