diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/data_stores/json.py b/spiffworkflow-backend/src/spiffworkflow_backend/data_stores/json.py index 6f7223c4a..751867f8f 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/data_stores/json.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/data_stores/json.py @@ -7,6 +7,7 @@ from SpiffWorkflow.task import Task as SpiffTask # type: ignore from spiffworkflow_backend.models.db import db from spiffworkflow_backend.models.json_data_store import JSONDataStoreModel from spiffworkflow_backend.services.file_system_service import FileSystemService +from spiffworkflow_backend.services.reference_cache_service import ReferenceCacheService def _process_model_location_for_task(spiff_task: SpiffTask) -> str | None: @@ -24,14 +25,26 @@ def _data_store_exists_at_location(location: str, name: str) -> bool: return FileSystemService.file_exists_at_relative_path(location, _data_store_filename(name)) +def _data_store_location_for_task(spiff_task: SpiffTask, name: str) -> str | None: + location = _process_model_location_for_task(spiff_task) + if location is None: + return None + if _data_store_exists_at_location(location, name): + return location + location = ReferenceCacheService.upsearch(location, name, "data_store") + if location is None or not _data_store_exists_at_location(location, name): + return None + return location + + class JSONDataStore(BpmnDataStoreSpecification): # type: ignore """JSONDataStore.""" def get(self, my_task: SpiffTask) -> None: """get.""" model: JSONDataStoreModel | None = None - location = _process_model_location_for_task(my_task) - if location is not None and _data_store_exists_at_location(location, self.bpmn_id): + location = _data_store_location_for_task(my_task, self.bpmn_id) + if location is not None: model = db.session.query(JSONDataStoreModel).filter_by(name=self.bpmn_id, location=location).first() if model is None: raise Exception(f"Unable to read from data store '{self.bpmn_id}' using location '{location}'.") @@ -39,8 +52,8 @@ class JSONDataStore(BpmnDataStoreSpecification): # type: ignore def set(self, my_task: SpiffTask) -> None: """set.""" - location = _process_model_location_for_task(my_task) - if location is None or not _data_store_exists_at_location(location, self.bpmn_id): + location = _data_store_location_for_task(my_task, self.bpmn_id) + if location is None: raise Exception(f"Unable to write to data store '{self.bpmn_id}' using location '{location}'.") data = my_task.data[self.bpmn_id] model = JSONDataStoreModel( @@ -89,8 +102,8 @@ class JSONFileDataStore(BpmnDataStoreSpecification): # type: ignore def get(self, my_task: SpiffTask) -> None: """get.""" - location = _process_model_location_for_task(my_task) - if location is None or not _data_store_exists_at_location(location, self.bpmn_id): + location = _data_store_location_for_task(my_task, self.bpmn_id) + if location is None: raise Exception(f"Unable to read from data store '{self.bpmn_id}' using location '{location}'.") contents = FileSystemService.contents_of_json_file_at_relative_path( location, _data_store_filename(self.bpmn_id) @@ -99,8 +112,8 @@ class JSONFileDataStore(BpmnDataStoreSpecification): # type: ignore def set(self, my_task: SpiffTask) -> None: """set.""" - location = _process_model_location_for_task(my_task) - if location is None or not _data_store_exists_at_location(location, self.bpmn_id): + location = _data_store_location_for_task(my_task, self.bpmn_id) + if location is None: raise Exception(f"Unable to write to data store '{self.bpmn_id}' using location '{location}'.") data = my_task.data[self.bpmn_id] FileSystemService.write_to_json_file_at_relative_path(location, _data_store_filename(self.bpmn_id), data) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/services/data_setup_service.py b/spiffworkflow-backend/src/spiffworkflow_backend/services/data_setup_service.py index d8c4ba23e..075840233 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/services/data_setup_service.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/services/data_setup_service.py @@ -1,13 +1,12 @@ import os from flask import current_app -from spiffworkflow_backend.models.cache_generation import CacheGenerationModel from spiffworkflow_backend.models.db import db from spiffworkflow_backend.models.reference_cache import ReferenceCacheModel from spiffworkflow_backend.services.file_system_service import FileSystemService from spiffworkflow_backend.services.process_model_service import ProcessModelService +from spiffworkflow_backend.services.reference_cache_service import ReferenceCacheService from spiffworkflow_backend.services.spec_file_service import SpecFileService -from sqlalchemy import insert class DataSetupService: @@ -15,15 +14,6 @@ class DataSetupService: def run_setup(cls) -> list: return cls.save_all_process_models() - @classmethod - def add_unique_reference_cache_object( - cls, reference_objects: dict[str, ReferenceCacheModel], reference_cache: ReferenceCacheModel - ) -> None: - reference_cache_unique = ( - f"{reference_cache.identifier}{reference_cache.relative_location}{reference_cache.type}" - ) - reference_objects[reference_cache_unique] = reference_cache - @classmethod def save_all_process_models(cls) -> list: """Build a cache of all processes, messages, correlation keys, and start events. @@ -47,7 +37,7 @@ class DataSetupService: for ref in refs: try: reference_cache = ReferenceCacheModel.from_spec_reference(ref) - cls.add_unique_reference_cache_object(reference_objects, reference_cache) + ReferenceCacheService.add_unique_reference_cache_object(reference_objects, reference_cache) SpecFileService.update_caches_except_process(ref) db.session.commit() except Exception as ex: @@ -77,21 +67,10 @@ class DataSetupService: None, False, ) - cls.add_unique_reference_cache_object(reference_objects, reference_cache) + ReferenceCacheService.add_unique_reference_cache_object(reference_objects, reference_cache) current_app.logger.debug("DataSetupService.save_all_process_models() end") - # get inserted autoincrement primary key value back in a database agnostic way without committing the db session - ins = insert(CacheGenerationModel).values(cache_table="reference_cache") # type: ignore - res = db.session.execute(ins) - cache_generation_id = res.inserted_primary_key[0] + ReferenceCacheService.add_new_generation(reference_objects) - # add primary key value to each element in reference objects list and store in new list - reference_object_list_with_cache_generation_id = [] - for reference_object in reference_objects.values(): - reference_object.generation_id = cache_generation_id - reference_object_list_with_cache_generation_id.append(reference_object) - - db.session.bulk_save_objects(reference_object_list_with_cache_generation_id) - db.session.commit() return failing_process_models diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/services/reference_cache_service.py b/spiffworkflow-backend/src/spiffworkflow_backend/services/reference_cache_service.py new file mode 100644 index 000000000..a72894ac9 --- /dev/null +++ b/spiffworkflow-backend/src/spiffworkflow_backend/services/reference_cache_service.py @@ -0,0 +1,67 @@ +import os + +from spiffworkflow_backend.models.cache_generation import CacheGenerationModel +from spiffworkflow_backend.models.db import db +from spiffworkflow_backend.models.reference_cache import ReferenceCacheModel +from sqlalchemy import insert + + +class ReferenceCacheService: + @classmethod + def add_unique_reference_cache_object( + cls, reference_objects: dict[str, ReferenceCacheModel], reference_cache: ReferenceCacheModel + ) -> None: + reference_cache_unique = ( + f"{reference_cache.identifier}{reference_cache.relative_location}{reference_cache.type}" + ) + reference_objects[reference_cache_unique] = reference_cache + + @classmethod + def add_new_generation(cls, reference_objects: dict[str, ReferenceCacheModel]) -> None: + # get inserted autoincrement primary key value back in a database agnostic way without committing the db session + ins = insert(CacheGenerationModel).values(cache_table="reference_cache") # type: ignore + res = db.session.execute(ins) + cache_generation_id = res.inserted_primary_key[0] + + # add primary key value to each element in reference objects list and store in new list + reference_object_list_with_cache_generation_id = [] + for reference_object in reference_objects.values(): + reference_object.generation_id = cache_generation_id + reference_object_list_with_cache_generation_id.append(reference_object) + + db.session.bulk_save_objects(reference_object_list_with_cache_generation_id) + db.session.commit() + + @classmethod + def upsearch(cls, location: str, identifier: str, type: str) -> str | None: + # really want to be able to join to this table on max(id) + cache_generation = CacheGenerationModel.newest_generation_for_table("reference_cache") + if cache_generation is None: + return None + locations = cls.upsearch_locations(location) + references = ( + ReferenceCacheModel.query.filter_by( + identifier=identifier, + type=type, + generation=cache_generation, + ) + .filter(ReferenceCacheModel.relative_location.in_(locations)) # type: ignore + .order_by(ReferenceCacheModel.relative_location.desc()) # type: ignore + .all() + ) + + for reference in references: + # TODO: permissions check + return reference.relative_location # type: ignore + + return None + + @classmethod + def upsearch_locations(cls, location: str) -> list[str]: + locations = [] + + while location != "": + locations.append(location) + location = os.path.dirname(location) + + return locations diff --git a/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_reference_cache_service.py b/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_reference_cache_service.py new file mode 100644 index 000000000..ceb2a0681 --- /dev/null +++ b/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_reference_cache_service.py @@ -0,0 +1,69 @@ +from collections.abc import Generator + +import pytest +from flask.app import Flask +from spiffworkflow_backend.models.reference_cache import ReferenceCacheModel +from spiffworkflow_backend.services.reference_cache_service import ReferenceCacheService + +from tests.spiffworkflow_backend.helpers.base_test import BaseTest + + +@pytest.fixture() +def with_loaded_reference_cache(app: Flask, with_db_and_bpmn_file_cleanup: None) -> Generator[None, None, None]: + reference_objects: dict[str, ReferenceCacheModel] = {} + ReferenceCacheService.add_unique_reference_cache_object( + reference_objects, + ReferenceCacheModel.from_params( + "contacts_datastore", + "contacts_datastore", + "data_store", + "contacts_datastore.bpmn", + "misc/jonjon", + None, + False, + ), + ) + ReferenceCacheService.add_unique_reference_cache_object( + reference_objects, + ReferenceCacheModel.from_params( + "contacts_datastore", + "contacts_datastore", + "data_store", + "contacts_datastore.bpmn", + "misc/jonjon/generic-data-store-area/test-level-1", + None, + False, + ), + ) + + ReferenceCacheService.add_new_generation(reference_objects) + yield + + +class TestReferenceCacheService(BaseTest): + def test_upsearch_locations( + self, + ) -> None: + locations = ReferenceCacheService.upsearch_locations("misc/jonjon/generic-data-store-area/test-level-2") + assert locations == [ + "misc/jonjon/generic-data-store-area/test-level-2", + "misc/jonjon/generic-data-store-area", + "misc/jonjon", + "misc", + ] + + def test_can_find_data_store_in_current_location(self, with_loaded_reference_cache: None) -> None: + location = ReferenceCacheService.upsearch( + "misc/jonjon/generic-data-store-area/test-level-1", "contacts_datastore", "data_store" + ) + assert location == "misc/jonjon/generic-data-store-area/test-level-1" + + def test_can_find_data_store_in_upsearched_location(self, with_loaded_reference_cache: None) -> None: + location = ReferenceCacheService.upsearch( + "misc/jonjon/generic-data-store-area/test-level-2", "contacts_datastore", "data_store" + ) + assert location == "misc/jonjon" + + def test_does_not_find_data_store_in_non_upsearched_location(self, with_loaded_reference_cache: None) -> None: + location = ReferenceCacheService.upsearch("some/other/place", "contacts_datastore", "data_store") + assert location is None