sartography
/
spiff-arena
mirror of https://github.com/sartography/spiff-arena.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Upsearch for data stores (#520)

This commit is contained in:
jbirddog 2023-10-04 09:42:25 -04:00 committed by GitHub
parent 22570ce8d3
commit 34a0323c4d
4 changed files with 161 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
spiffworkflow-backend/src/spiffworkflow_backend/data_stores/json.py
View File

@ -7,6 +7,7 @@ from SpiffWorkflow.task import Task as SpiffTask # type: ignore
from spiffworkflow_backend.models.db import db
from spiffworkflow_backend.models.json_data_store import JSONDataStoreModel
from spiffworkflow_backend.services.file_system_service import FileSystemService
from spiffworkflow_backend.services.reference_cache_service import ReferenceCacheService
def _process_model_location_for_task(spiff_task: SpiffTask) -> str | None:
@ -24,14 +25,26 @@ def _data_store_exists_at_location(location: str, name: str) -> bool:
return FileSystemService.file_exists_at_relative_path(location, _data_store_filename(name))
def _data_store_location_for_task(spiff_task: SpiffTask, name: str) -> str | None:
location = _process_model_location_for_task(spiff_task)
if location is None:
return None
if _data_store_exists_at_location(location, name):
return location
location = ReferenceCacheService.upsearch(location, name, "data_store")
if location is None or not _data_store_exists_at_location(location, name):
return None
return location
class JSONDataStore(BpmnDataStoreSpecification): # type: ignore
"""JSONDataStore."""
def get(self, my_task: SpiffTask) -> None:
"""get."""
model: JSONDataStoreModel | None = None
location = _process_model_location_for_task(my_task)
if location is not None and _data_store_exists_at_location(location, self.bpmn_id):
location = _data_store_location_for_task(my_task, self.bpmn_id)
if location is not None:
model = db.session.query(JSONDataStoreModel).filter_by(name=self.bpmn_id, location=location).first()
if model is None:
raise Exception(f"Unable to read from data store '{self.bpmn_id}' using location '{location}'.")
@ -39,8 +52,8 @@ class JSONDataStore(BpmnDataStoreSpecification): # type: ignore
def set(self, my_task: SpiffTask) -> None:
"""set."""
location = _process_model_location_for_task(my_task)
if location is None or not _data_store_exists_at_location(location, self.bpmn_id):
location = _data_store_location_for_task(my_task, self.bpmn_id)
if location is None:
raise Exception(f"Unable to write to data store '{self.bpmn_id}' using location '{location}'.")
data = my_task.data[self.bpmn_id]
model = JSONDataStoreModel(
@ -89,8 +102,8 @@ class JSONFileDataStore(BpmnDataStoreSpecification): # type: ignore
def get(self, my_task: SpiffTask) -> None:
"""get."""
location = _process_model_location_for_task(my_task)
if location is None or not _data_store_exists_at_location(location, self.bpmn_id):
location = _data_store_location_for_task(my_task, self.bpmn_id)
if location is None:
raise Exception(f"Unable to read from data store '{self.bpmn_id}' using location '{location}'.")
contents = FileSystemService.contents_of_json_file_at_relative_path(
location, _data_store_filename(self.bpmn_id)
@ -99,8 +112,8 @@ class JSONFileDataStore(BpmnDataStoreSpecification): # type: ignore
def set(self, my_task: SpiffTask) -> None:
"""set."""
location = _process_model_location_for_task(my_task)
if location is None or not _data_store_exists_at_location(location, self.bpmn_id):
location = _data_store_location_for_task(my_task, self.bpmn_id)
if location is None:
raise Exception(f"Unable to write to data store '{self.bpmn_id}' using location '{location}'.")
data = my_task.data[self.bpmn_id]
FileSystemService.write_to_json_file_at_relative_path(location, _data_store_filename(self.bpmn_id), data)

29
spiffworkflow-backend/src/spiffworkflow_backend/services/data_setup_service.py
View File

@ -1,13 +1,12 @@
import os
from flask import current_app
from spiffworkflow_backend.models.cache_generation import CacheGenerationModel
from spiffworkflow_backend.models.db import db
from spiffworkflow_backend.models.reference_cache import ReferenceCacheModel
from spiffworkflow_backend.services.file_system_service import FileSystemService
from spiffworkflow_backend.services.process_model_service import ProcessModelService
from spiffworkflow_backend.services.reference_cache_service import ReferenceCacheService
from spiffworkflow_backend.services.spec_file_service import SpecFileService
from sqlalchemy import insert
class DataSetupService:
@ -15,15 +14,6 @@ class DataSetupService:
def run_setup(cls) -> list:
return cls.save_all_process_models()
@classmethod
def add_unique_reference_cache_object(
cls, reference_objects: dict[str, ReferenceCacheModel], reference_cache: ReferenceCacheModel
) -> None:
reference_cache_unique = (
f"{reference_cache.identifier}{reference_cache.relative_location}{reference_cache.type}"
)
reference_objects[reference_cache_unique] = reference_cache
@classmethod
def save_all_process_models(cls) -> list:
"""Build a cache of all processes, messages, correlation keys, and start events.
@ -47,7 +37,7 @@ class DataSetupService:
for ref in refs:
try:
reference_cache = ReferenceCacheModel.from_spec_reference(ref)
cls.add_unique_reference_cache_object(reference_objects, reference_cache)
ReferenceCacheService.add_unique_reference_cache_object(reference_objects, reference_cache)
SpecFileService.update_caches_except_process(ref)
db.session.commit()
except Exception as ex:
@ -77,21 +67,10 @@ class DataSetupService:
None,
False,
)
cls.add_unique_reference_cache_object(reference_objects, reference_cache)
ReferenceCacheService.add_unique_reference_cache_object(reference_objects, reference_cache)
current_app.logger.debug("DataSetupService.save_all_process_models() end")
# get inserted autoincrement primary key value back in a database agnostic way without committing the db session
ins = insert(CacheGenerationModel).values(cache_table="reference_cache") # type: ignore
res = db.session.execute(ins)
cache_generation_id = res.inserted_primary_key[0]
ReferenceCacheService.add_new_generation(reference_objects)
# add primary key value to each element in reference objects list and store in new list
reference_object_list_with_cache_generation_id = []
for reference_object in reference_objects.values():
reference_object.generation_id = cache_generation_id
reference_object_list_with_cache_generation_id.append(reference_object)
db.session.bulk_save_objects(reference_object_list_with_cache_generation_id)
db.session.commit()
return failing_process_models

67
spiffworkflow-backend/src/spiffworkflow_backend/services/reference_cache_service.py Normal file
View File

@ -0,0 +1,67 @@
import os
from spiffworkflow_backend.models.cache_generation import CacheGenerationModel
from spiffworkflow_backend.models.db import db
from spiffworkflow_backend.models.reference_cache import ReferenceCacheModel
from sqlalchemy import insert
class ReferenceCacheService:
@classmethod
def add_unique_reference_cache_object(
cls, reference_objects: dict[str, ReferenceCacheModel], reference_cache: ReferenceCacheModel
) -> None:
reference_cache_unique = (
f"{reference_cache.identifier}{reference_cache.relative_location}{reference_cache.type}"
)
reference_objects[reference_cache_unique] = reference_cache
@classmethod
def add_new_generation(cls, reference_objects: dict[str, ReferenceCacheModel]) -> None:
# get inserted autoincrement primary key value back in a database agnostic way without committing the db session
ins = insert(CacheGenerationModel).values(cache_table="reference_cache") # type: ignore
res = db.session.execute(ins)
cache_generation_id = res.inserted_primary_key[0]
# add primary key value to each element in reference objects list and store in new list
reference_object_list_with_cache_generation_id = []
for reference_object in reference_objects.values():
reference_object.generation_id = cache_generation_id
reference_object_list_with_cache_generation_id.append(reference_object)
db.session.bulk_save_objects(reference_object_list_with_cache_generation_id)
db.session.commit()
@classmethod
def upsearch(cls, location: str, identifier: str, type: str) -> str | None:
# really want to be able to join to this table on max(id)
cache_generation = CacheGenerationModel.newest_generation_for_table("reference_cache")
if cache_generation is None:
return None
locations = cls.upsearch_locations(location)
references = (
ReferenceCacheModel.query.filter_by(
identifier=identifier,
type=type,
generation=cache_generation,
)
.filter(ReferenceCacheModel.relative_location.in_(locations)) # type: ignore
.order_by(ReferenceCacheModel.relative_location.desc()) # type: ignore
.all()
)
for reference in references:
# TODO: permissions check
return reference.relative_location # type: ignore
return None
@classmethod
def upsearch_locations(cls, location: str) -> list[str]:
locations = []
while location != "":
locations.append(location)
location = os.path.dirname(location)
return locations

69
spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_reference_cache_service.py Normal file
View File

@ -0,0 +1,69 @@
from collections.abc import Generator
import pytest
from flask.app import Flask
from spiffworkflow_backend.models.reference_cache import ReferenceCacheModel
from spiffworkflow_backend.services.reference_cache_service import ReferenceCacheService
from tests.spiffworkflow_backend.helpers.base_test import BaseTest
@pytest.fixture()
def with_loaded_reference_cache(app: Flask, with_db_and_bpmn_file_cleanup: None) -> Generator[None, None, None]:
reference_objects: dict[str, ReferenceCacheModel] = {}
ReferenceCacheService.add_unique_reference_cache_object(
reference_objects,
ReferenceCacheModel.from_params(
"contacts_datastore",
"contacts_datastore",
"data_store",
"contacts_datastore.bpmn",
"misc/jonjon",
None,
False,
),
)
ReferenceCacheService.add_unique_reference_cache_object(
reference_objects,
ReferenceCacheModel.from_params(
"contacts_datastore",
"contacts_datastore",
"data_store",
"contacts_datastore.bpmn",
"misc/jonjon/generic-data-store-area/test-level-1",
None,
False,
),
)
ReferenceCacheService.add_new_generation(reference_objects)
yield
class TestReferenceCacheService(BaseTest):
def test_upsearch_locations(
self,
) -> None:
locations = ReferenceCacheService.upsearch_locations("misc/jonjon/generic-data-store-area/test-level-2")
assert locations == [
"misc/jonjon/generic-data-store-area/test-level-2",
"misc/jonjon/generic-data-store-area",
"misc/jonjon",
"misc",
]
def test_can_find_data_store_in_current_location(self, with_loaded_reference_cache: None) -> None:
location = ReferenceCacheService.upsearch(
"misc/jonjon/generic-data-store-area/test-level-1", "contacts_datastore", "data_store"
)
assert location == "misc/jonjon/generic-data-store-area/test-level-1"
def test_can_find_data_store_in_upsearched_location(self, with_loaded_reference_cache: None) -> None:
location = ReferenceCacheService.upsearch(
"misc/jonjon/generic-data-store-area/test-level-2", "contacts_datastore", "data_store"
)
assert location == "misc/jonjon"
def test_does_not_find_data_store_in_non_upsearched_location(self, with_loaded_reference_cache: None) -> None:
location = ReferenceCacheService.upsearch("some/other/place", "contacts_datastore", "data_store")
assert location is None