diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/data_stores/json.py b/spiffworkflow-backend/src/spiffworkflow_backend/data_stores/json.py index fd2035eb..6f7223c4 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/data_stores/json.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/data_stores/json.py @@ -6,6 +6,7 @@ from SpiffWorkflow.bpmn.specs.data_spec import BpmnDataStoreSpecification # typ from SpiffWorkflow.task import Task as SpiffTask # type: ignore from spiffworkflow_backend.models.db import db from spiffworkflow_backend.models.json_data_store import JSONDataStoreModel +from spiffworkflow_backend.services.file_system_service import FileSystemService def _process_model_location_for_task(spiff_task: SpiffTask) -> str | None: @@ -15,6 +16,14 @@ def _process_model_location_for_task(spiff_task: SpiffTask) -> str | None: return None +def _data_store_filename(name: str) -> str: + return f"{name}.json" + + +def _data_store_exists_at_location(location: str, name: str) -> bool: + return FileSystemService.file_exists_at_relative_path(location, _data_store_filename(name)) + + class JSONDataStore(BpmnDataStoreSpecification): # type: ignore """JSONDataStore.""" @@ -22,7 +31,7 @@ class JSONDataStore(BpmnDataStoreSpecification): # type: ignore """get.""" model: JSONDataStoreModel | None = None location = _process_model_location_for_task(my_task) - if location is not None: + if location is not None and _data_store_exists_at_location(location, self.bpmn_id): model = db.session.query(JSONDataStoreModel).filter_by(name=self.bpmn_id, location=location).first() if model is None: raise Exception(f"Unable to read from data store '{self.bpmn_id}' using location '{location}'.") @@ -31,7 +40,7 @@ class JSONDataStore(BpmnDataStoreSpecification): # type: ignore def set(self, my_task: SpiffTask) -> None: """set.""" location = _process_model_location_for_task(my_task) - if location is None: + if location is None or not _data_store_exists_at_location(location, self.bpmn_id): raise Exception(f"Unable to write to data store '{self.bpmn_id}' using location '{location}'.") data = my_task.data[self.bpmn_id] model = JSONDataStoreModel( @@ -73,3 +82,55 @@ class JSONDataStoreConverter(BpmnSpecConverter): # type: ignore def from_dict(self, dct: dict[str, Any]) -> JSONDataStore: """from_dict.""" return JSONDataStore(**dct) + + +class JSONFileDataStore(BpmnDataStoreSpecification): # type: ignore + """JSONFileDataStore.""" + + def get(self, my_task: SpiffTask) -> None: + """get.""" + location = _process_model_location_for_task(my_task) + if location is None or not _data_store_exists_at_location(location, self.bpmn_id): + raise Exception(f"Unable to read from data store '{self.bpmn_id}' using location '{location}'.") + contents = FileSystemService.contents_of_json_file_at_relative_path( + location, _data_store_filename(self.bpmn_id) + ) + my_task.data[self.bpmn_id] = contents + + def set(self, my_task: SpiffTask) -> None: + """set.""" + location = _process_model_location_for_task(my_task) + if location is None or not _data_store_exists_at_location(location, self.bpmn_id): + raise Exception(f"Unable to write to data store '{self.bpmn_id}' using location '{location}'.") + data = my_task.data[self.bpmn_id] + FileSystemService.write_to_json_file_at_relative_path(location, _data_store_filename(self.bpmn_id), data) + del my_task.data[self.bpmn_id] + + @staticmethod + def register_converter(spec_config: dict[str, Any]) -> None: + spec_config["task_specs"].append(JSONFileDataStoreConverter) + + @staticmethod + def register_data_store_class(data_store_classes: dict[str, Any]) -> None: + data_store_classes["JSONFileDataStore"] = JSONFileDataStore + + +class JSONFileDataStoreConverter(BpmnSpecConverter): # type: ignore + """JSONFileDataStoreConverter.""" + + def __init__(self, registry): # type: ignore + """__init__.""" + super().__init__(JSONFileDataStore, registry) + + def to_dict(self, spec: Any) -> dict[str, Any]: + """to_dict.""" + return { + "bpmn_id": spec.bpmn_id, + "bpmn_name": spec.bpmn_name, + "capacity": spec.capacity, + "is_unlimited": spec.is_unlimited, + } + + def from_dict(self, dct: dict[str, Any]) -> JSONFileDataStore: + """from_dict.""" + return JSONFileDataStore(**dct) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/services/custom_parser.py b/spiffworkflow-backend/src/spiffworkflow_backend/services/custom_parser.py index 27deba44..f2ca25bf 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/services/custom_parser.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/services/custom_parser.py @@ -3,6 +3,7 @@ from typing import Any from SpiffWorkflow.dmn.parser.BpmnDmnParser import BpmnDmnParser # type: ignore from SpiffWorkflow.spiff.parser.process import SpiffBpmnParser # type: ignore from spiffworkflow_backend.data_stores.json import JSONDataStore +from spiffworkflow_backend.data_stores.json import JSONFileDataStore from spiffworkflow_backend.data_stores.typeahead import TypeaheadDataStore from spiffworkflow_backend.specs.start_event import StartEvent @@ -18,4 +19,5 @@ class MyCustomParser(BpmnDmnParser): # type: ignore DATA_STORE_CLASSES: dict[str, Any] = {} JSONDataStore.register_data_store_class(DATA_STORE_CLASSES) + JSONFileDataStore.register_data_store_class(DATA_STORE_CLASSES) TypeaheadDataStore.register_data_store_class(DATA_STORE_CLASSES) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/services/file_system_service.py b/spiffworkflow-backend/src/spiffworkflow_backend/services/file_system_service.py index 6feb9965..6df87b7d 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/services/file_system_service.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/services/file_system_service.py @@ -1,7 +1,9 @@ +import json import os from collections.abc import Generator from contextlib import contextmanager from datetime import datetime +from typing import Any import pytz from flask import current_app @@ -99,6 +101,32 @@ class FileSystemService: def full_path_from_relative_path(relative_path: str) -> str: return os.path.join(FileSystemService.root_path(), relative_path) + @classmethod + def file_exists_at_relative_path(cls, relative_path: str, file_name: str) -> bool: + full_path = cls.full_path_from_relative_path(os.path.join(relative_path, file_name)) + return os.path.isfile(full_path) + + @classmethod + def contents_of_file_at_relative_path(cls, relative_path: str, file_name: str) -> str: + full_path = cls.full_path_from_relative_path(os.path.join(relative_path, file_name)) + with open(full_path) as f: + return f.read() + + @classmethod + def contents_of_json_file_at_relative_path(cls, relative_path: str, file_name: str) -> Any: + contents = cls.contents_of_file_at_relative_path(relative_path, file_name) + return json.loads(contents) + + @classmethod + def write_to_file_at_relative_path(cls, relative_path: str, file_name: str, contents: str) -> None: + full_path = cls.full_path_from_relative_path(os.path.join(relative_path, file_name)) + with open(full_path, "w") as f: + f.write(contents) + + @classmethod + def write_to_json_file_at_relative_path(cls, relative_path: str, file_name: str, contents: Any) -> None: + cls.write_to_file_at_relative_path(relative_path, file_name, json.dumps(contents, indent=4, sort_keys=True)) + @staticmethod def process_model_relative_path(process_model: ProcessModelInfo) -> str: """Get the file path to a process model relative to SPIFFWORKFLOW_BACKEND_BPMN_SPEC_ABSOLUTE_DIR. diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py b/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py index 91ae3659..0d37f8fc 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py @@ -44,6 +44,7 @@ from SpiffWorkflow.task import Task as SpiffTask # type: ignore from SpiffWorkflow.task import TaskState from SpiffWorkflow.util.deep_merge import DeepMerge # type: ignore from spiffworkflow_backend.data_stores.json import JSONDataStore +from spiffworkflow_backend.data_stores.json import JSONFileDataStore from spiffworkflow_backend.data_stores.typeahead import TypeaheadDataStore from spiffworkflow_backend.exceptions.api_error import ApiError from spiffworkflow_backend.models.bpmn_process import BpmnProcessModel @@ -93,6 +94,7 @@ from sqlalchemy import and_ StartEvent.register_converter(SPIFF_SPEC_CONFIG) JSONDataStore.register_converter(SPIFF_SPEC_CONFIG) +JSONFileDataStore.register_converter(SPIFF_SPEC_CONFIG) TypeaheadDataStore.register_converter(SPIFF_SPEC_CONFIG) # Sorry about all this crap. I wanted to move this thing to another file, but diff --git a/spiffworkflow-backend/tests/data/json_file_data_store/contacts_datastore.json b/spiffworkflow-backend/tests/data/json_file_data_store/contacts_datastore.json new file mode 100644 index 00000000..fe51488c --- /dev/null +++ b/spiffworkflow-backend/tests/data/json_file_data_store/contacts_datastore.json @@ -0,0 +1 @@ +[] diff --git a/spiffworkflow-backend/tests/data/json_file_data_store/load.bpmn b/spiffworkflow-backend/tests/data/json_file_data_store/load.bpmn new file mode 100644 index 00000000..9a19e447 --- /dev/null +++ b/spiffworkflow-backend/tests/data/json_file_data_store/load.bpmn @@ -0,0 +1,85 @@ + + + + + + Flow_15zp7wu + + + + Flow_0citdoo + + + + Flow_15zp7wu + Flow_114y4md + + DataStoreReference_1b40zg5 + + contacts_datastore = [ + {"contact": "Joe Bob", "company": "Some Job", "email": "joebob@email.ai"}, + {"contact": "Sue Smith", "company": "Some Job", "email": "sue@email.ai", "notes": "Decision Maker\nDoes'nt answer emails."}, + {"contact": "Some Person", "company": "Another Job", "email": "person@test.com"}, + {"contact": "Them Person", "company": "Them Company", "email": "them@test.com"}, +] + + + + + Flow_114y4md + Flow_0citdoo + + + DataStoreReference_1nsdav3 + Property_1pbyq4i + + x = contacts_datastore[1] + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/spiffworkflow-backend/tests/data/json_file_data_store/process_model.json b/spiffworkflow-backend/tests/data/json_file_data_store/process_model.json new file mode 100644 index 00000000..4f264c70 --- /dev/null +++ b/spiffworkflow-backend/tests/data/json_file_data_store/process_model.json @@ -0,0 +1,9 @@ +{ + "description": "", + "display_name": "Test Level 1", + "exception_notification_addresses": [], + "fault_or_suspend_on_exception": "fault", + "metadata_extraction_paths": null, + "primary_file_name": "load.bpmn", + "primary_process_id": "Process_fil0r1s" +} diff --git a/spiffworkflow-backend/tests/spiffworkflow_backend/integration/test_json_file_data_store.py b/spiffworkflow-backend/tests/spiffworkflow_backend/integration/test_json_file_data_store.py new file mode 100644 index 00000000..6c294170 --- /dev/null +++ b/spiffworkflow-backend/tests/spiffworkflow_backend/integration/test_json_file_data_store.py @@ -0,0 +1,35 @@ +from flask.app import Flask +from flask.testing import FlaskClient +from spiffworkflow_backend.models.user import UserModel +from spiffworkflow_backend.services.process_instance_processor import ProcessInstanceProcessor + +from tests.spiffworkflow_backend.helpers.base_test import BaseTest +from tests.spiffworkflow_backend.helpers.test_data import load_test_spec + + +class TestJSONFileDataStore(BaseTest): + def test_can_execute_diagram( + self, + app: Flask, + client: FlaskClient, + with_db_and_bpmn_file_cleanup: None, + with_super_admin_user: UserModel, + ) -> None: + process_model = load_test_spec( + process_model_id="tests/data/json_file_data_store", + process_model_source_directory="json_file_data_store", + ) + process_instance = self.create_process_instance_from_process_model(process_model=process_model) + processor = ProcessInstanceProcessor(process_instance) + processor.do_engine_steps() + + assert "x" in processor.bpmn_process_instance.data + + result = processor.bpmn_process_instance.data["x"] + + assert result == { + "company": "Some Job", + "contact": "Sue Smith", + "email": "sue@email.ai", + "notes": "Decision Maker\nDoes'nt answer emails.", + }