Data Store CRUD interface (#689)

spiffworkflow-backend/poetry.lock

@@ -2482,21 +2482,21 @@ files = [
 
 [[package]]
 name = "typeguard"
-version = "3.0.2"
+version = "4.1.5"
 description = "Run-time type checker for Python"
 optional = false
-python-versions = ">=3.7.4"
+python-versions = ">=3.8"
 files = [
-    {file = "typeguard-3.0.2-py3-none-any.whl", hash = "sha256:bbe993854385284ab42fd5bd3bee6f6556577ce8b50696d6cb956d704f286c8e"},
+    {file = "typeguard-4.1.5-py3-none-any.whl", hash = "sha256:8923e55f8873caec136c892c3bed1f676eae7be57cdb94819281b3d3bc9c0953"},
-    {file = "typeguard-3.0.2.tar.gz", hash = "sha256:fee5297fdb28f8e9efcb8142b5ee219e02375509cd77ea9d270b5af826358d5a"},
+    {file = "typeguard-4.1.5.tar.gz", hash = "sha256:ea0a113bbc111bcffc90789ebb215625c963411f7096a7e9062d4e4630c155fd"},
 ]
 
 [package.dependencies]
-typing-extensions = {version = ">=4.4.0", markers = "python_version < \"3.11\""}
+typing-extensions = {version = ">=4.7.0", markers = "python_version < \"3.12\""}
 
 [package.extras]
-doc = ["packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme"]
+doc = ["Sphinx (>=7)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)"]
-test = ["mypy (>=0.991)", "pytest (>=7)"]
+test = ["coverage[toml] (>=7)", "mypy (>=1.2.0)", "pytest (>=7)"]
 
 [[package]]
 name = "types-click"
@@ -2762,4 +2762,4 @@ tests-strict = ["codecov (==2.0.15)", "pytest (==4.6.0)", "pytest (==4.6.0)", "p
 [metadata]
 lock-version = "2.0"
 python-versions = ">=3.10,<3.12"
-content-hash = "ab3695ca05af534fae5b930a40654bff67504763477e877284c7cb5afe30f847"
+content-hash = "de46efa0cb4ac5980cbd4fac7ac296d6405e33ca495e05ae3140586702228354"

spiffworkflow-backend/pyproject.toml

@@ -84,7 +84,7 @@ pytest = "^7.1.2"
 coverage = {extras = ["toml"], version = "^6.1"}
 safety = "^2.3.5"
 mypy = ">=0.961"
-typeguard = "^3"
+typeguard = "^4"
 xdoctest = {extras = ["colors"], version = "^1.0.1"}
 pre-commit = "^2.20.0"
 black = ">=21.10b0"

spiffworkflow-backend/src/spiffworkflow_backend/config/normalized_environment.py

@@ -77,7 +77,7 @@ def _parse_environment(key_values: os._Environ | dict) -> list | dict:
             by_first_component,
         )
 
-    def items_with_first_component(items: ItemsView, first_component: str) -> dict:
+    def items_with_first_component(items: Iterable, first_component: str) -> dict:
         return {
             get_later_components(key): value for key, value in items if get_first_component(key) == first_component
         }

spiffworkflow-backend/src/spiffworkflow_backend/data_stores/__init__.py

@@ -0,0 +1,20 @@
+from typing import Any
+
+from SpiffWorkflow.bpmn.specs.data_spec import BpmnDataStoreSpecification  # type: ignore
+
+from spiffworkflow_backend.data_stores.json import JSONDataStore
+from spiffworkflow_backend.data_stores.json import JSONFileDataStore
+from spiffworkflow_backend.data_stores.kkv import KKVDataStore
+from spiffworkflow_backend.data_stores.typeahead import TypeaheadDataStore
+
+DATA_STORES: list[BpmnDataStoreSpecification] = [
+    KKVDataStore,
+    JSONDataStore,
+    JSONFileDataStore,
+    TypeaheadDataStore,
+]
+
+
+def register_data_store_classes(data_store_classes: dict[str, Any]) -> None:
+    for ds in DATA_STORES:
+        ds.register_data_store_class(data_store_classes)

spiffworkflow-backend/src/spiffworkflow_backend/data_stores/crud.py

@@ -0,0 +1,27 @@
+from typing import Any
+
+
+class DataStoreCRUD:
+    @staticmethod
+    def existing_data_stores() -> list[dict[str, Any]]:
+        raise Exception("must implement")
+
+    @staticmethod
+    def query_data_store(name: str) -> Any:
+        raise Exception("must implement")
+
+    @staticmethod
+    def build_response_item(model: Any) -> dict[str, Any]:
+        raise Exception("must implement")
+
+    @staticmethod
+    def create_record(name: str, data: dict[str, Any]) -> None:
+        raise Exception("must implement")
+
+    @staticmethod
+    def update_record(name: str, data: dict[str, Any]) -> None:
+        raise Exception("must implement")
+
+    @staticmethod
+    def delete_record(name: str, data: dict[str, Any]) -> None:
+        raise Exception("must implement")

spiffworkflow-backend/src/spiffworkflow_backend/data_stores/json.py

@@ -4,6 +4,8 @@ from flask import current_app
 from SpiffWorkflow.bpmn.serializer.helpers.registry import BpmnConverter  # type: ignore
 from SpiffWorkflow.bpmn.specs.data_spec import BpmnDataStoreSpecification  # type: ignore
 from SpiffWorkflow.task import Task as SpiffTask  # type: ignore
+
+from spiffworkflow_backend.data_stores.crud import DataStoreCRUD
 from spiffworkflow_backend.models.db import db
 from spiffworkflow_backend.models.json_data_store import JSONDataStoreModel
 from spiffworkflow_backend.services.file_system_service import FileSystemService
@@ -37,9 +39,27 @@ def _data_store_location_for_task(spiff_task: SpiffTask, name: str) -> str | Non
     return location
 
 
-class JSONDataStore(BpmnDataStoreSpecification):  # type: ignore
+class JSONDataStore(BpmnDataStoreSpecification, DataStoreCRUD):  # type: ignore
     """JSONDataStore."""
 
+    @staticmethod
+    def existing_data_stores() -> list[dict[str, Any]]:
+        data_stores = []
+
+        keys = db.session.query(JSONDataStoreModel.name).distinct().order_by(JSONDataStoreModel.name)  # type: ignore
+        for key in keys:
+            data_stores.append({"name": key[0], "type": "json"})
+
+        return data_stores
+
+    @staticmethod
+    def query_data_store(name: str) -> Any:
+        return JSONDataStoreModel.query.filter_by(name=name).order_by(JSONDataStoreModel.name)
+
+    @staticmethod
+    def build_response_item(model: Any) -> dict[str, Any]:
+        return {"location": model.location, "data": model.data}
+
     def get(self, my_task: SpiffTask) -> None:
         """get."""
         model: JSONDataStoreModel | None = None

spiffworkflow-backend/src/spiffworkflow_backend/data_stores/kkv.py

@@ -3,13 +3,40 @@ from typing import Any
 from SpiffWorkflow.bpmn.serializer.helpers.registry import BpmnConverter  # type: ignore
 from SpiffWorkflow.bpmn.specs.data_spec import BpmnDataStoreSpecification  # type: ignore
 from SpiffWorkflow.task import Task as SpiffTask  # type: ignore
+
+from spiffworkflow_backend.data_stores.crud import DataStoreCRUD
 from spiffworkflow_backend.models.db import db
 from spiffworkflow_backend.models.kkv_data_store import KKVDataStoreModel
 
 
-class KKVDataStore(BpmnDataStoreSpecification):  # type: ignore
+class KKVDataStore(BpmnDataStoreSpecification, DataStoreCRUD):  # type: ignore
     """KKVDataStore."""
 
+    @staticmethod
+    def existing_data_stores() -> list[dict[str, Any]]:
+        data_stores = []
+
+        keys = (
+            db.session.query(KKVDataStoreModel.top_level_key).distinct().order_by(KKVDataStoreModel.top_level_key)  # type: ignore
+        )
+        for key in keys:
+            data_stores.append({"name": key[0], "type": "kkv"})
+
+        return data_stores
+
+    @staticmethod
+    def query_data_store(name: str) -> Any:
+        return KKVDataStoreModel.query.filter_by(top_level_key=name).order_by(
+            KKVDataStoreModel.top_level_key, KKVDataStoreModel.secondary_key
+        )
+
+    @staticmethod
+    def build_response_item(model: Any) -> dict[str, Any]:
+        return {
+            "secondary_key": model.secondary_key,
+            "value": model.value,
+        }
+
     def _get_model(self, top_level_key: str, secondary_key: str) -> KKVDataStoreModel | None:
         model = (
             db.session.query(KKVDataStoreModel)

spiffworkflow-backend/src/spiffworkflow_backend/data_stores/typeahead.py

@@ -4,13 +4,37 @@ from typing import Any
 from SpiffWorkflow.bpmn.serializer.helpers.registry import BpmnConverter  # type: ignore
 from SpiffWorkflow.bpmn.specs.data_spec import BpmnDataStoreSpecification  # type: ignore
 from SpiffWorkflow.task import Task as SpiffTask  # type: ignore
+
+from spiffworkflow_backend.data_stores.crud import DataStoreCRUD
 from spiffworkflow_backend.models.db import db
 from spiffworkflow_backend.models.typeahead import TypeaheadModel
 
 
-class TypeaheadDataStore(BpmnDataStoreSpecification):  # type: ignore
+class TypeaheadDataStore(BpmnDataStoreSpecification, DataStoreCRUD):  # type: ignore
     """TypeaheadDataStore."""
 
+    @staticmethod
+    def existing_data_stores() -> list[dict[str, Any]]:
+        data_stores = []
+
+        keys = db.session.query(TypeaheadModel.category).distinct().order_by(TypeaheadModel.category)  # type: ignore
+        for key in keys:
+            data_stores.append({"name": key[0], "type": "typeahead"})
+
+        return data_stores
+
+    @staticmethod
+    def query_data_store(name: str) -> Any:
+        return TypeaheadModel.query.filter_by(category=name).order_by(
+            TypeaheadModel.category, TypeaheadModel.search_term
+        )
+
+    @staticmethod
+    def build_response_item(model: Any) -> dict[str, Any]:
+        result = model.result
+        result["search_term"] = model.search_term
+        return result  # type: ignore
+
     def get(self, my_task: SpiffTask) -> None:
         """get."""
         raise Exception("This is a write only data store.")

spiffworkflow-backend/src/spiffworkflow_backend/routes/data_store_controller.py

@@ -1,75 +1,60 @@
 """APIs for dealing with process groups, process models, and process instances."""
 
+from typing import Any
+
 import flask.wrappers
 from flask import jsonify
 from flask import make_response
 
-from spiffworkflow_backend import db
+from spiffworkflow_backend.data_stores.json import JSONDataStore
+from spiffworkflow_backend.data_stores.kkv import KKVDataStore
+from spiffworkflow_backend.data_stores.typeahead import TypeaheadDataStore
 from spiffworkflow_backend.exceptions.api_error import ApiError
-from spiffworkflow_backend.models.kkv_data_store import KKVDataStoreModel
-from spiffworkflow_backend.models.typeahead import TypeaheadModel
 
 
 def data_store_list() -> flask.wrappers.Response:
     """Returns a list of the names of all the data stores."""
     data_stores = []
 
-    # Right now the only data stores we support are type ahead and kkv
+    # Right now the only data stores we support are type ahead, kkv, json
 
-    for cat in db.session.query(TypeaheadModel.category).distinct().order_by(TypeaheadModel.category):  # type: ignore
+    data_stores.extend(JSONDataStore.existing_data_stores())
-        data_stores.append({"name": cat[0], "type": "typeahead"})
+    data_stores.extend(TypeaheadDataStore.existing_data_stores())
-
+    data_stores.extend(KKVDataStore.existing_data_stores())
-    keys = db.session.query(KKVDataStoreModel.top_level_key).distinct().order_by(KKVDataStoreModel.top_level_key)  # type: ignore
-    for key in keys:
-        data_stores.append({"name": key[0], "type": "kkv"})
 
     return make_response(jsonify(data_stores), 200)
 
 
+def _build_response(data_store_class: Any, name: str, page: int, per_page: int) -> flask.wrappers.Response:
+    data_store_query = data_store_class.query_data_store(name)
+    data = data_store_query.paginate(page=page, per_page=per_page, error_out=False)
+    results = []
+    for item in data.items:
+        result = data_store_class.build_response_item(item)
+        results.append(result)
+    response_json = {
+        "results": results,
+        "pagination": {
+            "count": len(data.items),
+            "total": data.total,
+            "pages": data.pages,
+        },
+    }
+    return make_response(jsonify(response_json), 200)
+
+
 def data_store_item_list(
     data_store_type: str, name: str, page: int = 1, per_page: int = 100
 ) -> flask.wrappers.Response:
     """Returns a list of the items in a data store."""
+
     if data_store_type == "typeahead":
-        data_store_query = TypeaheadModel.query.filter_by(category=name).order_by(
+        return _build_response(TypeaheadDataStore, name, page, per_page)
-            TypeaheadModel.category, TypeaheadModel.search_term
-        )
-        data = data_store_query.paginate(page=page, per_page=per_page, error_out=False)
-        results = []
-        for typeahead in data.items:
-            result = typeahead.result
-            result["search_term"] = typeahead.search_term
-            results.append(result)
-        response_json = {
-            "results": results,
-            "pagination": {
-                "count": len(data.items),
-                "total": data.total,
-                "pages": data.pages,
-            },
-        }
-        return make_response(jsonify(response_json), 200)
 
     if data_store_type == "kkv":
-        data_store_query = KKVDataStoreModel.query.filter_by(top_level_key=name).order_by(
+        return _build_response(KKVDataStore, name, page, per_page)
-            KKVDataStoreModel.top_level_key, KKVDataStoreModel.secondary_key
+
-        )
+    if data_store_type == "json":
-        data = data_store_query.paginate(page=page, per_page=per_page, error_out=False)
+        return _build_response(JSONDataStore, name, page, per_page)
-        results = []
-        for kkv in data.items:
-            result = {
-                "secondary_key": kkv.secondary_key,
-                "value": kkv.value,
-            }
-            results.append(result)
-        response_json = {
-            "results": results,
-            "pagination": {
-                "count": len(data.items),
-                "total": data.total,
-                "pages": data.pages,
-            },
-        }
-        return make_response(jsonify(response_json), 200)
 
     raise ApiError("unknown_data_store", f"Unknown data store type: {data_store_type}", status_code=400)

spiffworkflow-backend/src/spiffworkflow_backend/services/custom_parser.py

@@ -4,10 +4,7 @@ from SpiffWorkflow.bpmn.parser.BpmnParser import full_tag  # type: ignore
 from SpiffWorkflow.dmn.parser.BpmnDmnParser import BpmnDmnParser  # type: ignore
 from SpiffWorkflow.spiff.parser.process import SpiffBpmnParser  # type: ignore
 from SpiffWorkflow.spiff.parser.task_spec import ServiceTaskParser  # type: ignore
-from spiffworkflow_backend.data_stores.json import JSONDataStore
+from spiffworkflow_backend.data_stores import register_data_store_classes
-from spiffworkflow_backend.data_stores.json import JSONFileDataStore
-from spiffworkflow_backend.data_stores.kkv import KKVDataStore
-from spiffworkflow_backend.data_stores.typeahead import TypeaheadDataStore
 from spiffworkflow_backend.services.service_task_service import CustomServiceTask
 from spiffworkflow_backend.specs.start_event import StartEvent
 
@@ -23,7 +20,4 @@ class MyCustomParser(BpmnDmnParser):  # type: ignore
 
     DATA_STORE_CLASSES: dict[str, Any] = {}
 
-    KKVDataStore.register_data_store_class(DATA_STORE_CLASSES)
+    register_data_store_classes(DATA_STORE_CLASSES)
-    JSONDataStore.register_data_store_class(DATA_STORE_CLASSES)
-    JSONFileDataStore.register_data_store_class(DATA_STORE_CLASSES)
-    TypeaheadDataStore.register_data_store_class(DATA_STORE_CLASSES)