From abe865c19b49283dcbd69c0d5e71e97a65e1f34b Mon Sep 17 00:00:00 2001 From: jbirddog <100367399+jbirddog@users.noreply.github.com> Date: Fri, 24 Nov 2023 13:27:46 -0500 Subject: [PATCH] Data Store CRUD interface (#689) --- spiffworkflow-backend/poetry.lock | 16 ++-- spiffworkflow-backend/pyproject.toml | 2 +- .../config/normalized_environment.py | 2 +- .../data_stores/__init__.py | 20 +++++ .../spiffworkflow_backend/data_stores/crud.py | 27 +++++++ .../spiffworkflow_backend/data_stores/json.py | 22 ++++- .../spiffworkflow_backend/data_stores/kkv.py | 29 ++++++- .../data_stores/typeahead.py | 26 +++++- .../routes/data_store_controller.py | 81 ++++++++----------- .../services/custom_parser.py | 10 +-- 10 files changed, 166 insertions(+), 69 deletions(-) create mode 100644 spiffworkflow-backend/src/spiffworkflow_backend/data_stores/__init__.py create mode 100644 spiffworkflow-backend/src/spiffworkflow_backend/data_stores/crud.py diff --git a/spiffworkflow-backend/poetry.lock b/spiffworkflow-backend/poetry.lock index 07ba439e6..a13642ac4 100644 --- a/spiffworkflow-backend/poetry.lock +++ b/spiffworkflow-backend/poetry.lock @@ -2482,21 +2482,21 @@ files = [ [[package]] name = "typeguard" -version = "3.0.2" +version = "4.1.5" description = "Run-time type checker for Python" optional = false -python-versions = ">=3.7.4" +python-versions = ">=3.8" files = [ - {file = "typeguard-3.0.2-py3-none-any.whl", hash = "sha256:bbe993854385284ab42fd5bd3bee6f6556577ce8b50696d6cb956d704f286c8e"}, - {file = "typeguard-3.0.2.tar.gz", hash = "sha256:fee5297fdb28f8e9efcb8142b5ee219e02375509cd77ea9d270b5af826358d5a"}, + {file = "typeguard-4.1.5-py3-none-any.whl", hash = "sha256:8923e55f8873caec136c892c3bed1f676eae7be57cdb94819281b3d3bc9c0953"}, + {file = "typeguard-4.1.5.tar.gz", hash = "sha256:ea0a113bbc111bcffc90789ebb215625c963411f7096a7e9062d4e4630c155fd"}, ] [package.dependencies] -typing-extensions = {version = ">=4.4.0", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.7.0", markers = "python_version < \"3.12\""} [package.extras] -doc = ["packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme"] -test = ["mypy (>=0.991)", "pytest (>=7)"] +doc = ["Sphinx (>=7)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)"] +test = ["coverage[toml] (>=7)", "mypy (>=1.2.0)", "pytest (>=7)"] [[package]] name = "types-click" @@ -2762,4 +2762,4 @@ tests-strict = ["codecov (==2.0.15)", "pytest (==4.6.0)", "pytest (==4.6.0)", "p [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.12" -content-hash = "ab3695ca05af534fae5b930a40654bff67504763477e877284c7cb5afe30f847" +content-hash = "de46efa0cb4ac5980cbd4fac7ac296d6405e33ca495e05ae3140586702228354" diff --git a/spiffworkflow-backend/pyproject.toml b/spiffworkflow-backend/pyproject.toml index 215b52854..de62e6ca3 100644 --- a/spiffworkflow-backend/pyproject.toml +++ b/spiffworkflow-backend/pyproject.toml @@ -84,7 +84,7 @@ pytest = "^7.1.2" coverage = {extras = ["toml"], version = "^6.1"} safety = "^2.3.5" mypy = ">=0.961" -typeguard = "^3" +typeguard = "^4" xdoctest = {extras = ["colors"], version = "^1.0.1"} pre-commit = "^2.20.0" black = ">=21.10b0" diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/config/normalized_environment.py b/spiffworkflow-backend/src/spiffworkflow_backend/config/normalized_environment.py index 6632c7107..9cfcb2084 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/config/normalized_environment.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/config/normalized_environment.py @@ -77,7 +77,7 @@ def _parse_environment(key_values: os._Environ | dict) -> list | dict: by_first_component, ) - def items_with_first_component(items: ItemsView, first_component: str) -> dict: + def items_with_first_component(items: Iterable, first_component: str) -> dict: return { get_later_components(key): value for key, value in items if get_first_component(key) == first_component } diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/data_stores/__init__.py b/spiffworkflow-backend/src/spiffworkflow_backend/data_stores/__init__.py new file mode 100644 index 000000000..e22c46670 --- /dev/null +++ b/spiffworkflow-backend/src/spiffworkflow_backend/data_stores/__init__.py @@ -0,0 +1,20 @@ +from typing import Any + +from SpiffWorkflow.bpmn.specs.data_spec import BpmnDataStoreSpecification # type: ignore + +from spiffworkflow_backend.data_stores.json import JSONDataStore +from spiffworkflow_backend.data_stores.json import JSONFileDataStore +from spiffworkflow_backend.data_stores.kkv import KKVDataStore +from spiffworkflow_backend.data_stores.typeahead import TypeaheadDataStore + +DATA_STORES: list[BpmnDataStoreSpecification] = [ + KKVDataStore, + JSONDataStore, + JSONFileDataStore, + TypeaheadDataStore, +] + + +def register_data_store_classes(data_store_classes: dict[str, Any]) -> None: + for ds in DATA_STORES: + ds.register_data_store_class(data_store_classes) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/data_stores/crud.py b/spiffworkflow-backend/src/spiffworkflow_backend/data_stores/crud.py new file mode 100644 index 000000000..17b1e06a5 --- /dev/null +++ b/spiffworkflow-backend/src/spiffworkflow_backend/data_stores/crud.py @@ -0,0 +1,27 @@ +from typing import Any + + +class DataStoreCRUD: + @staticmethod + def existing_data_stores() -> list[dict[str, Any]]: + raise Exception("must implement") + + @staticmethod + def query_data_store(name: str) -> Any: + raise Exception("must implement") + + @staticmethod + def build_response_item(model: Any) -> dict[str, Any]: + raise Exception("must implement") + + @staticmethod + def create_record(name: str, data: dict[str, Any]) -> None: + raise Exception("must implement") + + @staticmethod + def update_record(name: str, data: dict[str, Any]) -> None: + raise Exception("must implement") + + @staticmethod + def delete_record(name: str, data: dict[str, Any]) -> None: + raise Exception("must implement") diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/data_stores/json.py b/spiffworkflow-backend/src/spiffworkflow_backend/data_stores/json.py index ddc0c8380..c09257693 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/data_stores/json.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/data_stores/json.py @@ -4,6 +4,8 @@ from flask import current_app from SpiffWorkflow.bpmn.serializer.helpers.registry import BpmnConverter # type: ignore from SpiffWorkflow.bpmn.specs.data_spec import BpmnDataStoreSpecification # type: ignore from SpiffWorkflow.task import Task as SpiffTask # type: ignore + +from spiffworkflow_backend.data_stores.crud import DataStoreCRUD from spiffworkflow_backend.models.db import db from spiffworkflow_backend.models.json_data_store import JSONDataStoreModel from spiffworkflow_backend.services.file_system_service import FileSystemService @@ -37,9 +39,27 @@ def _data_store_location_for_task(spiff_task: SpiffTask, name: str) -> str | Non return location -class JSONDataStore(BpmnDataStoreSpecification): # type: ignore +class JSONDataStore(BpmnDataStoreSpecification, DataStoreCRUD): # type: ignore """JSONDataStore.""" + @staticmethod + def existing_data_stores() -> list[dict[str, Any]]: + data_stores = [] + + keys = db.session.query(JSONDataStoreModel.name).distinct().order_by(JSONDataStoreModel.name) # type: ignore + for key in keys: + data_stores.append({"name": key[0], "type": "json"}) + + return data_stores + + @staticmethod + def query_data_store(name: str) -> Any: + return JSONDataStoreModel.query.filter_by(name=name).order_by(JSONDataStoreModel.name) + + @staticmethod + def build_response_item(model: Any) -> dict[str, Any]: + return {"location": model.location, "data": model.data} + def get(self, my_task: SpiffTask) -> None: """get.""" model: JSONDataStoreModel | None = None diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/data_stores/kkv.py b/spiffworkflow-backend/src/spiffworkflow_backend/data_stores/kkv.py index 6977263a2..8ab0ec0eb 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/data_stores/kkv.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/data_stores/kkv.py @@ -3,13 +3,40 @@ from typing import Any from SpiffWorkflow.bpmn.serializer.helpers.registry import BpmnConverter # type: ignore from SpiffWorkflow.bpmn.specs.data_spec import BpmnDataStoreSpecification # type: ignore from SpiffWorkflow.task import Task as SpiffTask # type: ignore + +from spiffworkflow_backend.data_stores.crud import DataStoreCRUD from spiffworkflow_backend.models.db import db from spiffworkflow_backend.models.kkv_data_store import KKVDataStoreModel -class KKVDataStore(BpmnDataStoreSpecification): # type: ignore +class KKVDataStore(BpmnDataStoreSpecification, DataStoreCRUD): # type: ignore """KKVDataStore.""" + @staticmethod + def existing_data_stores() -> list[dict[str, Any]]: + data_stores = [] + + keys = ( + db.session.query(KKVDataStoreModel.top_level_key).distinct().order_by(KKVDataStoreModel.top_level_key) # type: ignore + ) + for key in keys: + data_stores.append({"name": key[0], "type": "kkv"}) + + return data_stores + + @staticmethod + def query_data_store(name: str) -> Any: + return KKVDataStoreModel.query.filter_by(top_level_key=name).order_by( + KKVDataStoreModel.top_level_key, KKVDataStoreModel.secondary_key + ) + + @staticmethod + def build_response_item(model: Any) -> dict[str, Any]: + return { + "secondary_key": model.secondary_key, + "value": model.value, + } + def _get_model(self, top_level_key: str, secondary_key: str) -> KKVDataStoreModel | None: model = ( db.session.query(KKVDataStoreModel) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/data_stores/typeahead.py b/spiffworkflow-backend/src/spiffworkflow_backend/data_stores/typeahead.py index 60e04c520..6a3f79222 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/data_stores/typeahead.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/data_stores/typeahead.py @@ -4,13 +4,37 @@ from typing import Any from SpiffWorkflow.bpmn.serializer.helpers.registry import BpmnConverter # type: ignore from SpiffWorkflow.bpmn.specs.data_spec import BpmnDataStoreSpecification # type: ignore from SpiffWorkflow.task import Task as SpiffTask # type: ignore + +from spiffworkflow_backend.data_stores.crud import DataStoreCRUD from spiffworkflow_backend.models.db import db from spiffworkflow_backend.models.typeahead import TypeaheadModel -class TypeaheadDataStore(BpmnDataStoreSpecification): # type: ignore +class TypeaheadDataStore(BpmnDataStoreSpecification, DataStoreCRUD): # type: ignore """TypeaheadDataStore.""" + @staticmethod + def existing_data_stores() -> list[dict[str, Any]]: + data_stores = [] + + keys = db.session.query(TypeaheadModel.category).distinct().order_by(TypeaheadModel.category) # type: ignore + for key in keys: + data_stores.append({"name": key[0], "type": "typeahead"}) + + return data_stores + + @staticmethod + def query_data_store(name: str) -> Any: + return TypeaheadModel.query.filter_by(category=name).order_by( + TypeaheadModel.category, TypeaheadModel.search_term + ) + + @staticmethod + def build_response_item(model: Any) -> dict[str, Any]: + result = model.result + result["search_term"] = model.search_term + return result # type: ignore + def get(self, my_task: SpiffTask) -> None: """get.""" raise Exception("This is a write only data store.") diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/routes/data_store_controller.py b/spiffworkflow-backend/src/spiffworkflow_backend/routes/data_store_controller.py index a2a3f2abd..f83ec5cd6 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/routes/data_store_controller.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/routes/data_store_controller.py @@ -1,75 +1,60 @@ """APIs for dealing with process groups, process models, and process instances.""" +from typing import Any + import flask.wrappers from flask import jsonify from flask import make_response -from spiffworkflow_backend import db +from spiffworkflow_backend.data_stores.json import JSONDataStore +from spiffworkflow_backend.data_stores.kkv import KKVDataStore +from spiffworkflow_backend.data_stores.typeahead import TypeaheadDataStore from spiffworkflow_backend.exceptions.api_error import ApiError -from spiffworkflow_backend.models.kkv_data_store import KKVDataStoreModel -from spiffworkflow_backend.models.typeahead import TypeaheadModel def data_store_list() -> flask.wrappers.Response: """Returns a list of the names of all the data stores.""" data_stores = [] - # Right now the only data stores we support are type ahead and kkv + # Right now the only data stores we support are type ahead, kkv, json - for cat in db.session.query(TypeaheadModel.category).distinct().order_by(TypeaheadModel.category): # type: ignore - data_stores.append({"name": cat[0], "type": "typeahead"}) - - keys = db.session.query(KKVDataStoreModel.top_level_key).distinct().order_by(KKVDataStoreModel.top_level_key) # type: ignore - for key in keys: - data_stores.append({"name": key[0], "type": "kkv"}) + data_stores.extend(JSONDataStore.existing_data_stores()) + data_stores.extend(TypeaheadDataStore.existing_data_stores()) + data_stores.extend(KKVDataStore.existing_data_stores()) return make_response(jsonify(data_stores), 200) +def _build_response(data_store_class: Any, name: str, page: int, per_page: int) -> flask.wrappers.Response: + data_store_query = data_store_class.query_data_store(name) + data = data_store_query.paginate(page=page, per_page=per_page, error_out=False) + results = [] + for item in data.items: + result = data_store_class.build_response_item(item) + results.append(result) + response_json = { + "results": results, + "pagination": { + "count": len(data.items), + "total": data.total, + "pages": data.pages, + }, + } + return make_response(jsonify(response_json), 200) + + def data_store_item_list( data_store_type: str, name: str, page: int = 1, per_page: int = 100 ) -> flask.wrappers.Response: """Returns a list of the items in a data store.""" + if data_store_type == "typeahead": - data_store_query = TypeaheadModel.query.filter_by(category=name).order_by( - TypeaheadModel.category, TypeaheadModel.search_term - ) - data = data_store_query.paginate(page=page, per_page=per_page, error_out=False) - results = [] - for typeahead in data.items: - result = typeahead.result - result["search_term"] = typeahead.search_term - results.append(result) - response_json = { - "results": results, - "pagination": { - "count": len(data.items), - "total": data.total, - "pages": data.pages, - }, - } - return make_response(jsonify(response_json), 200) + return _build_response(TypeaheadDataStore, name, page, per_page) if data_store_type == "kkv": - data_store_query = KKVDataStoreModel.query.filter_by(top_level_key=name).order_by( - KKVDataStoreModel.top_level_key, KKVDataStoreModel.secondary_key - ) - data = data_store_query.paginate(page=page, per_page=per_page, error_out=False) - results = [] - for kkv in data.items: - result = { - "secondary_key": kkv.secondary_key, - "value": kkv.value, - } - results.append(result) - response_json = { - "results": results, - "pagination": { - "count": len(data.items), - "total": data.total, - "pages": data.pages, - }, - } - return make_response(jsonify(response_json), 200) + return _build_response(KKVDataStore, name, page, per_page) + + if data_store_type == "json": + return _build_response(JSONDataStore, name, page, per_page) raise ApiError("unknown_data_store", f"Unknown data store type: {data_store_type}", status_code=400) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/services/custom_parser.py b/spiffworkflow-backend/src/spiffworkflow_backend/services/custom_parser.py index a4ddfd3d0..d4cd4b474 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/services/custom_parser.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/services/custom_parser.py @@ -4,10 +4,7 @@ from SpiffWorkflow.bpmn.parser.BpmnParser import full_tag # type: ignore from SpiffWorkflow.dmn.parser.BpmnDmnParser import BpmnDmnParser # type: ignore from SpiffWorkflow.spiff.parser.process import SpiffBpmnParser # type: ignore from SpiffWorkflow.spiff.parser.task_spec import ServiceTaskParser # type: ignore -from spiffworkflow_backend.data_stores.json import JSONDataStore -from spiffworkflow_backend.data_stores.json import JSONFileDataStore -from spiffworkflow_backend.data_stores.kkv import KKVDataStore -from spiffworkflow_backend.data_stores.typeahead import TypeaheadDataStore +from spiffworkflow_backend.data_stores import register_data_store_classes from spiffworkflow_backend.services.service_task_service import CustomServiceTask from spiffworkflow_backend.specs.start_event import StartEvent @@ -23,7 +20,4 @@ class MyCustomParser(BpmnDmnParser): # type: ignore DATA_STORE_CLASSES: dict[str, Any] = {} - KKVDataStore.register_data_store_class(DATA_STORE_CLASSES) - JSONDataStore.register_data_store_class(DATA_STORE_CLASSES) - JSONFileDataStore.register_data_store_class(DATA_STORE_CLASSES) - TypeaheadDataStore.register_data_store_class(DATA_STORE_CLASSES) + register_data_store_classes(DATA_STORE_CLASSES)