sartography
/
spiff-arena
mirror of https://github.com/sartography/spiff-arena.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Data Store CRUD interface (#689)

This commit is contained in:
jbirddog 2023-11-24 13:27:46 -05:00 committed by GitHub
parent 55983a5aea
commit abe865c19b
10 changed files with 166 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
spiffworkflow-backend/poetry.lock generated
View File

@ -2482,21 +2482,21 @@ files = [
[[package]]
name = "typeguard"
version = "3.0.2"
version = "4.1.5"
description = "Run-time type checker for Python"
optional = false
python-versions = ">=3.7.4"
python-versions = ">=3.8"
files = [
{file = "typeguard-3.0.2-py3-none-any.whl", hash = "sha256:bbe993854385284ab42fd5bd3bee6f6556577ce8b50696d6cb956d704f286c8e"},
{file = "typeguard-3.0.2.tar.gz", hash = "sha256:fee5297fdb28f8e9efcb8142b5ee219e02375509cd77ea9d270b5af826358d5a"},
{file = "typeguard-4.1.5-py3-none-any.whl", hash = "sha256:8923e55f8873caec136c892c3bed1f676eae7be57cdb94819281b3d3bc9c0953"},
{file = "typeguard-4.1.5.tar.gz", hash = "sha256:ea0a113bbc111bcffc90789ebb215625c963411f7096a7e9062d4e4630c155fd"},
]
[package.dependencies]
typing-extensions = {version = ">=4.4.0", markers = "python_version < \"3.11\""}
typing-extensions = {version = ">=4.7.0", markers = "python_version < \"3.12\""}
[package.extras]
doc = ["packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme"]
test = ["mypy (>=0.991)", "pytest (>=7)"]
doc = ["Sphinx (>=7)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)"]
test = ["coverage[toml] (>=7)", "mypy (>=1.2.0)", "pytest (>=7)"]
[[package]]
name = "types-click"
@ -2762,4 +2762,4 @@ tests-strict = ["codecov (==2.0.15)", "pytest (==4.6.0)", "pytest (==4.6.0)", "p
[metadata]
lock-version = "2.0"
python-versions = ">=3.10,<3.12"
content-hash = "ab3695ca05af534fae5b930a40654bff67504763477e877284c7cb5afe30f847"
content-hash = "de46efa0cb4ac5980cbd4fac7ac296d6405e33ca495e05ae3140586702228354"

2
spiffworkflow-backend/pyproject.toml
View File

@ -84,7 +84,7 @@ pytest = "^7.1.2"
coverage = {extras = ["toml"], version = "^6.1"}
safety = "^2.3.5"
mypy = ">=0.961"
typeguard = "^3"
typeguard = "^4"
xdoctest = {extras = ["colors"], version = "^1.0.1"}
pre-commit = "^2.20.0"
black = ">=21.10b0"

2
spiffworkflow-backend/src/spiffworkflow_backend/config/normalized_environment.py
View File

@ -77,7 +77,7 @@ def _parse_environment(key_values: os._Environ | dict) -> list | dict:
by_first_component,
)
def items_with_first_component(items: ItemsView, first_component: str) -> dict:
def items_with_first_component(items: Iterable, first_component: str) -> dict:
return {
get_later_components(key): value for key, value in items if get_first_component(key) == first_component
}

20
spiffworkflow-backend/src/spiffworkflow_backend/data_stores/__init__.py Normal file
View File

@ -0,0 +1,20 @@
from typing import Any
from SpiffWorkflow.bpmn.specs.data_spec import BpmnDataStoreSpecification # type: ignore
from spiffworkflow_backend.data_stores.json import JSONDataStore
from spiffworkflow_backend.data_stores.json import JSONFileDataStore
from spiffworkflow_backend.data_stores.kkv import KKVDataStore
from spiffworkflow_backend.data_stores.typeahead import TypeaheadDataStore
DATA_STORES: list[BpmnDataStoreSpecification] = [
KKVDataStore,
JSONDataStore,
JSONFileDataStore,
TypeaheadDataStore,
]
def register_data_store_classes(data_store_classes: dict[str, Any]) -> None:
for ds in DATA_STORES:
ds.register_data_store_class(data_store_classes)

27
spiffworkflow-backend/src/spiffworkflow_backend/data_stores/crud.py Normal file
View File

@ -0,0 +1,27 @@
from typing import Any
class DataStoreCRUD:
@staticmethod
def existing_data_stores() -> list[dict[str, Any]]:
raise Exception("must implement")
@staticmethod
def query_data_store(name: str) -> Any:
raise Exception("must implement")
@staticmethod
def build_response_item(model: Any) -> dict[str, Any]:
raise Exception("must implement")
@staticmethod
def create_record(name: str, data: dict[str, Any]) -> None:
raise Exception("must implement")
@staticmethod
def update_record(name: str, data: dict[str, Any]) -> None:
raise Exception("must implement")
@staticmethod
def delete_record(name: str, data: dict[str, Any]) -> None:
raise Exception("must implement")

22
spiffworkflow-backend/src/spiffworkflow_backend/data_stores/json.py
View File

@ -4,6 +4,8 @@ from flask import current_app
from SpiffWorkflow.bpmn.serializer.helpers.registry import BpmnConverter # type: ignore
from SpiffWorkflow.bpmn.specs.data_spec import BpmnDataStoreSpecification # type: ignore
from SpiffWorkflow.task import Task as SpiffTask # type: ignore
from spiffworkflow_backend.data_stores.crud import DataStoreCRUD
from spiffworkflow_backend.models.db import db
from spiffworkflow_backend.models.json_data_store import JSONDataStoreModel
from spiffworkflow_backend.services.file_system_service import FileSystemService
@ -37,9 +39,27 @@ def _data_store_location_for_task(spiff_task: SpiffTask, name: str) -> str | Non
return location
class JSONDataStore(BpmnDataStoreSpecification): # type: ignore
class JSONDataStore(BpmnDataStoreSpecification, DataStoreCRUD): # type: ignore
"""JSONDataStore."""
@staticmethod
def existing_data_stores() -> list[dict[str, Any]]:
data_stores = []
keys = db.session.query(JSONDataStoreModel.name).distinct().order_by(JSONDataStoreModel.name) # type: ignore
for key in keys:
data_stores.append({"name": key[0], "type": "json"})
return data_stores
@staticmethod
def query_data_store(name: str) -> Any:
return JSONDataStoreModel.query.filter_by(name=name).order_by(JSONDataStoreModel.name)
@staticmethod
def build_response_item(model: Any) -> dict[str, Any]:
return {"location": model.location, "data": model.data}
def get(self, my_task: SpiffTask) -> None:
"""get."""
model: JSONDataStoreModel | None = None

29
spiffworkflow-backend/src/spiffworkflow_backend/data_stores/kkv.py
View File

@ -3,13 +3,40 @@ from typing import Any
from SpiffWorkflow.bpmn.serializer.helpers.registry import BpmnConverter # type: ignore
from SpiffWorkflow.bpmn.specs.data_spec import BpmnDataStoreSpecification # type: ignore
from SpiffWorkflow.task import Task as SpiffTask # type: ignore
from spiffworkflow_backend.data_stores.crud import DataStoreCRUD
from spiffworkflow_backend.models.db import db
from spiffworkflow_backend.models.kkv_data_store import KKVDataStoreModel
class KKVDataStore(BpmnDataStoreSpecification): # type: ignore
class KKVDataStore(BpmnDataStoreSpecification, DataStoreCRUD): # type: ignore
"""KKVDataStore."""
@staticmethod
def existing_data_stores() -> list[dict[str, Any]]:
data_stores = []
keys = (
db.session.query(KKVDataStoreModel.top_level_key).distinct().order_by(KKVDataStoreModel.top_level_key) # type: ignore
)
for key in keys:
data_stores.append({"name": key[0], "type": "kkv"})
return data_stores
@staticmethod
def query_data_store(name: str) -> Any:
return KKVDataStoreModel.query.filter_by(top_level_key=name).order_by(
KKVDataStoreModel.top_level_key, KKVDataStoreModel.secondary_key
)
@staticmethod
def build_response_item(model: Any) -> dict[str, Any]:
return {
"secondary_key": model.secondary_key,
"value": model.value,
}
def _get_model(self, top_level_key: str, secondary_key: str) -> KKVDataStoreModel | None:
model = (
db.session.query(KKVDataStoreModel)

26
spiffworkflow-backend/src/spiffworkflow_backend/data_stores/typeahead.py
View File

@ -4,13 +4,37 @@ from typing import Any
from SpiffWorkflow.bpmn.serializer.helpers.registry import BpmnConverter # type: ignore
from SpiffWorkflow.bpmn.specs.data_spec import BpmnDataStoreSpecification # type: ignore
from SpiffWorkflow.task import Task as SpiffTask # type: ignore
from spiffworkflow_backend.data_stores.crud import DataStoreCRUD
from spiffworkflow_backend.models.db import db
from spiffworkflow_backend.models.typeahead import TypeaheadModel
class TypeaheadDataStore(BpmnDataStoreSpecification): # type: ignore
class TypeaheadDataStore(BpmnDataStoreSpecification, DataStoreCRUD): # type: ignore
"""TypeaheadDataStore."""
@staticmethod
def existing_data_stores() -> list[dict[str, Any]]:
data_stores = []
keys = db.session.query(TypeaheadModel.category).distinct().order_by(TypeaheadModel.category) # type: ignore
for key in keys:
data_stores.append({"name": key[0], "type": "typeahead"})
return data_stores
@staticmethod
def query_data_store(name: str) -> Any:
return TypeaheadModel.query.filter_by(category=name).order_by(
TypeaheadModel.category, TypeaheadModel.search_term
)
@staticmethod
def build_response_item(model: Any) -> dict[str, Any]:
result = model.result
result["search_term"] = model.search_term
return result # type: ignore
def get(self, my_task: SpiffTask) -> None:
"""get."""
raise Exception("This is a write only data store.")

81
spiffworkflow-backend/src/spiffworkflow_backend/routes/data_store_controller.py
View File

@ -1,75 +1,60 @@
"""APIs for dealing with process groups, process models, and process instances."""
from typing import Any
import flask.wrappers
from flask import jsonify
from flask import make_response
from spiffworkflow_backend import db
from spiffworkflow_backend.data_stores.json import JSONDataStore
from spiffworkflow_backend.data_stores.kkv import KKVDataStore
from spiffworkflow_backend.data_stores.typeahead import TypeaheadDataStore
from spiffworkflow_backend.exceptions.api_error import ApiError
from spiffworkflow_backend.models.kkv_data_store import KKVDataStoreModel
from spiffworkflow_backend.models.typeahead import TypeaheadModel
def data_store_list() -> flask.wrappers.Response:
"""Returns a list of the names of all the data stores."""
data_stores = []
# Right now the only data stores we support are type ahead and kkv
# Right now the only data stores we support are type ahead, kkv, json
for cat in db.session.query(TypeaheadModel.category).distinct().order_by(TypeaheadModel.category): # type: ignore
data_stores.append({"name": cat[0], "type": "typeahead"})
keys = db.session.query(KKVDataStoreModel.top_level_key).distinct().order_by(KKVDataStoreModel.top_level_key) # type: ignore
for key in keys:
data_stores.append({"name": key[0], "type": "kkv"})
data_stores.extend(JSONDataStore.existing_data_stores())
data_stores.extend(TypeaheadDataStore.existing_data_stores())
data_stores.extend(KKVDataStore.existing_data_stores())
return make_response(jsonify(data_stores), 200)
def _build_response(data_store_class: Any, name: str, page: int, per_page: int) -> flask.wrappers.Response:
data_store_query = data_store_class.query_data_store(name)
data = data_store_query.paginate(page=page, per_page=per_page, error_out=False)
results = []
for item in data.items:
result = data_store_class.build_response_item(item)
results.append(result)
response_json = {
"results": results,
"pagination": {
"count": len(data.items),
"total": data.total,
"pages": data.pages,
},
}
return make_response(jsonify(response_json), 200)
def data_store_item_list(
data_store_type: str, name: str, page: int = 1, per_page: int = 100
) -> flask.wrappers.Response:
"""Returns a list of the items in a data store."""
if data_store_type == "typeahead":
data_store_query = TypeaheadModel.query.filter_by(category=name).order_by(
TypeaheadModel.category, TypeaheadModel.search_term
)
data = data_store_query.paginate(page=page, per_page=per_page, error_out=False)
results = []
for typeahead in data.items:
result = typeahead.result
result["search_term"] = typeahead.search_term
results.append(result)
response_json = {
"results": results,
"pagination": {
"count": len(data.items),
"total": data.total,
"pages": data.pages,
},
}
return make_response(jsonify(response_json), 200)
return _build_response(TypeaheadDataStore, name, page, per_page)
if data_store_type == "kkv":
data_store_query = KKVDataStoreModel.query.filter_by(top_level_key=name).order_by(
KKVDataStoreModel.top_level_key, KKVDataStoreModel.secondary_key
)
data = data_store_query.paginate(page=page, per_page=per_page, error_out=False)
results = []
for kkv in data.items:
result = {
"secondary_key": kkv.secondary_key,
"value": kkv.value,
}
results.append(result)
response_json = {
"results": results,
"pagination": {
"count": len(data.items),
"total": data.total,
"pages": data.pages,
},
}
return make_response(jsonify(response_json), 200)
return _build_response(KKVDataStore, name, page, per_page)
if data_store_type == "json":
return _build_response(JSONDataStore, name, page, per_page)
raise ApiError("unknown_data_store", f"Unknown data store type: {data_store_type}", status_code=400)

10
spiffworkflow-backend/src/spiffworkflow_backend/services/custom_parser.py
View File

@ -4,10 +4,7 @@ from SpiffWorkflow.bpmn.parser.BpmnParser import full_tag # type: ignore
from SpiffWorkflow.dmn.parser.BpmnDmnParser import BpmnDmnParser # type: ignore
from SpiffWorkflow.spiff.parser.process import SpiffBpmnParser # type: ignore
from SpiffWorkflow.spiff.parser.task_spec import ServiceTaskParser # type: ignore
from spiffworkflow_backend.data_stores.json import JSONDataStore
from spiffworkflow_backend.data_stores.json import JSONFileDataStore
from spiffworkflow_backend.data_stores.kkv import KKVDataStore
from spiffworkflow_backend.data_stores.typeahead import TypeaheadDataStore
from spiffworkflow_backend.data_stores import register_data_store_classes
from spiffworkflow_backend.services.service_task_service import CustomServiceTask
from spiffworkflow_backend.specs.start_event import StartEvent
@ -23,7 +20,4 @@ class MyCustomParser(BpmnDmnParser): # type: ignore
DATA_STORE_CLASSES: dict[str, Any] = {}
KKVDataStore.register_data_store_class(DATA_STORE_CLASSES)
JSONDataStore.register_data_store_class(DATA_STORE_CLASSES)
JSONFileDataStore.register_data_store_class(DATA_STORE_CLASSES)
TypeaheadDataStore.register_data_store_class(DATA_STORE_CLASSES)
register_data_store_classes(DATA_STORE_CLASSES)