From 305a6fb67ad1fc4a5888587895146426aea0b75b Mon Sep 17 00:00:00 2001 From: jbirddog <100367399+jbirddog@users.noreply.github.com> Date: Wed, 7 Feb 2024 09:01:52 -0500 Subject: [PATCH] KKV Revamp (#968) --- .../migrations/versions/60c13a48d675_.py | 47 +++++++ .../migrations/versions/6344d90d20fa_.py | 70 +++++++++ .../src/spiffworkflow_backend/api.yml | 12 +- .../spiffworkflow_backend/data_stores/crud.py | 39 +++++ .../spiffworkflow_backend/data_stores/json.py | 55 ++------ .../spiffworkflow_backend/data_stores/kkv.py | 133 +++++++++++++----- .../load_database_models.py | 3 + .../models/kkv_data_store.py | 18 ++- .../models/kkv_data_store_entry.py | 25 ++++ .../routes/data_store_controller.py | 18 ++- .../create_unique_kkv_top_level_key.py | 20 ++- .../unit/test_kkv_data_store.py | 54 ++++--- .../src/components/DataStoreListTable.tsx | 28 ++-- .../src/components/DataStoreListTiles.tsx | 2 +- spiffworkflow-frontend/src/interfaces.ts | 1 + .../src/routes/DataStoreEdit.tsx | 5 +- 16 files changed, 407 insertions(+), 123 deletions(-) create mode 100644 spiffworkflow-backend/migrations/versions/60c13a48d675_.py create mode 100644 spiffworkflow-backend/migrations/versions/6344d90d20fa_.py create mode 100644 spiffworkflow-backend/src/spiffworkflow_backend/models/kkv_data_store_entry.py diff --git a/spiffworkflow-backend/migrations/versions/60c13a48d675_.py b/spiffworkflow-backend/migrations/versions/60c13a48d675_.py new file mode 100644 index 000000000..78ca1517d --- /dev/null +++ b/spiffworkflow-backend/migrations/versions/60c13a48d675_.py @@ -0,0 +1,47 @@ +"""empty message + +Revision ID: 60c13a48d675 +Revises: 29b261f5edf4 +Create Date: 2024-02-01 08:43:01.666683 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import mysql + +# revision identifiers, used by Alembic. +revision = '60c13a48d675' +down_revision = '29b261f5edf4' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + # with op.batch_alter_table('kkv_data_store', schema=None) as batch_op: + # batch_op.drop_index('ix_kkv_data_store_secondary_key') + # batch_op.drop_index('ix_kkv_data_store_top_level_key') + + op.drop_table('kkv_data_store') + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('kkv_data_store', + sa.Column('id', mysql.INTEGER(), autoincrement=True, nullable=False), + sa.Column('top_level_key', mysql.VARCHAR(length=255), nullable=False), + sa.Column('secondary_key', mysql.VARCHAR(length=255), nullable=False), + sa.Column('value', mysql.JSON(), nullable=False), + sa.Column('updated_at_in_seconds', mysql.INTEGER(), autoincrement=False, nullable=False), + sa.Column('created_at_in_seconds', mysql.INTEGER(), autoincrement=False, nullable=False), + sa.PrimaryKeyConstraint('id'), + mysql_collate='utf8mb4_0900_ai_ci', + mysql_default_charset='utf8mb4', + mysql_engine='InnoDB' + ) + with op.batch_alter_table('kkv_data_store', schema=None) as batch_op: + batch_op.create_index('ix_kkv_data_store_top_level_key', ['top_level_key'], unique=False) + batch_op.create_index('ix_kkv_data_store_secondary_key', ['secondary_key'], unique=False) + + # ### end Alembic commands ### diff --git a/spiffworkflow-backend/migrations/versions/6344d90d20fa_.py b/spiffworkflow-backend/migrations/versions/6344d90d20fa_.py new file mode 100644 index 000000000..70e250c40 --- /dev/null +++ b/spiffworkflow-backend/migrations/versions/6344d90d20fa_.py @@ -0,0 +1,70 @@ +"""empty message + +Revision ID: 6344d90d20fa +Revises: 60c13a48d675 +Create Date: 2024-02-02 08:33:49.669497 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '6344d90d20fa' +down_revision = '60c13a48d675' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('kkv_data_store', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.String(length=255), nullable=False), + sa.Column('identifier', sa.String(length=255), nullable=False), + sa.Column('location', sa.String(length=255), nullable=False), + sa.Column('schema', sa.JSON(), nullable=False), + sa.Column('description', sa.String(length=255), nullable=True), + sa.Column('updated_at_in_seconds', sa.Integer(), nullable=False), + sa.Column('created_at_in_seconds', sa.Integer(), nullable=False), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('identifier', 'location', name='_kkv_identifier_location_unique') + ) + with op.batch_alter_table('kkv_data_store', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_kkv_data_store_identifier'), ['identifier'], unique=False) + batch_op.create_index(batch_op.f('ix_kkv_data_store_name'), ['name'], unique=False) + + op.create_table('kkv_data_store_entry', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('kkv_data_store_id', sa.Integer(), nullable=False), + sa.Column('top_level_key', sa.String(length=255), nullable=False), + sa.Column('secondary_key', sa.String(length=255), nullable=False), + sa.Column('value', sa.JSON(), nullable=False), + sa.Column('updated_at_in_seconds', sa.Integer(), nullable=False), + sa.Column('created_at_in_seconds', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['kkv_data_store_id'], ['kkv_data_store.id'], ), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('kkv_data_store_id', 'top_level_key', 'secondary_key', name='_instance_keys_unique') + ) + with op.batch_alter_table('kkv_data_store_entry', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_kkv_data_store_entry_kkv_data_store_id'), ['kkv_data_store_id'], unique=False) + batch_op.create_index(batch_op.f('ix_kkv_data_store_entry_secondary_key'), ['secondary_key'], unique=False) + batch_op.create_index(batch_op.f('ix_kkv_data_store_entry_top_level_key'), ['top_level_key'], unique=False) + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('kkv_data_store_entry', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_kkv_data_store_entry_top_level_key')) + batch_op.drop_index(batch_op.f('ix_kkv_data_store_entry_secondary_key')) + batch_op.drop_index(batch_op.f('ix_kkv_data_store_entry_kkv_data_store_id')) + + op.drop_table('kkv_data_store_entry') + with op.batch_alter_table('kkv_data_store', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_kkv_data_store_name')) + batch_op.drop_index(batch_op.f('ix_kkv_data_store_identifier')) + + op.drop_table('kkv_data_store') + # ### end Alembic commands ### diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/api.yml b/spiffworkflow-backend/src/spiffworkflow_backend/api.yml index 1f653afac..5cfd234b1 100755 --- a/spiffworkflow-backend/src/spiffworkflow_backend/api.yml +++ b/spiffworkflow-backend/src/spiffworkflow_backend/api.yml @@ -2821,7 +2821,7 @@ paths: responses: "200": description: The list of currently defined data store types - /data-stores/{data_store_type}/{name}/items: + /data-stores/{data_store_type}/{identifier}/items: parameters: - name: data_store_type in: path @@ -2829,10 +2829,16 @@ paths: description: The type of datastore, such as "typeahead" schema: type: string - - name: name + - name: identifier in: path required: true - description: The name of the datastore, such as "cities" + description: The identifier of the datastore, such as "cities" + schema: + type: string + - name: location + in: query + required: false + description: The location of the data store schema: type: string - name: page diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/data_stores/crud.py b/spiffworkflow-backend/src/spiffworkflow_backend/data_stores/crud.py index 9721ce470..fe1c60c6b 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/data_stores/crud.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/data_stores/crud.py @@ -1,5 +1,18 @@ from typing import Any +from flask import current_app +from SpiffWorkflow.task import Task as SpiffTask # type: ignore + +from spiffworkflow_backend.services.upsearch_service import UpsearchService + + +class DataStoreReadError(Exception): + pass + + +class DataStoreWriteError(Exception): + pass + class DataStoreCRUD: @staticmethod @@ -33,3 +46,29 @@ class DataStoreCRUD: @staticmethod def delete_record(name: str, data: dict[str, Any]) -> None: raise Exception("must implement") + + @staticmethod + def process_model_location_for_task(spiff_task: SpiffTask) -> str | None: + tld = current_app.config.get("THREAD_LOCAL_DATA") + if tld and hasattr(tld, "process_model_identifier"): + return tld.process_model_identifier # type: ignore + return None + + @classmethod + def data_store_location_for_task(cls, model: Any, spiff_task: SpiffTask, identifier: str) -> str | None: + location = cls.process_model_location_for_task(spiff_task) + if location is None: + return None + + locations = UpsearchService.upsearch_locations(location) + model = ( + model.query.filter_by(identifier=identifier) + .filter(model.location.in_(locations)) + .order_by(model.location.desc()) + .first() + ) + + if model is None: + return None + + return model.location # type: ignore diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/data_stores/json.py b/spiffworkflow-backend/src/spiffworkflow_backend/data_stores/json.py index 5b069e088..0eaac672d 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/data_stores/json.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/data_stores/json.py @@ -1,32 +1,17 @@ from typing import Any import jsonschema # type: ignore -from flask import current_app from SpiffWorkflow.bpmn.serializer.helpers import BpmnConverter # type: ignore from SpiffWorkflow.bpmn.specs.data_spec import BpmnDataStoreSpecification # type: ignore from SpiffWorkflow.task import Task as SpiffTask # type: ignore from spiffworkflow_backend.data_stores.crud import DataStoreCRUD +from spiffworkflow_backend.data_stores.crud import DataStoreReadError +from spiffworkflow_backend.data_stores.crud import DataStoreWriteError from spiffworkflow_backend.models.db import db from spiffworkflow_backend.models.json_data_store import JSONDataStoreModel from spiffworkflow_backend.services.file_system_service import FileSystemService from spiffworkflow_backend.services.reference_cache_service import ReferenceCacheService -from spiffworkflow_backend.services.upsearch_service import UpsearchService - - -class DataStoreReadError(Exception): - pass - - -class DataStoreWriteError(Exception): - pass - - -def _process_model_location_for_task(spiff_task: SpiffTask) -> str | None: - tld = current_app.config.get("THREAD_LOCAL_DATA") - if tld and hasattr(tld, "process_model_identifier"): - return tld.process_model_identifier # type: ignore - return None class JSONDataStore(BpmnDataStoreSpecification, DataStoreCRUD): # type: ignore @@ -48,12 +33,12 @@ class JSONDataStore(BpmnDataStoreSpecification, DataStoreCRUD): # type: ignore def existing_data_stores(process_group_identifier: str | None = None) -> list[dict[str, Any]]: data_stores = [] - query = db.session.query(JSONDataStoreModel.name, JSONDataStoreModel.identifier) + query = db.session.query(JSONDataStoreModel.name, JSONDataStoreModel.identifier, JSONDataStoreModel.location) if process_group_identifier is not None: query = query.filter_by(location=process_group_identifier) - keys = query.distinct().order_by(JSONDataStoreModel.name).all() # type: ignore + keys = query.order_by(JSONDataStoreModel.name).all() for key in keys: - data_stores.append({"name": key[0], "type": "json", "id": key[1], "clz": "JSONDataStore"}) + data_stores.append({"name": key[0], "type": "json", "id": key[1], "clz": "JSONDataStore", "location": key[2]}) return data_stores @@ -68,26 +53,28 @@ class JSONDataStore(BpmnDataStoreSpecification, DataStoreCRUD): # type: ignore @staticmethod def build_response_item(model: Any) -> dict[str, Any]: - return {"location": model.location, "identifier": model.identifier, "data": model.data} + return {"data": model.data} def get(self, my_task: SpiffTask) -> None: """get.""" model: JSONDataStoreModel | None = None - location = self._data_store_location_for_task(my_task, self.bpmn_id) + location = self.data_store_location_for_task(JSONDataStoreModel, my_task, self.bpmn_id) + if location is not None: model = db.session.query(JSONDataStoreModel).filter_by(identifier=self.bpmn_id, location=location).first() if model is None: raise DataStoreReadError(f"Unable to read from data store '{self.bpmn_id}' using location '{location}'.") + my_task.data[self.bpmn_id] = model.data def set(self, my_task: SpiffTask) -> None: """set.""" model: JSONDataStoreModel | None = None - location = self._data_store_location_for_task(my_task, self.bpmn_id) + location = self.data_store_location_for_task(JSONDataStoreModel, my_task, self.bpmn_id) if location is not None: model = JSONDataStoreModel.query.filter_by(identifier=self.bpmn_id, location=location).first() - if location is None or model is None: + if model is None: raise DataStoreWriteError(f"Unable to write to data store '{self.bpmn_id}' using location '{location}'.") data = my_task.data[self.bpmn_id] @@ -105,24 +92,6 @@ class JSONDataStore(BpmnDataStoreSpecification, DataStoreCRUD): # type: ignore db.session.commit() del my_task.data[self.bpmn_id] - def _data_store_location_for_task(self, spiff_task: SpiffTask, identifier: str) -> str | None: - location = _process_model_location_for_task(spiff_task) - if location is None: - return None - - locations = UpsearchService.upsearch_locations(location) - model = ( - JSONDataStoreModel.query.filter_by(identifier=identifier) - .filter(JSONDataStoreModel.location.in_(locations)) # type: ignore - .order_by(JSONDataStoreModel.location.desc()) # type: ignore - .first() - ) - - if model is None: - return None - - return model.location # type: ignore - @staticmethod def register_data_store_class(data_store_classes: dict[str, Any]) -> None: data_store_classes["JSONDataStore"] = JSONDataStore @@ -166,7 +135,7 @@ class JSONFileDataStore(BpmnDataStoreSpecification): # type: ignore del my_task.data[self.bpmn_id] def _data_store_location_for_task(self, spiff_task: SpiffTask, identifier: str) -> str | None: - location = _process_model_location_for_task(spiff_task) + location = DataStoreCRUD.process_model_location_for_task(spiff_task) if location is None: return None if self._data_store_exists_at_location(location, identifier): diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/data_stores/kkv.py b/spiffworkflow-backend/src/spiffworkflow_backend/data_stores/kkv.py index 2c384e181..a522d1756 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/data_stores/kkv.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/data_stores/kkv.py @@ -1,63 +1,92 @@ from typing import Any +import jsonschema # type: ignore from SpiffWorkflow.bpmn.serializer.helpers import BpmnConverter # type: ignore from SpiffWorkflow.bpmn.specs.data_spec import BpmnDataStoreSpecification # type: ignore from SpiffWorkflow.task import Task as SpiffTask # type: ignore from spiffworkflow_backend.data_stores.crud import DataStoreCRUD +from spiffworkflow_backend.data_stores.crud import DataStoreReadError +from spiffworkflow_backend.data_stores.crud import DataStoreWriteError from spiffworkflow_backend.models.db import db from spiffworkflow_backend.models.kkv_data_store import KKVDataStoreModel +from spiffworkflow_backend.models.kkv_data_store_entry import KKVDataStoreEntryModel class KKVDataStore(BpmnDataStoreSpecification, DataStoreCRUD): # type: ignore """KKVDataStore.""" @staticmethod - def existing_data_stores(process_group_identifier: str | None = None) -> list[dict[str, Any]]: - data_stores: list[dict[str, Any]] = [] - - if process_group_identifier is not None: - # temporary until this data store gets location support - return data_stores - - keys = ( - db.session.query(KKVDataStoreModel.top_level_key) - .distinct() # type: ignore - .order_by(KKVDataStoreModel.top_level_key) - .all() + def create_instance(identifier: str, location: str) -> Any: + return KKVDataStoreModel( + identifier=identifier, + location=location, ) - for key in keys: - data_stores.append({"name": key[0], "type": "kkv", "id": "", "clz": "KKVDataStore"}) + + @staticmethod + def existing_instance(identifier: str, location: str) -> Any: + return db.session.query(KKVDataStoreModel).filter_by(identifier=identifier, location=location).first() + + @staticmethod + def existing_data_stores(process_group_identifier: str | None = None) -> list[dict[str, Any]]: + data_stores = [] + + query = db.session.query(KKVDataStoreModel) + if process_group_identifier is not None: + query = query.filter_by(location=process_group_identifier) + models = query.order_by(KKVDataStoreModel.name).all() + for model in models: + data_stores.append( + {"name": model.name, "type": "kkv", "id": model.identifier, "clz": "KKVDataStore", "location": model.location} + ) return data_stores @staticmethod - def get_data_store_query(name: str, process_group_identifier: str | None) -> Any: - return KKVDataStoreModel.query.filter_by(top_level_key=name).order_by( - KKVDataStoreModel.top_level_key, KKVDataStoreModel.secondary_key - ) + def get_data_store_query(identifier: str, process_group_identifier: str | None) -> Any: + query = KKVDataStoreModel.query + if process_group_identifier is not None: + query = query.filter_by(identifier=identifier, location=process_group_identifier) + else: + query = query.filter_by(name=identifier) + return query.order_by(KKVDataStoreModel.name) @staticmethod def build_response_item(model: Any) -> dict[str, Any]: + data = [] + + for entry in model.entries: + data.append( + { + "top_level_key": entry.top_level_key, + "secondary_key": entry.secondary_key, + "value": entry.value, + } + ) + return { - "secondary_key": model.secondary_key, - "value": model.value, + "data": data, } - def _get_model(self, top_level_key: str, secondary_key: str) -> KKVDataStoreModel | None: - model = db.session.query(KKVDataStoreModel).filter_by(top_level_key=top_level_key, secondary_key=secondary_key).first() - return model - - def _delete_all_for_top_level_key(self, top_level_key: str) -> None: - models = db.session.query(KKVDataStoreModel).filter_by(top_level_key=top_level_key).all() - for model in models: - db.session.delete(model) - def get(self, my_task: SpiffTask) -> None: """get.""" def getter(top_level_key: str, secondary_key: str) -> Any | None: - model = self._get_model(top_level_key, secondary_key) + location = self.data_store_location_for_task(KKVDataStoreModel, my_task, self.bpmn_id) + store_model: KKVDataStoreModel | None = None + + if location is not None: + store_model = db.session.query(KKVDataStoreModel).filter_by(identifier=self.bpmn_id, location=location).first() + + if store_model is None: + raise DataStoreReadError(f"Unable to locate kkv data store '{self.bpmn_id}'.") + + model = ( + db.session.query(KKVDataStoreEntryModel) + .filter_by(kkv_data_store_id=store_model.id, top_level_key=top_level_key, secondary_key=secondary_key) + .first() + ) + if model is not None: return model.value return None @@ -66,29 +95,63 @@ class KKVDataStore(BpmnDataStoreSpecification, DataStoreCRUD): # type: ignore def set(self, my_task: SpiffTask) -> None: """set.""" + location = self.data_store_location_for_task(KKVDataStoreModel, my_task, self.bpmn_id) + store_model: KKVDataStoreModel | None = None + + if location is not None: + store_model = db.session.query(KKVDataStoreModel).filter_by(identifier=self.bpmn_id, location=location).first() + + if store_model is None: + raise DataStoreWriteError(f"Unable to locate kkv data store '{self.bpmn_id}'.") + data = my_task.data[self.bpmn_id] + if not isinstance(data, dict): - raise Exception( + raise DataStoreWriteError( f"When writing to this data store, a dictionary is expected as the value for variable '{self.bpmn_id}'" ) for top_level_key, second_level in data.items(): if second_level is None: - self._delete_all_for_top_level_key(top_level_key) + models = ( + db.session.query(KKVDataStoreEntryModel) + .filter_by(kkv_data_store_id=store_model.id, top_level_key=top_level_key) + .all() + ) + for model_to_delete in models: + db.session.delete(model_to_delete) continue if not isinstance(second_level, dict): - raise Exception( + raise DataStoreWriteError( "When writing to this data store, a dictionary is expected as the value for" f" '{self.bpmn_id}[\"{top_level_key}\"]'" ) for secondary_key, value in second_level.items(): - model = self._get_model(top_level_key, secondary_key) + model = ( + db.session.query(KKVDataStoreEntryModel) + .filter_by(kkv_data_store_id=store_model.id, top_level_key=top_level_key, secondary_key=secondary_key) + .first() + ) + if model is None and value is None: continue if value is None: db.session.delete(model) continue + + try: + jsonschema.validate(instance=value, schema=store_model.schema) + except jsonschema.exceptions.ValidationError as e: + raise DataStoreWriteError( + f"Attempting to write data that does not match the provided schema for '{self.bpmn_id}': {e}" + ) from e + if model is None: - model = KKVDataStoreModel(top_level_key=top_level_key, secondary_key=secondary_key, value=value) + model = KKVDataStoreEntryModel( + kkv_data_store_id=store_model.id, + top_level_key=top_level_key, + secondary_key=secondary_key, + value=value, + ) else: model.value = value db.session.add(model) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/load_database_models.py b/spiffworkflow-backend/src/spiffworkflow_backend/load_database_models.py index 9893239e3..ee340047a 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/load_database_models.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/load_database_models.py @@ -91,6 +91,9 @@ from spiffworkflow_backend.models.json_data_store import ( from spiffworkflow_backend.models.kkv_data_store import ( KKVDataStoreModel, ) # noqa: F401 +from spiffworkflow_backend.models.kkv_data_store_entry import ( + KKVDataStoreEntryModel, +) # noqa: F401 from spiffworkflow_backend.models.task_draft_data import ( TaskDraftDataModel, ) # noqa: F401 diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/kkv_data_store.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/kkv_data_store.py index 7a9583578..116dbb126 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/kkv_data_store.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/kkv_data_store.py @@ -1,16 +1,28 @@ from dataclasses import dataclass +from typing import TYPE_CHECKING + +from sqlalchemy import UniqueConstraint +from sqlalchemy.orm import relationship from spiffworkflow_backend.models.db import SpiffworkflowBaseDBModel from spiffworkflow_backend.models.db import db +if TYPE_CHECKING: + from spiffworkflow_backend.models.kkv_data_store_entry import KKVDataStoreEntryModel # noqa: F401 + @dataclass class KKVDataStoreModel(SpiffworkflowBaseDBModel): __tablename__ = "kkv_data_store" + __table_args__ = (UniqueConstraint("identifier", "location", name="_kkv_identifier_location_unique"),) id: int = db.Column(db.Integer, primary_key=True) - top_level_key: str = db.Column(db.String(255), nullable=False, index=True) - secondary_key: str = db.Column(db.String(255), nullable=False, index=True) - value: dict = db.Column(db.JSON, nullable=False) + name: str = db.Column(db.String(255), index=True, nullable=False) + identifier: str = db.Column(db.String(255), index=True, nullable=False) + location: str = db.Column(db.String(255), nullable=False) + schema: dict = db.Column(db.JSON, nullable=False) + description: str = db.Column(db.String(255)) updated_at_in_seconds: int = db.Column(db.Integer, nullable=False) created_at_in_seconds: int = db.Column(db.Integer, nullable=False) + + entries = relationship("KKVDataStoreEntryModel", cascade="delete") diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/kkv_data_store_entry.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/kkv_data_store_entry.py new file mode 100644 index 000000000..7eed27094 --- /dev/null +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/kkv_data_store_entry.py @@ -0,0 +1,25 @@ +from dataclasses import dataclass + +from sqlalchemy import ForeignKey +from sqlalchemy import UniqueConstraint +from sqlalchemy.orm import relationship + +from spiffworkflow_backend.models.db import SpiffworkflowBaseDBModel +from spiffworkflow_backend.models.db import db +from spiffworkflow_backend.models.kkv_data_store import KKVDataStoreModel + + +@dataclass +class KKVDataStoreEntryModel(SpiffworkflowBaseDBModel): + __tablename__ = "kkv_data_store_entry" + __table_args__ = (UniqueConstraint("kkv_data_store_id", "top_level_key", "secondary_key", name="_instance_keys_unique"),) + + id: int = db.Column(db.Integer, primary_key=True) + kkv_data_store_id: int = db.Column(ForeignKey(KKVDataStoreModel.id), nullable=False, index=True) # type: ignore + top_level_key: str = db.Column(db.String(255), nullable=False, index=True) + secondary_key: str = db.Column(db.String(255), nullable=False, index=True) + value: dict = db.Column(db.JSON, nullable=False) + updated_at_in_seconds: int = db.Column(db.Integer, nullable=False) + created_at_in_seconds: int = db.Column(db.Integer, nullable=False) + + instance = relationship("KKVDataStoreModel", back_populates="entries") diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/routes/data_store_controller.py b/spiffworkflow-backend/src/spiffworkflow_backend/routes/data_store_controller.py index 85e4da926..266240104 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/routes/data_store_controller.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/routes/data_store_controller.py @@ -35,15 +35,19 @@ def data_store_list(process_group_identifier: str | None = None, page: int = 1, def data_store_types() -> flask.wrappers.Response: """Returns a list of the types of available data stores.""" - # this if == "json" check is temporary while we roll out support for other data stores + # this if != "typeahead" check is temporary while we roll out support for other data stores # being created with locations, identifiers and schemas - data_store_types = [{"type": k, "name": v[0].__name__, "description": v[1]} for k, v in DATA_STORES.items() if k == "json"] + data_store_types = [ + {"type": k, "name": v[0].__name__, "description": v[1]} for k, v in DATA_STORES.items() if k != "typeahead" + ] return make_response(jsonify(data_store_types), 200) -def _build_response(data_store_class: Any, name: str, page: int, per_page: int) -> flask.wrappers.Response: - data_store_query = data_store_class.get_data_store_query(name, None) +def _build_response( + data_store_class: Any, identifier: str, location: str | None, page: int, per_page: int +) -> flask.wrappers.Response: + data_store_query = data_store_class.get_data_store_query(identifier, location) data = data_store_query.paginate(page=page, per_page=per_page, error_out=False) results = [] for item in data.items: @@ -60,14 +64,16 @@ def _build_response(data_store_class: Any, name: str, page: int, per_page: int) return make_response(jsonify(response_json), 200) -def data_store_item_list(data_store_type: str, name: str, page: int = 1, per_page: int = 100) -> flask.wrappers.Response: +def data_store_item_list( + data_store_type: str, identifier: str, location: str | None = None, page: int = 1, per_page: int = 100 +) -> flask.wrappers.Response: """Returns a list of the items in a data store.""" if data_store_type not in DATA_STORES: raise ApiError("unknown_data_store", f"Unknown data store type: {data_store_type}", status_code=400) data_store_class, _ = DATA_STORES[data_store_type] - return _build_response(data_store_class, name, page, per_page) + return _build_response(data_store_class, identifier, location, page, per_page) def data_store_create(body: dict) -> flask.wrappers.Response: diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/scripts/create_unique_kkv_top_level_key.py b/spiffworkflow-backend/src/spiffworkflow_backend/scripts/create_unique_kkv_top_level_key.py index 4a7025ff6..de1a6dbeb 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/scripts/create_unique_kkv_top_level_key.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/scripts/create_unique_kkv_top_level_key.py @@ -1,7 +1,9 @@ from typing import Any +from spiffworkflow_backend.data_stores.crud import DataStoreCRUD from spiffworkflow_backend.models.db import db from spiffworkflow_backend.models.kkv_data_store import KKVDataStoreModel +from spiffworkflow_backend.models.kkv_data_store_entry import KKVDataStoreEntryModel from spiffworkflow_backend.models.script_attributes_context import ScriptAttributesContext from spiffworkflow_backend.scripts.script import Script @@ -19,9 +21,23 @@ class CreateUniqueKKVTopLevelKey(Script): ) def run(self, script_attributes_context: ScriptAttributesContext, *args: Any, **kwargs: Any) -> Any: - top_level_key_prefix = args[0] + identifier = args[0] + top_level_key_prefix = args[1] + spiff_task = script_attributes_context.task + location: str | None = None - model = KKVDataStoreModel(top_level_key="", secondary_key="", value={}) + if identifier is not None and spiff_task is not None: + location = DataStoreCRUD.data_store_location_for_task(KKVDataStoreModel, spiff_task, identifier) + + store_model: KKVDataStoreModel | None = None + + if location is not None: + store_model = db.session.query(KKVDataStoreModel).filter_by(identifier=identifier, location=location).first() + + if store_model is None: + raise Exception(f"Could not find KKV data store with the identifier '{identifier}'") + + model = KKVDataStoreEntryModel(kkv_data_store_id=store_model.id, top_level_key="", secondary_key="", value={}) db.session.add(model) db.session.commit() diff --git a/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_kkv_data_store.py b/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_kkv_data_store.py index c6445dccf..38add5c3f 100644 --- a/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_kkv_data_store.py +++ b/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_kkv_data_store.py @@ -7,6 +7,7 @@ from flask.app import Flask from spiffworkflow_backend.data_stores.kkv import KKVDataStore from spiffworkflow_backend.models.db import db from spiffworkflow_backend.models.kkv_data_store import KKVDataStoreModel +from spiffworkflow_backend.models.kkv_data_store_entry import KKVDataStoreEntryModel from tests.spiffworkflow_backend.helpers.base_test import BaseTest @@ -17,23 +18,36 @@ class MockTask: @pytest.fixture() -def with_clean_data_store(app: Flask, with_db_and_bpmn_file_cleanup: None) -> Generator[None, None, None]: +def with_clean_data_store(app: Flask, with_db_and_bpmn_file_cleanup: None) -> Generator[KKVDataStoreModel, None, None]: + app.config["THREAD_LOCAL_DATA"].process_model_identifier = "the_location/of/some/process-model" + db.session.query(KKVDataStoreModel).delete() db.session.commit() - yield + + model = KKVDataStoreModel(identifier="the_id", name="the_name", location="the_location", schema={}) + db.session.add(model) + db.session.commit() + + yield model @pytest.fixture() -def with_key1_key2_record(with_clean_data_store: None) -> Generator[None, None, None]: - model = KKVDataStoreModel(top_level_key="key1", secondary_key="key2", value="value1") # type: ignore +def with_key1_key2_record(with_clean_data_store: KKVDataStoreModel) -> Generator[KKVDataStoreModel, None, None]: + model = KKVDataStoreEntryModel( + kkv_data_store_id=with_clean_data_store.id, top_level_key="key1", secondary_key="key2", value={"key": "value"} + ) db.session.add(model) db.session.commit() - yield + + yield with_clean_data_store class TestKKVDataStore(BaseTest): """Infer from class name.""" + def _entry_count(self, model: KKVDataStoreModel) -> int: + return db.session.query(KKVDataStoreEntryModel).filter_by(kkv_data_store_id=model.id).count() + def test_returns_none_if_no_records_exist(self, with_clean_data_store: None) -> None: kkv_data_store = KKVDataStore("the_id", "the_name") my_task = MockTask(data={}) @@ -48,7 +62,7 @@ class TestKKVDataStore(BaseTest): kkv_data_store.get(my_task) assert len(my_task.data) == 1 result = my_task.data["the_id"]("key1", "key2") - assert result == "value1" + assert result == {"key": "value"} def test_returns_none_if_first_key_does_not_match(self, with_key1_key2_record: None) -> None: kkv_data_store = KKVDataStore("the_id", "the_name") @@ -66,25 +80,25 @@ class TestKKVDataStore(BaseTest): result = my_task.data["the_id"]("key1", "key22") assert result is None - def test_can_insert_a_value(self, with_clean_data_store: None) -> None: + def test_can_insert_a_value(self, with_clean_data_store: KKVDataStoreModel) -> None: kkv_data_store = KKVDataStore("the_id", "the_name") my_task = MockTask(data={"the_id": {"newKey1": {"newKey2": "newValue"}}}) kkv_data_store.set(my_task) - count = db.session.query(KKVDataStoreModel).count() + count = self._entry_count(with_clean_data_store) assert count == 1 - def test_can_insert_mulitple_values_for_same_top_key(self, with_clean_data_store: None) -> None: + def test_can_insert_multiple_values_for_same_top_key(self, with_clean_data_store: KKVDataStoreModel) -> None: kkv_data_store = KKVDataStore("the_id", "the_name") my_task = MockTask(data={"the_id": {"newKey1": {"newKey2": "newValue", "newKey3": "newValue2"}}}) kkv_data_store.set(my_task) - count = db.session.query(KKVDataStoreModel).count() + count = self._entry_count(with_clean_data_store) assert count == 2 - def test_can_insert_mulitple_values_for_different_top_key(self, with_clean_data_store: None) -> None: + def test_can_insert_multiple_values_for_different_top_key(self, with_clean_data_store: KKVDataStoreModel) -> None: kkv_data_store = KKVDataStore("the_id", "the_name") my_task = MockTask(data={"the_id": {"newKey1": {"newKey2": "newValue"}, "newKey3": {"newKey4": "newValue2"}}}) kkv_data_store.set(my_task) - count = db.session.query(KKVDataStoreModel).count() + count = self._entry_count(with_clean_data_store) assert count == 2 def test_value_is_removed_from_task_data_after_insert(self, with_clean_data_store: None) -> None: @@ -103,44 +117,44 @@ class TestKKVDataStore(BaseTest): result2 = my_task.data["the_id"]("newKey3", "newKey4") assert result2 == "newValue2" - def test_can_update_a_value(self, with_clean_data_store: None) -> None: + def test_can_update_a_value(self, with_clean_data_store: KKVDataStore) -> None: kkv_data_store = KKVDataStore("the_id", "the_name") my_task = MockTask(data={"the_id": {"newKey1": {"newKey2": "newValue"}}}) kkv_data_store.set(my_task) my_task.data = {"the_id": {"newKey1": {"newKey2": "newValue2"}}} kkv_data_store.set(my_task) - count = db.session.query(KKVDataStoreModel).count() + count = self._entry_count(with_clean_data_store) assert count == 1 kkv_data_store.get(my_task) result2 = my_task.data["the_id"]("newKey1", "newKey2") assert result2 == "newValue2" - def test_can_delete_record_by_nulling_a_secondary_key(self, with_key1_key2_record: None) -> None: + def test_can_delete_record_by_nulling_a_secondary_key(self, with_key1_key2_record: KKVDataStoreModel) -> None: kkv_data_store = KKVDataStore("the_id", "the_name") my_task = MockTask(data={"the_id": {"key1": {"key2": None}}}) kkv_data_store.set(my_task) kkv_data_store.get(my_task) result = my_task.data["the_id"]("key1", "key2") assert result is None - count = db.session.query(KKVDataStoreModel).count() + count = self._entry_count(with_key1_key2_record) assert count == 0 - def test_can_delete_all_records_by_nulling_a_top_level_key(self, with_clean_data_store: None) -> None: + def test_can_delete_all_records_by_nulling_a_top_level_key(self, with_clean_data_store: KKVDataStoreModel) -> None: kkv_data_store = KKVDataStore("the_id", "the_name") my_task = MockTask(data={"the_id": {"newKey1": {"newKey2": "newValue", "newKey3": "newValue2"}}}) kkv_data_store.set(my_task) my_task.data = {"the_id": {"newKey1": None}} kkv_data_store.set(my_task) - count = db.session.query(KKVDataStoreModel).count() + count = self._entry_count(with_clean_data_store) assert count == 0 - def test_top_key_delete_does_not_delete_for_other_top_levels(self, with_clean_data_store: None) -> None: + def test_top_key_delete_does_not_delete_for_other_top_levels(self, with_clean_data_store: KKVDataStoreModel) -> None: kkv_data_store = KKVDataStore("the_id", "the_name") my_task = MockTask(data={"the_id": {"newKey1": {"newKey2": "newValue"}, "newKey3": {"newKey4": "newValue2"}}}) kkv_data_store.set(my_task) my_task.data = {"the_id": {"newKey1": None}} kkv_data_store.set(my_task) - count = db.session.query(KKVDataStoreModel).count() + count = self._entry_count(with_clean_data_store) assert count == 1 kkv_data_store.get(my_task) result = my_task.data["the_id"]("newKey3", "newKey4") diff --git a/spiffworkflow-frontend/src/components/DataStoreListTable.tsx b/spiffworkflow-frontend/src/components/DataStoreListTable.tsx index d71fb9e89..4c919ca92 100644 --- a/spiffworkflow-frontend/src/components/DataStoreListTable.tsx +++ b/spiffworkflow-frontend/src/components/DataStoreListTable.tsx @@ -37,21 +37,26 @@ export default function DataStoreListTable() { 'datastore' ); const dataStoreType = searchParams.get('type') || ''; - const dataStoreName = searchParams.get('name') || ''; + const dataStoreIdentifier = searchParams.get('identifier') || ''; + const dataStoreLocation = searchParams.get('location') || ''; - if (dataStoreType === '' || dataStoreName === '') { + if (dataStoreType === '' || dataStoreIdentifier === '') { return; } - if (dataStores && dataStoreName && dataStoreType) { + if (dataStores && dataStoreIdentifier && dataStoreType) { dataStores.forEach((ds) => { - if (ds.name === dataStoreName && ds.type === dataStoreType) { + if ( + ds.id === dataStoreIdentifier && + ds.type === dataStoreType && + ds.location === dataStoreLocation + ) { setDataStore(ds); } }); } - const queryParamString = `per_page=${perPage}&page=${page}`; + const queryParamString = `per_page=${perPage}&page=${page}&location=${dataStoreLocation}`; HttpService.makeCallToBackend({ - path: `/data-stores/${dataStoreType}/${dataStoreName}/items?${queryParamString}`, + path: `/data-stores/${dataStoreType}/${dataStoreIdentifier}/items?${queryParamString}`, successCallback: (response: DataStoreRecords) => { setResults(response.results); setPagination(response.pagination); @@ -101,6 +106,10 @@ export default function DataStoreListTable() { ); }; + const locationDescription = (ds: DataStore) => { + return ds.location ? ` @ ${ds.location}` : ''; + }; + const { page, perPage } = getPageInfoFromSearchParams( searchParams, 10, @@ -116,13 +125,16 @@ export default function DataStoreListTable() { label="Please select a data store" items={dataStores} selectedItem={dataStore} - itemToString={(ds: DataStore) => (ds ? `${ds.name} (${ds.type})` : '')} + itemToString={(ds: DataStore) => + ds ? `${ds.name} (${ds.type}${locationDescription(ds)})` : '' + } onChange={(event: any) => { setDataStore(event.selectedItem); searchParams.set('datastore_page', '1'); searchParams.set('datastore_per_page', '10'); searchParams.set('type', event.selectedItem.type); - searchParams.set('name', event.selectedItem.name); + searchParams.set('identifier', event.selectedItem.id); + searchParams.set('location', event.selectedItem.location); setSearchParams(searchParams); }} /> diff --git a/spiffworkflow-frontend/src/components/DataStoreListTiles.tsx b/spiffworkflow-frontend/src/components/DataStoreListTiles.tsx index 2f4441510..d5cd06097 100644 --- a/spiffworkflow-frontend/src/components/DataStoreListTiles.tsx +++ b/spiffworkflow-frontend/src/components/DataStoreListTiles.tsx @@ -69,7 +69,7 @@ export default function DataStoreListTiles({ diff --git a/spiffworkflow-frontend/src/interfaces.ts b/spiffworkflow-frontend/src/interfaces.ts index 9045b1100..9d7c72bf3 100644 --- a/spiffworkflow-frontend/src/interfaces.ts +++ b/spiffworkflow-frontend/src/interfaces.ts @@ -464,6 +464,7 @@ export interface DataStore { id: string; schema: string; description?: string | null; + location?: string | null; } export interface DataStoreType { diff --git a/spiffworkflow-frontend/src/routes/DataStoreEdit.tsx b/spiffworkflow-frontend/src/routes/DataStoreEdit.tsx index 8489dcf13..7e98c4584 100644 --- a/spiffworkflow-frontend/src/routes/DataStoreEdit.tsx +++ b/spiffworkflow-frontend/src/routes/DataStoreEdit.tsx @@ -10,6 +10,7 @@ export default function DataStoreEdit() { const params = useParams(); const [searchParams] = useSearchParams(); const parentGroupId = searchParams.get('parentGroupId'); + const dataStoreType = searchParams.get('type'); const dataStoreIdentifier = params.data_store_identifier; const [dataStore, setDataStore] = useState({ id: '', @@ -30,10 +31,10 @@ export default function DataStoreEdit() { const queryParams = `?process_group_identifier=${parentGroupId}`; HttpService.makeCallToBackend({ - path: `/data-stores/json/${dataStoreIdentifier}${queryParams}`, + path: `/data-stores/${dataStoreType}/${dataStoreIdentifier}${queryParams}`, successCallback: setDataStoreFromResult, }); - }, [dataStoreIdentifier, parentGroupId]); + }, [dataStoreIdentifier, parentGroupId, dataStoreType]); const hotCrumbs: HotCrumbItem[] = [['Process Groups', '/process-groups']]; if (parentGroupId) {