From e7950456e92af58b6228f551dfcc53f6124cf7d7 Mon Sep 17 00:00:00 2001 From: jasquat Date: Fri, 3 Mar 2023 10:20:26 -0500 Subject: [PATCH] all tests are passing with new spec tables w/ burnettk --- .../load_database_models.py | 12 +- .../models/bpmn_process.py | 2 +- .../bpmn_process_definition_relationship.py | 22 ++-- .../spiffworkflow_backend/models/json_data.py | 3 - .../models/process_instance.py | 8 +- .../models/spiff_step_details.py | 1 + .../src/spiffworkflow_backend/models/task.py | 9 +- .../models/task_definition.py | 19 ++- .../services/process_instance_processor.py | 110 ++++++++++++------ 9 files changed, 123 insertions(+), 63 deletions(-) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/load_database_models.py b/spiffworkflow-backend/src/spiffworkflow_backend/load_database_models.py index ab518fa54..490cf0b1f 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/load_database_models.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/load_database_models.py @@ -58,10 +58,16 @@ from spiffworkflow_backend.models.process_instance_data import ( ) # noqa: F401 from spiffworkflow_backend.models.bpmn_process import BpmnProcessModel # noqa: F401 -from spiffworkflow_backend.models.bpmn_process_definition import BpmnProcessDefinitionModel # noqa: F401 +from spiffworkflow_backend.models.bpmn_process_definition import ( + BpmnProcessDefinitionModel, +) # noqa: F401 from spiffworkflow_backend.models.task import TaskModel # noqa: F401 -from spiffworkflow_backend.models.task_definition import TaskDefinitionModel # noqa: F401 +from spiffworkflow_backend.models.task_definition import ( + TaskDefinitionModel, +) # noqa: F401 from spiffworkflow_backend.models.json_data import JsonDataModel # noqa: F401 -from spiffworkflow_backend.models.bpmn_process_definition_relationship import BpmnProcessDefinitionRelationshipModel # noqa: F401 +from spiffworkflow_backend.models.bpmn_process_definition_relationship import ( + BpmnProcessDefinitionRelationshipModel, +) # noqa: F401 add_listeners() diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process.py index 05519d4b4..948cdffd1 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process.py @@ -1,6 +1,6 @@ from __future__ import annotations + from sqlalchemy import ForeignKey -from spiffworkflow_backend.models.bpmn_process_definition import BpmnProcessDefinitionModel from spiffworkflow_backend.models.db import db from spiffworkflow_backend.models.db import SpiffworkflowBaseDBModel diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process_definition_relationship.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process_definition_relationship.py index 218d226a7..8f6fdca2e 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process_definition_relationship.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process_definition_relationship.py @@ -1,9 +1,11 @@ from __future__ import annotations -from sqlalchemy import UniqueConstraint -from sqlalchemy.orm import deferred -from sqlalchemy import ForeignKey -from spiffworkflow_backend.models.bpmn_process_definition import BpmnProcessDefinitionModel +from sqlalchemy import ForeignKey +from sqlalchemy import UniqueConstraint + +from spiffworkflow_backend.models.bpmn_process_definition import ( + BpmnProcessDefinitionModel, +) from spiffworkflow_backend.models.db import db from spiffworkflow_backend.models.db import SpiffworkflowBaseDBModel @@ -12,10 +14,16 @@ class BpmnProcessDefinitionRelationshipModel(SpiffworkflowBaseDBModel): __tablename__ = "bpmn_process_definition_relationship" __table_args__ = ( UniqueConstraint( - "bpmn_process_definition_parent_id", "bpmn_process_definition_child_id", name="bpmn_process_definition_relationship_unique" + "bpmn_process_definition_parent_id", + "bpmn_process_definition_child_id", + name="bpmn_process_definition_relationship_unique", ), ) id: int = db.Column(db.Integer, primary_key=True) - bpmn_process_definition_parent_id: int = db.Column(ForeignKey(BpmnProcessDefinitionModel.id), nullable=False) # type: ignore - bpmn_process_definition_child_id: int = db.Column(ForeignKey(BpmnProcessDefinitionModel.id), nullable=False) # type: ignore + bpmn_process_definition_parent_id: int = db.Column( + ForeignKey(BpmnProcessDefinitionModel.id), nullable=False + ) # type: ignore + bpmn_process_definition_child_id: int = db.Column( + ForeignKey(BpmnProcessDefinitionModel.id), nullable=False + ) # type: ignore diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/json_data.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/json_data.py index 1e03b9ff3..103bdc57b 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/json_data.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/json_data.py @@ -1,7 +1,4 @@ from __future__ import annotations -from sqlalchemy.orm import deferred -from sqlalchemy import ForeignKey -from spiffworkflow_backend.models.bpmn_process_definition import BpmnProcessDefinitionModel from spiffworkflow_backend.models.db import db from spiffworkflow_backend.models.db import SpiffworkflowBaseDBModel diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/process_instance.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/process_instance.py index 4cfb52b7b..44cfe8067 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/process_instance.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/process_instance.py @@ -1,6 +1,5 @@ """Process_instance.""" from __future__ import annotations -from spiffworkflow_backend.models.bpmn_process_definition import BpmnProcessDefinitionModel from typing import Any from typing import cast @@ -16,6 +15,9 @@ from sqlalchemy.orm import relationship from sqlalchemy.orm import validates from spiffworkflow_backend.helpers.spiff_enum import SpiffEnum +from spiffworkflow_backend.models.bpmn_process_definition import ( + BpmnProcessDefinitionModel, +) from spiffworkflow_backend.models.db import db from spiffworkflow_backend.models.db import SpiffworkflowBaseDBModel from spiffworkflow_backend.models.process_instance_data import ProcessInstanceDataModel @@ -75,7 +77,9 @@ class ProcessInstanceModel(SpiffworkflowBaseDBModel): ) process_instance_data = relationship("ProcessInstanceDataModel", cascade="delete") - bpmn_process_definition_id: int = db.Column(ForeignKey(BpmnProcessDefinitionModel.id), nullable=True) # type: ignore + bpmn_process_definition_id: int = db.Column( + ForeignKey(BpmnProcessDefinitionModel.id), nullable=True + ) # type: ignore bpmn_process_definition = relationship(BpmnProcessDefinitionModel) active_human_tasks = relationship( diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/spiff_step_details.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/spiff_step_details.py index 503b2a418..6b91bafa1 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/spiff_step_details.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/spiff_step_details.py @@ -149,6 +149,7 @@ from spiffworkflow_backend.models.process_instance import ProcessInstanceModel # "data": {} # }, + @dataclass class SpiffStepDetailsModel(SpiffworkflowBaseDBModel): """SpiffStepDetailsModel.""" diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/task.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/task.py index 326ea5b45..3fda6677f 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/task.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/task.py @@ -1,10 +1,6 @@ """Task.""" import enum from dataclasses import dataclass -from sqlalchemy import ForeignKey -from spiffworkflow_backend.models.bpmn_process import BpmnProcessModel -from spiffworkflow_backend.models.db import db -from spiffworkflow_backend.models.db import SpiffworkflowBaseDBModel from typing import Any from typing import Optional from typing import Union @@ -13,6 +9,11 @@ import marshmallow from marshmallow import Schema from marshmallow_enum import EnumField # type: ignore from SpiffWorkflow.task import TaskStateNames +from sqlalchemy import ForeignKey + +from spiffworkflow_backend.models.bpmn_process import BpmnProcessModel +from spiffworkflow_backend.models.db import db +from spiffworkflow_backend.models.db import SpiffworkflowBaseDBModel from spiffworkflow_backend.models.task_definition import TaskDefinitionModel # type: ignore diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/task_definition.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/task_definition.py index d6e2061d1..7d7c50873 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/task_definition.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/task_definition.py @@ -1,9 +1,12 @@ from __future__ import annotations -from sqlalchemy.orm import relationship -from sqlalchemy import UniqueConstraint -from sqlalchemy import ForeignKey -from spiffworkflow_backend.models.bpmn_process_definition import BpmnProcessDefinitionModel +from sqlalchemy import ForeignKey +from sqlalchemy import UniqueConstraint +from sqlalchemy.orm import relationship + +from spiffworkflow_backend.models.bpmn_process_definition import ( + BpmnProcessDefinitionModel, +) from spiffworkflow_backend.models.db import db from spiffworkflow_backend.models.db import SpiffworkflowBaseDBModel @@ -12,12 +15,16 @@ class TaskDefinitionModel(SpiffworkflowBaseDBModel): __tablename__ = "task_definition" __table_args__ = ( UniqueConstraint( - "bpmn_process_definition_id", "bpmn_identifier", name="task_definition_unique" + "bpmn_process_definition_id", + "bpmn_identifier", + name="task_definition_unique", ), ) id: int = db.Column(db.Integer, primary_key=True) - bpmn_process_definition_id: int = db.Column(ForeignKey(BpmnProcessDefinitionModel.id), nullable=False) # type: ignore + bpmn_process_definition_id: int = db.Column( + ForeignKey(BpmnProcessDefinitionModel.id), nullable=False + ) # type: ignore bpmn_process_definition = relationship(BpmnProcessDefinitionModel) bpmn_identifier: str = db.Column(db.String(255), nullable=False, index=True) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py b/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py index aaa8a9c35..eaf0e433b 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py @@ -1,7 +1,5 @@ """Process_instance_processor.""" import _strptime -from spiffworkflow_backend.models import serialized_bpmn_definition # type: ignore -from spiffworkflow_backend.models.bpmn_process_definition_relationship import BpmnProcessDefinitionRelationshipModel # noqa: F401 import decimal import json import logging @@ -57,8 +55,13 @@ from SpiffWorkflow.util.deep_merge import DeepMerge # type: ignore from sqlalchemy import text from spiffworkflow_backend.exceptions.api_error import ApiError -from spiffworkflow_backend.models import bpmn_process_definition_relationship -from spiffworkflow_backend.models.bpmn_process_definition import BpmnProcessDefinitionModel +from spiffworkflow_backend.models import serialized_bpmn_definition # type: ignore +from spiffworkflow_backend.models.bpmn_process_definition import ( + BpmnProcessDefinitionModel, +) +from spiffworkflow_backend.models.bpmn_process_definition_relationship import ( + BpmnProcessDefinitionRelationshipModel, +) # noqa: F401 from spiffworkflow_backend.models.db import db from spiffworkflow_backend.models.file import File from spiffworkflow_backend.models.file import FileType @@ -525,6 +528,22 @@ class ProcessInstanceProcessor: self.bpmn_process_instance ) + @classmethod + def _get_definition_dict_for_bpmn_process_definition(cls, bpmn_process_definition: BpmnProcessDefinitionModel) -> dict: + task_definitions = TaskDefinitionModel.query.filter_by( + bpmn_process_definition_id=bpmn_process_definition.id + ).all() + bpmn_process_definition_dict: dict = json.loads( + bpmn_process_definition.properties_json + ) + bpmn_process_definition_dict["task_specs"] = {} + for task_definition in task_definitions: + bpmn_process_definition_dict["task_specs"][ + task_definition.bpmn_identifier + ] = json.loads(task_definition.properties_json) + return bpmn_process_definition_dict + + @classmethod def _get_full_bpmn_json(cls, process_instance_model: ProcessInstanceModel) -> dict: if process_instance_model.serialized_bpmn_definition_id is None: @@ -534,24 +553,28 @@ class ProcessInstanceProcessor: # loaded_json: dict = json.loads(serialized_bpmn_definition.static_json) # or "{}") serialized_bpmn_definition = {} - bpmn_process_definition = BpmnProcessDefinitionModel.query.filter_by(id=process_instance_model.bpmn_process_definition_id).first() - task_definitions = TaskDefinitionModel.query.filter_by(bpmn_process_definition_id=process_instance_model.bpmn_process_definition_id).all() + bpmn_process_definition = BpmnProcessDefinitionModel.query.filter_by( + id=process_instance_model.bpmn_process_definition_id + ).first() if bpmn_process_definition is not None: - serialized_bpmn_definition = {"serializer_version": cls.SERIALIZER_VERSION, "spec": {}, "subprocess_specs": {}} - bpmn_process_definition_dict = json.loads(bpmn_process_definition.properties_json) - bpmn_process_definition_dict['task_specs'] = {} - for task_definition in task_definitions: - bpmn_process_definition_dict['task_specs'][task_definition.bpmn_identifier] = json.loads(task_definition.properties_json) - serialized_bpmn_definition['spec'] = bpmn_process_definition_dict + serialized_bpmn_definition = { + "serializer_version": cls.SERIALIZER_VERSION, + "spec": {}, + "subprocess_specs": {}, + } + serialized_bpmn_definition['spec'] = cls._get_definition_dict_for_bpmn_process_definition(bpmn_process_definition) - bpmn_process_subprocess_definitions = BpmnProcessDefinitionRelationshipModel.query.filter_by(bpmn_process_definition_parent_id=bpmn_process_definition.id).all() - for bpmn_process_subprocess_definition in bpmn_process_subprocess_definitions: - subprocess_task_definitions = TaskDefinitionModel.query.filter_by(bpmn_process_definition_id=bpmn_process_subprocess_definition.id).all() - bpmn_process_subprocess_dict = json.loads(bpmn_process_definition.properties_json) - bpmn_process_subprocess_dict['task_specs'] = {} - for subprocess_task_definition in subprocess_task_definitions: - bpmn_process_subprocess_dict['task_specs'][subprocess_task_definition.bpmn_identifier] = json.loads(subprocess_task_definition.properties_json) - serialized_bpmn_definition['subprocess_specs'][bpmn_process_subprocess_definition.bpmn_identifier] = bpmn_process_subprocess_dict + bpmn_process_subprocess_definitions = ( + BpmnProcessDefinitionRelationshipModel.query.filter_by( + bpmn_process_definition_parent_id=bpmn_process_definition.id + ).all() + ) + for ( + bpmn_process_subprocess_definition + ) in bpmn_process_subprocess_definitions: + serialized_bpmn_definition["subprocess_specs"][ + bpmn_process_subprocess_definition.bpmn_identifier + ] = cls._get_definition_dict_for_bpmn_process_definition(bpmn_process_subprocess_definition) loaded_json: dict = serialized_bpmn_definition process_instance_data = process_instance_model.process_instance_data @@ -934,19 +957,25 @@ class ProcessInstanceProcessor: db.session.add(process_instance_data) self.process_instance_model.process_instance_data = process_instance_data - - def _store_bpmn_process_definition(self, process_bpmn_properties: dict, bpmn_process_definition_parent: Optional[BpmnProcessDefinitionModel] = None) -> BpmnProcessDefinitionModel: - process_bpmn_identifier = process_bpmn_properties['name'] + def _store_bpmn_process_definition( + self, + process_bpmn_properties: dict, + bpmn_process_definition_parent: Optional[BpmnProcessDefinitionModel] = None, + ) -> BpmnProcessDefinitionModel: + process_bpmn_identifier = process_bpmn_properties["name"] new_hash_digest = sha256( json.dumps(process_bpmn_properties, sort_keys=True).encode("utf8") ).hexdigest() - bpmn_process_definition: Optional[BpmnProcessDefinitionModel] = BpmnProcessDefinitionModel.query.filter_by( - hash=new_hash_digest - ).first() + bpmn_process_definition: Optional[BpmnProcessDefinitionModel] = ( + BpmnProcessDefinitionModel.query.filter_by(hash=new_hash_digest).first() + ) if bpmn_process_definition is None: task_specs = process_bpmn_properties.pop("task_specs") bpmn_process_definition = BpmnProcessDefinitionModel( - hash=new_hash_digest, bpmn_identifier=process_bpmn_identifier, properties_json=json.dumps(process_bpmn_properties), type="process" + hash=new_hash_digest, + bpmn_identifier=process_bpmn_identifier, + properties_json=json.dumps(process_bpmn_properties), + type="process", ) db.session.add(bpmn_process_definition) @@ -955,15 +984,17 @@ class ProcessInstanceProcessor: bpmn_process_definition=bpmn_process_definition, bpmn_identifier=task_bpmn_identifier, properties_json=json.dumps(task_bpmn_properties), - typename=task_bpmn_properties['typename'], + typename=task_bpmn_properties["typename"], ) db.session.add(task_definition) if bpmn_process_definition_parent: - bpmn_process_definition_relationship = BpmnProcessDefinitionRelationshipModel.query.filter_by( - bpmn_process_definition_parent_id=bpmn_process_definition_parent.id, - bpmn_process_definition_child_id=bpmn_process_definition.id, - ).first() + bpmn_process_definition_relationship = ( + BpmnProcessDefinitionRelationshipModel.query.filter_by( + bpmn_process_definition_parent_id=bpmn_process_definition_parent.id, + bpmn_process_definition_child_id=bpmn_process_definition.id, + ).first() + ) if bpmn_process_definition_relationship is None: bpmn_process_definition_relationship = BpmnProcessDefinitionRelationshipModel( bpmn_process_definition_parent_id=bpmn_process_definition_parent.id, @@ -987,11 +1018,16 @@ class ProcessInstanceProcessor: else: process_instance_data_dict[bpmn_key] = bpmn_dict[bpmn_key] - bpmn_process_definition_parent = self._store_bpmn_process_definition(bpmn_spec_dict['spec']) - for process_bpmn_properties in bpmn_spec_dict['subprocess_specs'].values(): - self._store_bpmn_process_definition(process_bpmn_properties, bpmn_process_definition_parent) - self.process_instance_model.bpmn_process_definition = bpmn_process_definition_parent - + bpmn_process_definition_parent = self._store_bpmn_process_definition( + bpmn_spec_dict["spec"] + ) + for process_bpmn_properties in bpmn_spec_dict["subprocess_specs"].values(): + self._store_bpmn_process_definition( + process_bpmn_properties, bpmn_process_definition_parent + ) + self.process_instance_model.bpmn_process_definition = ( + bpmn_process_definition_parent + ) # # FIXME: always save new hash until we get updated Spiff without loopresettask # # if self.process_instance_model.serialized_bpmn_definition_id is None: