From 971de5cb7c751926e05bfc74bd92488fe9f3fa9e Mon Sep 17 00:00:00 2001 From: jasquat Date: Thu, 2 Mar 2023 16:31:21 -0500 Subject: [PATCH 01/25] created models to hold the bpmn_json w/ burnettk --- .../models/bpmn_process.py | 25 ++++ .../models/bpmn_process_definition.py | 27 ++++ .../bpmn_process_definition_relationship.py | 14 ++ .../spiffworkflow_backend/models/json_data.py | 16 ++ .../models/serialized_bpmn_definition.py | 4 +- .../models/spiff_step_details.py | 138 ++++++++++++++++++ .../src/spiffworkflow_backend/models/task.py | 38 ++++- .../models/task_definition.py | 23 +++ 8 files changed, 283 insertions(+), 2 deletions(-) create mode 100644 spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process.py create mode 100644 spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process_definition.py create mode 100644 spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process_definition_relationship.py create mode 100644 spiffworkflow-backend/src/spiffworkflow_backend/models/json_data.py create mode 100644 spiffworkflow-backend/src/spiffworkflow_backend/models/task_definition.py diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process.py new file mode 100644 index 000000000..b704ce92e --- /dev/null +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process.py @@ -0,0 +1,25 @@ +from __future__ import annotations +from sqlalchemy import ForeignKey +from spiffworkflow_backend.models.bpmn_process_definition import BpmnProcessDefinitionModel + +from spiffworkflow_backend.models.db import db +from spiffworkflow_backend.models.db import SpiffworkflowBaseDBModel + + +# properties_json attributes: +# "last_task", # guid generated by spiff +# "root", # guid generated by spiff +# "success", # boolean +# "bpmn_messages", # if top-level process +# "correlations", # if top-level process +class BpmnProcessModel(SpiffworkflowBaseDBModel): + __tablename__ = "bpmn_process" + id: int = db.Column(db.Integer, primary_key=True) + + parent_process_id: int = db.Column(ForeignKey("BpmnProcessModel.id"), nullable=True) + + properties_json: dict = db.Column(db.JSON, nullable=False) + json_data_hash: str = db.Column(db.String(255), nullable=False, index=True) + + # subprocess or top_level_process + process_type: str = db.Column(db.String(30), nullable=False) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process_definition.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process_definition.py new file mode 100644 index 000000000..54d70c9fc --- /dev/null +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process_definition.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +from spiffworkflow_backend.models.db import db +from spiffworkflow_backend.models.db import SpiffworkflowBaseDBModel + + +# contents of top-level attributes from spiff: +# "subprocess_specs", +# "spec", +# +# each subprocess will have its own row in this table. +# there is a join table to link them together: bpmn_process_definition_relationship +class BpmnProcessDefinitionModel(SpiffworkflowBaseDBModel): + __tablename__ = "bpmn_process_definition" + id: int = db.Column(db.Integer, primary_key=True) + + # this is a sha256 hash of spec and serializer_version + hash: str = db.Column(db.String(255), nullable=False, index=True, unique=True) + + properties_json: str = db.Column(db.JSON, nullable=False) + + # process or subprocess + type: str = db.Column(db.String(32), nullable=False, index=True) + + # TODO: remove these from process_instance + bpmn_version_control_type: str = db.Column(db.String(50)) + bpmn_version_control_identifier: str = db.Column(db.String(255)) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process_definition_relationship.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process_definition_relationship.py new file mode 100644 index 000000000..80f5c9bb2 --- /dev/null +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process_definition_relationship.py @@ -0,0 +1,14 @@ +from __future__ import annotations +from sqlalchemy.orm import deferred +from sqlalchemy import ForeignKey +from spiffworkflow_backend.models.bpmn_process_definition import BpmnProcessDefinitionModel + +from spiffworkflow_backend.models.db import db +from spiffworkflow_backend.models.db import SpiffworkflowBaseDBModel + + +class BpmnProcessDefinitionRelationshipModel(SpiffworkflowBaseDBModel): + __tablename__ = "bpmn_process_definition_relationship" + id: int = db.Column(db.Integer, primary_key=True) + bpmn_process_definition_parent_id: int = db.Column(ForeignKey(BpmnProcessDefinitionModel.id), nullable=False) # type: ignore + bpmn_process_definition_child_id: int = db.Column(ForeignKey(BpmnProcessDefinitionModel.id), nullable=False) # type: ignore diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/json_data.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/json_data.py new file mode 100644 index 000000000..1e03b9ff3 --- /dev/null +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/json_data.py @@ -0,0 +1,16 @@ +from __future__ import annotations +from sqlalchemy.orm import deferred +from sqlalchemy import ForeignKey +from spiffworkflow_backend.models.bpmn_process_definition import BpmnProcessDefinitionModel + +from spiffworkflow_backend.models.db import db +from spiffworkflow_backend.models.db import SpiffworkflowBaseDBModel + + +class JsonDataModel(SpiffworkflowBaseDBModel): + __tablename__ = "json_data" + id: int = db.Column(db.Integer, primary_key=True) + + # this is a sha256 hash of spec and serializer_version + hash: str = db.Column(db.String(255), nullable=False, index=True, unique=True) + data: dict = db.Column(db.JSON, nullable=False) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/serialized_bpmn_definition.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/serialized_bpmn_definition.py index 5a3d983e9..5ccb71466 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/serialized_bpmn_definition.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/serialized_bpmn_definition.py @@ -15,8 +15,10 @@ from spiffworkflow_backend.models.db import SpiffworkflowBaseDBModel # runtime # "bpmn_messages", # "correlations", -# "data", # "subprocesses", +# +# also in subprocesses +# "data", # "tasks" # "last_task", # guid generated by spiff # "root", # guid generated by spiff diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/spiff_step_details.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/spiff_step_details.py index 713bd3cd8..503b2a418 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/spiff_step_details.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/spiff_step_details.py @@ -10,6 +10,144 @@ from spiffworkflow_backend.models.db import db from spiffworkflow_backend.models.db import SpiffworkflowBaseDBModel from spiffworkflow_backend.models.process_instance import ProcessInstanceModel +# process_instance: +# process_model_definition_id +# process_id + + +# "bpmn_messages", +# "correlations", +# "subprocesses", <-- omit from json +# +# also in subprocesses +# "data", +# "tasks" <-- omit from json +# "last_task", # guid generated by spiff +# "root", # guid generated by spiff +# "success", # boolean +# +# runtime_json: +# "last_task", # guid generated by spiff +# "root", # guid generated by spiff +# "success", # boolean +# "bpmn_messages", # if top-level process +# "correlations", # if top-level process +# process: +# id +# parent_process_id +# runtime_json <-- minus tasks and subproceses +# type <-- subprocess, top_level_process +# data + +# runtime_json: +# "id": "a56e1403-2838-4f03-a31f-f99afe16f38d", +# "parent": null, +# "children": [ +# "af6ba340-71e7-46d7-b2d4-e3db1751785d" +# ], +# "last_state_change": 1677775475.18116, +# "state": 32, +# "task_spec": "Root", +# "triggered": false, +# "workflow_name": "Process_category_number_one_call_activity_call_activity_test_bd2e724", +# "internal_data": {}, +# "data": {} +# task: +# id +# guid +# process_id +# task_definition_id +# state <-- store string value +# runtime_json +# data + +# ### workflow +# "io_specification": null, +# "data_objects": {}, +# "correlation_keys": {}, +# "typename": "BpmnProcessSpec" +# "name": "Process_category_number_one_call_activity_call_activity_test_bd2e724", +# "description": "Process_category_number_one_call_activity_call_activity_test_bd2e724", +# "file": "call_activity_test.bpmn", +# "task_specs": {} + +# ### definition +# "Root": { +# "id": "Process_category_number_one_call_activity_call_activity_test_bd2e724_8", +# "name": "Root", +# "description": "", +# "manual": false, +# "internal": false, +# "lookahead": 2, +# "inputs": [], +# "outputs": [], +# "typename": "Simple" +# } +# ## runtime +# "a56e1403-2838-4f03-a31f-f99afe16f38d": { +# "id": "a56e1403-2838-4f03-a31f-f99afe16f38d", +# "parent": null, +# "children": [ +# "af6ba340-71e7-46d7-b2d4-e3db1751785d" +# ], +# "last_state_change": 1677775475.18116, +# "state": 32, +# "task_spec": "Root", +# "triggered": false, +# "workflow_name": "Process_category_number_one_call_activity_call_activity_test_bd2e724", +# "internal_data": {}, +# "data": {} +# }, + + +# ### definition +# "StartEvent_1": { +# "id": "Process_category_number_one_call_activity_call_activity_test_bd2e724_4", +# "name": "StartEvent_1", +# "description": null, +# "manual": false, +# "internal": false, +# "lookahead": 2, +# "inputs": [ +# "Start" +# ], +# "outputs": [ +# "same_process_model" +# ], +# "lane": null, +# "documentation": null, +# "position": { +# "x": 179, +# "y": 159 +# }, +# "data_input_associations": [], +# "data_output_associations": [], +# "io_specification": null, +# "event_definition": { +# "internal": false, +# "external": false, +# "typename": "NoneEventDefinition" +# }, +# "typename": "StartEvent", +# "extensions": {} +# }, +# ## runtime +# "b86b5552-c541-4afe-b200-db0190439f38": { +# "id": "b86b5552-c541-4afe-b200-db0190439f38", +# "parent": "af6ba340-71e7-46d7-b2d4-e3db1751785d", +# "children": [ +# "e9525f55-794f-450d-b5da-bee1951f99fc" +# ], +# "last_state_change": 1677775475.1963174, +# "state": 32, +# "task_spec": "StartEvent_1", +# "triggered": false, +# "workflow_name": "Process_category_number_one_call_activity_call_activity_test_bd2e724", +# "internal_data": { +# "event_fired": true +# }, +# "data": {} +# }, @dataclass class SpiffStepDetailsModel(SpiffworkflowBaseDBModel): diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/task.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/task.py index 148df2310..326ea5b45 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/task.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/task.py @@ -1,5 +1,10 @@ """Task.""" import enum +from dataclasses import dataclass +from sqlalchemy import ForeignKey +from spiffworkflow_backend.models.bpmn_process import BpmnProcessModel +from spiffworkflow_backend.models.db import db +from spiffworkflow_backend.models.db import SpiffworkflowBaseDBModel from typing import Any from typing import Optional from typing import Union @@ -7,7 +12,8 @@ from typing import Union import marshmallow from marshmallow import Schema from marshmallow_enum import EnumField # type: ignore -from SpiffWorkflow.task import TaskStateNames # type: ignore +from SpiffWorkflow.task import TaskStateNames +from spiffworkflow_backend.models.task_definition import TaskDefinitionModel # type: ignore class MultiInstanceType(enum.Enum): @@ -19,6 +25,36 @@ class MultiInstanceType(enum.Enum): sequential = "sequential" +# properties_json attributes: +# "id": "a56e1403-2838-4f03-a31f-f99afe16f38d", +# "parent": null, +# "children": [ +# "af6ba340-71e7-46d7-b2d4-e3db1751785d" +# ], +# "last_state_change": 1677775475.18116, +# "state": 32, +# "task_spec": "Root", +# "triggered": false, +# "workflow_name": "Process_category_number_one_call_activity_call_activity_test_bd2e724", +# "internal_data": {}, +@dataclass +class TaskModel(SpiffworkflowBaseDBModel): + __tablename__ = "task" + id: int = db.Column(db.Integer, primary_key=True) + guid: str = db.Column(db.String(36), nullable=False, unique=True, index=True) + bpmn_process_id: int = db.Column( + ForeignKey(BpmnProcessModel.id), nullable=False # type: ignore + ) + + # find this by looking up the "workflow_name" and "task_spec" from the properties_json + task_definition_id: int = db.Column( + ForeignKey(TaskDefinitionModel.id), nullable=False # type: ignore + ) + state: str = db.Column(db.String(10), nullable=False) + properties_json: dict = db.Column(db.JSON, nullable=False) + json_data_hash: str = db.Column(db.String(255), nullable=False, index=True) + + class Task: """Task.""" diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/task_definition.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/task_definition.py new file mode 100644 index 000000000..7e809d3fd --- /dev/null +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/task_definition.py @@ -0,0 +1,23 @@ +from __future__ import annotations +from sqlalchemy import UniqueConstraint +from sqlalchemy import ForeignKey +from spiffworkflow_backend.models.bpmn_process_definition import BpmnProcessDefinitionModel + +from spiffworkflow_backend.models.db import db +from spiffworkflow_backend.models.db import SpiffworkflowBaseDBModel + + +class TaskDefinitionModel(SpiffworkflowBaseDBModel): + __tablename__ = "task_definition" + __table_args__ = ( + UniqueConstraint( + "bpmn_process_definition_id", "bpmn_identifier", name="task_definition_unique" + ), + ) + + id: int = db.Column(db.Integer, primary_key=True) + bpmn_process_definition_id: int = db.Column(ForeignKey(BpmnProcessDefinitionModel.id), nullable=False) # type: ignore + + bpmn_identifier: str = db.Column(db.String(255), nullable=False, index=True) + properties_json: dict = db.Column(db.JSON, nullable=False) + typename: str = db.Column(db.String(255), nullable=False) From 8d67e8cc8738ea967ff71e78c8221cfbdb09aacc Mon Sep 17 00:00:00 2001 From: jasquat Date: Thu, 2 Mar 2023 17:28:31 -0500 Subject: [PATCH 02/25] we can save the top level spec to the database with its tasks w/ burnettk --- .../migrations/versions/2fe2830f45e1_.py | 44 ++++++++++ .../migrations/versions/317dd5155137_.py | 85 +++++++++++++++++++ .../load_database_models.py | 6 ++ .../models/bpmn_process.py | 2 +- .../models/bpmn_process_definition.py | 1 + .../models/task_definition.py | 2 + .../services/process_instance_processor.py | 81 ++++++++++++++++++ 7 files changed, 220 insertions(+), 1 deletion(-) create mode 100644 spiffworkflow-backend/migrations/versions/2fe2830f45e1_.py create mode 100644 spiffworkflow-backend/migrations/versions/317dd5155137_.py diff --git a/spiffworkflow-backend/migrations/versions/2fe2830f45e1_.py b/spiffworkflow-backend/migrations/versions/2fe2830f45e1_.py new file mode 100644 index 000000000..57072697d --- /dev/null +++ b/spiffworkflow-backend/migrations/versions/2fe2830f45e1_.py @@ -0,0 +1,44 @@ +"""empty message + +Revision ID: 2fe2830f45e1 +Revises: 317dd5155137 +Create Date: 2023-03-02 17:19:08.535027 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '2fe2830f45e1' +down_revision = '317dd5155137' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('json_data', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('hash', sa.String(length=255), nullable=False), + sa.Column('data', sa.JSON(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_json_data_hash'), 'json_data', ['hash'], unique=True) + op.create_table('bpmn_process_definition_relationship', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('bpmn_process_definition_parent_id', sa.Integer(), nullable=False), + sa.Column('bpmn_process_definition_child_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['bpmn_process_definition_child_id'], ['bpmn_process_definition.id'], ), + sa.ForeignKeyConstraint(['bpmn_process_definition_parent_id'], ['bpmn_process_definition.id'], ), + sa.PrimaryKeyConstraint('id') + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('bpmn_process_definition_relationship') + op.drop_index(op.f('ix_json_data_hash'), table_name='json_data') + op.drop_table('json_data') + # ### end Alembic commands ### diff --git a/spiffworkflow-backend/migrations/versions/317dd5155137_.py b/spiffworkflow-backend/migrations/versions/317dd5155137_.py new file mode 100644 index 000000000..e64254c7e --- /dev/null +++ b/spiffworkflow-backend/migrations/versions/317dd5155137_.py @@ -0,0 +1,85 @@ +"""empty message + +Revision ID: 317dd5155137 +Revises: 8930711a75a4 +Create Date: 2023-03-02 17:16:15.687837 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '317dd5155137' +down_revision = '8930711a75a4' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('bpmn_process', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('parent_process_id', sa.Integer(), nullable=True), + sa.Column('properties_json', sa.JSON(), nullable=False), + sa.Column('json_data_hash', sa.String(length=255), nullable=False), + sa.Column('process_type', sa.String(length=30), nullable=False), + sa.ForeignKeyConstraint(['parent_process_id'], ['bpmn_process.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_bpmn_process_json_data_hash'), 'bpmn_process', ['json_data_hash'], unique=False) + op.create_table('bpmn_process_definition', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('hash', sa.String(length=255), nullable=False), + sa.Column('bpmn_identifier', sa.String(length=255), nullable=False), + sa.Column('properties_json', sa.JSON(), nullable=False), + sa.Column('type', sa.String(length=32), nullable=False), + sa.Column('bpmn_version_control_type', sa.String(length=50), nullable=True), + sa.Column('bpmn_version_control_identifier', sa.String(length=255), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_bpmn_process_definition_bpmn_identifier'), 'bpmn_process_definition', ['bpmn_identifier'], unique=False) + op.create_index(op.f('ix_bpmn_process_definition_hash'), 'bpmn_process_definition', ['hash'], unique=True) + op.create_index(op.f('ix_bpmn_process_definition_type'), 'bpmn_process_definition', ['type'], unique=False) + op.create_table('task_definition', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('bpmn_process_definition_id', sa.Integer(), nullable=False), + sa.Column('bpmn_identifier', sa.String(length=255), nullable=False), + sa.Column('properties_json', sa.JSON(), nullable=False), + sa.Column('typename', sa.String(length=255), nullable=False), + sa.ForeignKeyConstraint(['bpmn_process_definition_id'], ['bpmn_process_definition.id'], ), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('bpmn_process_definition_id', 'bpmn_identifier', name='task_definition_unique') + ) + op.create_index(op.f('ix_task_definition_bpmn_identifier'), 'task_definition', ['bpmn_identifier'], unique=False) + op.create_table('task', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('guid', sa.String(length=36), nullable=False), + sa.Column('bpmn_process_id', sa.Integer(), nullable=False), + sa.Column('task_definition_id', sa.Integer(), nullable=False), + sa.Column('state', sa.String(length=10), nullable=False), + sa.Column('properties_json', sa.JSON(), nullable=False), + sa.Column('json_data_hash', sa.String(length=255), nullable=False), + sa.ForeignKeyConstraint(['bpmn_process_id'], ['bpmn_process.id'], ), + sa.ForeignKeyConstraint(['task_definition_id'], ['task_definition.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_task_guid'), 'task', ['guid'], unique=True) + op.create_index(op.f('ix_task_json_data_hash'), 'task', ['json_data_hash'], unique=False) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f('ix_task_json_data_hash'), table_name='task') + op.drop_index(op.f('ix_task_guid'), table_name='task') + op.drop_table('task') + op.drop_index(op.f('ix_task_definition_bpmn_identifier'), table_name='task_definition') + op.drop_table('task_definition') + op.drop_index(op.f('ix_bpmn_process_definition_type'), table_name='bpmn_process_definition') + op.drop_index(op.f('ix_bpmn_process_definition_hash'), table_name='bpmn_process_definition') + op.drop_index(op.f('ix_bpmn_process_definition_bpmn_identifier'), table_name='bpmn_process_definition') + op.drop_table('bpmn_process_definition') + op.drop_index(op.f('ix_bpmn_process_json_data_hash'), table_name='bpmn_process') + op.drop_table('bpmn_process') + # ### end Alembic commands ### diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/load_database_models.py b/spiffworkflow-backend/src/spiffworkflow_backend/load_database_models.py index 9447bcd95..ab518fa54 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/load_database_models.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/load_database_models.py @@ -57,5 +57,11 @@ from spiffworkflow_backend.models.process_instance_data import ( ProcessInstanceDataModel, ) # noqa: F401 +from spiffworkflow_backend.models.bpmn_process import BpmnProcessModel # noqa: F401 +from spiffworkflow_backend.models.bpmn_process_definition import BpmnProcessDefinitionModel # noqa: F401 +from spiffworkflow_backend.models.task import TaskModel # noqa: F401 +from spiffworkflow_backend.models.task_definition import TaskDefinitionModel # noqa: F401 +from spiffworkflow_backend.models.json_data import JsonDataModel # noqa: F401 +from spiffworkflow_backend.models.bpmn_process_definition_relationship import BpmnProcessDefinitionRelationshipModel # noqa: F401 add_listeners() diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process.py index b704ce92e..05519d4b4 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process.py @@ -16,7 +16,7 @@ class BpmnProcessModel(SpiffworkflowBaseDBModel): __tablename__ = "bpmn_process" id: int = db.Column(db.Integer, primary_key=True) - parent_process_id: int = db.Column(ForeignKey("BpmnProcessModel.id"), nullable=True) + parent_process_id: int = db.Column(ForeignKey("bpmn_process.id"), nullable=True) properties_json: dict = db.Column(db.JSON, nullable=False) json_data_hash: str = db.Column(db.String(255), nullable=False, index=True) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process_definition.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process_definition.py index 54d70c9fc..348aae5d1 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process_definition.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process_definition.py @@ -16,6 +16,7 @@ class BpmnProcessDefinitionModel(SpiffworkflowBaseDBModel): # this is a sha256 hash of spec and serializer_version hash: str = db.Column(db.String(255), nullable=False, index=True, unique=True) + bpmn_identifier: str = db.Column(db.String(255), nullable=False, index=True) properties_json: str = db.Column(db.JSON, nullable=False) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/task_definition.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/task_definition.py index 7e809d3fd..d6e2061d1 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/task_definition.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/task_definition.py @@ -1,4 +1,5 @@ from __future__ import annotations +from sqlalchemy.orm import relationship from sqlalchemy import UniqueConstraint from sqlalchemy import ForeignKey from spiffworkflow_backend.models.bpmn_process_definition import BpmnProcessDefinitionModel @@ -17,6 +18,7 @@ class TaskDefinitionModel(SpiffworkflowBaseDBModel): id: int = db.Column(db.Integer, primary_key=True) bpmn_process_definition_id: int = db.Column(ForeignKey(BpmnProcessDefinitionModel.id), nullable=False) # type: ignore + bpmn_process_definition = relationship(BpmnProcessDefinitionModel) bpmn_identifier: str = db.Column(db.String(255), nullable=False, index=True) properties_json: dict = db.Column(db.JSON, nullable=False) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py b/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py index 8cd663af0..172b9542b 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py @@ -55,6 +55,7 @@ from SpiffWorkflow.util.deep_merge import DeepMerge # type: ignore from sqlalchemy import text from spiffworkflow_backend.exceptions.api_error import ApiError +from spiffworkflow_backend.models.bpmn_process_definition import BpmnProcessDefinitionModel from spiffworkflow_backend.models.db import db from spiffworkflow_backend.models.file import File from spiffworkflow_backend.models.file import FileType @@ -80,6 +81,7 @@ from spiffworkflow_backend.models.serialized_bpmn_definition import ( ) # noqa: F401 from spiffworkflow_backend.models.spec_reference import SpecReferenceCache from spiffworkflow_backend.models.spiff_step_details import SpiffStepDetailsModel +from spiffworkflow_backend.models.task_definition import TaskDefinitionModel from spiffworkflow_backend.models.user import UserModel from spiffworkflow_backend.scripts.script import Script from spiffworkflow_backend.services.custom_parser import MyCustomParser @@ -906,9 +908,88 @@ class ProcessInstanceProcessor: db.session.add(process_instance_data) self.process_instance_model.process_instance_data = process_instance_data + + def _store_bpmn_process_definitions(self, process_bpmn_properties: dict) -> None: + # for process_bpmn_identifier, process_bpmn_properties in bpmn_spec_dict.items(): + print(f"process_bpmn_properties: {process_bpmn_properties}") + process_bpmn_identifier = process_bpmn_properties['name'] + new_hash_digest = sha256( + json.dumps(process_bpmn_properties, sort_keys=True).encode("utf8") + ).hexdigest() + bpmn_process_definition = BpmnProcessDefinitionModel.query.filter_by( + hash=new_hash_digest + ).first() + if bpmn_process_definition is None: + # print(f"process_bpmn_identifier: {process_bpmn_identifier}") + print(f"process_bpmn_properties: {process_bpmn_properties}") + task_specs = process_bpmn_properties.pop("task_specs") + bpmn_process_definition = BpmnProcessDefinitionModel( + hash=new_hash_digest, bpmn_identifier=process_bpmn_identifier, properties_json=json.dumps(process_bpmn_properties), type="process" + ) + db.session.add(bpmn_process_definition) + + for task_bpmn_identifier, task_bpmn_properties in task_specs.items(): + task_definition = TaskDefinitionModel( + bpmn_process_definition=bpmn_process_definition, + bpmn_identifier=task_bpmn_identifier, + properties_json=json.dumps(task_bpmn_properties), + typename=task_bpmn_properties['typename'], + ) + db.session.add(task_definition) + + def _add_bpmn_json_records_new(self) -> None: + """Adds serialized_bpmn_definition and process_instance_data records to the db session. + + Expects the save method to commit it. + """ + bpmn_dict = json.loads(self.serialize()) + bpmn_dict_keys = ("spec", "subprocess_specs", "serializer_version") + bpmn_spec_dict = {} + process_instance_data_dict = {} + for bpmn_key in bpmn_dict.keys(): + if bpmn_key in bpmn_dict_keys: + bpmn_spec_dict[bpmn_key] = bpmn_dict[bpmn_key] + else: + process_instance_data_dict[bpmn_key] = bpmn_dict[bpmn_key] + + self._store_bpmn_process_definitions(bpmn_spec_dict['spec']) + + # # FIXME: always save new hash until we get updated Spiff without loopresettask + # # if self.process_instance_model.serialized_bpmn_definition_id is None: + # new_hash_digest = sha256( + # json.dumps(bpmn_spec_dict, sort_keys=True).encode("utf8") + # ).hexdigest() + # serialized_bpmn_definition = SerializedBpmnDefinitionModel.query.filter_by( + # hash=new_hash_digest + # ).first() + # if serialized_bpmn_definition is None: + # serialized_bpmn_definition = SerializedBpmnDefinitionModel( + # hash=new_hash_digest, static_json=json.dumps(bpmn_spec_dict) + # ) + # db.session.add(serialized_bpmn_definition) + # if ( + # self.process_instance_model.serialized_bpmn_definition_id is None + # or self.process_instance_model.serialized_bpmn_definition.hash + # != new_hash_digest + # ): + # self.process_instance_model.serialized_bpmn_definition = ( + # serialized_bpmn_definition + # ) + # + # process_instance_data = None + # if self.process_instance_model.process_instance_data_id is None: + # process_instance_data = ProcessInstanceDataModel() + # else: + # process_instance_data = self.process_instance_model.process_instance_data + # + # process_instance_data.runtime_json = json.dumps(process_instance_data_dict) + # db.session.add(process_instance_data) + # self.process_instance_model.process_instance_data = process_instance_data + def save(self) -> None: """Saves the current state of this processor to the database.""" self._add_bpmn_json_records() + self._add_bpmn_json_records_new() complete_states = [TaskState.CANCELLED, TaskState.COMPLETED] user_tasks = list(self.get_all_user_tasks()) From 0c6e9a63ba1430cb4b3e2bb7cc76255567e5051c Mon Sep 17 00:00:00 2001 From: jasquat Date: Fri, 3 Mar 2023 10:08:14 -0500 Subject: [PATCH 03/25] unit tests are passing with the new spec tables --- .../migrations/versions/567d22ded3af_.py | 28 +++++++++ .../migrations/versions/6315ff2525b0_.py | 30 ++++++++++ .../migrations/versions/def2cbb0ca6b_.py | 32 ++++++++++ .../models/bpmn_process_definition.py | 1 + .../bpmn_process_definition_relationship.py | 7 +++ .../models/process_instance.py | 4 ++ .../services/process_instance_processor.py | 59 +++++++++++++++---- .../tests/data/dot_notation/diagram.bpmn | 12 ++-- .../integration/test_process_api.py | 3 +- .../unit/test_dot_notation.py | 4 +- 10 files changed, 160 insertions(+), 20 deletions(-) create mode 100644 spiffworkflow-backend/migrations/versions/567d22ded3af_.py create mode 100644 spiffworkflow-backend/migrations/versions/6315ff2525b0_.py create mode 100644 spiffworkflow-backend/migrations/versions/def2cbb0ca6b_.py diff --git a/spiffworkflow-backend/migrations/versions/567d22ded3af_.py b/spiffworkflow-backend/migrations/versions/567d22ded3af_.py new file mode 100644 index 000000000..1e41244d2 --- /dev/null +++ b/spiffworkflow-backend/migrations/versions/567d22ded3af_.py @@ -0,0 +1,28 @@ +"""empty message + +Revision ID: 567d22ded3af +Revises: 2fe2830f45e1 +Create Date: 2023-03-03 08:38:25.855923 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '567d22ded3af' +down_revision = '2fe2830f45e1' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_unique_constraint('bpmn_process_definition_relationship_unique', 'bpmn_process_definition_relationship', ['bpmn_process_definition_parent_id', 'bpmn_process_definition_child_id']) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint('bpmn_process_definition_relationship_unique', 'bpmn_process_definition_relationship', type_='unique') + # ### end Alembic commands ### diff --git a/spiffworkflow-backend/migrations/versions/6315ff2525b0_.py b/spiffworkflow-backend/migrations/versions/6315ff2525b0_.py new file mode 100644 index 000000000..d21097a2c --- /dev/null +++ b/spiffworkflow-backend/migrations/versions/6315ff2525b0_.py @@ -0,0 +1,30 @@ +"""empty message + +Revision ID: 6315ff2525b0 +Revises: 567d22ded3af +Create Date: 2023-03-03 09:19:28.098057 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '6315ff2525b0' +down_revision = '567d22ded3af' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('process_instance', sa.Column('bpmn_process_definition_id', sa.Integer(), nullable=False)) + op.create_foreign_key(None, 'process_instance', 'bpmn_process_definition', ['bpmn_process_definition_id'], ['id']) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint(None, 'process_instance', type_='foreignkey') + op.drop_column('process_instance', 'bpmn_process_definition_id') + # ### end Alembic commands ### diff --git a/spiffworkflow-backend/migrations/versions/def2cbb0ca6b_.py b/spiffworkflow-backend/migrations/versions/def2cbb0ca6b_.py new file mode 100644 index 000000000..33bb26948 --- /dev/null +++ b/spiffworkflow-backend/migrations/versions/def2cbb0ca6b_.py @@ -0,0 +1,32 @@ +"""empty message + +Revision ID: def2cbb0ca6b +Revises: 6315ff2525b0 +Create Date: 2023-03-03 09:23:19.480250 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import mysql + +# revision identifiers, used by Alembic. +revision = 'def2cbb0ca6b' +down_revision = '6315ff2525b0' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('process_instance', 'bpmn_process_definition_id', + existing_type=mysql.INTEGER(), + nullable=True) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('process_instance', 'bpmn_process_definition_id', + existing_type=mysql.INTEGER(), + nullable=False) + # ### end Alembic commands ### diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process_definition.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process_definition.py index 348aae5d1..d60a6c0dd 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process_definition.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process_definition.py @@ -21,6 +21,7 @@ class BpmnProcessDefinitionModel(SpiffworkflowBaseDBModel): properties_json: str = db.Column(db.JSON, nullable=False) # process or subprocess + # FIXME: will probably ignore for now since we do not strictly need it type: str = db.Column(db.String(32), nullable=False, index=True) # TODO: remove these from process_instance diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process_definition_relationship.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process_definition_relationship.py index 80f5c9bb2..218d226a7 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process_definition_relationship.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process_definition_relationship.py @@ -1,4 +1,5 @@ from __future__ import annotations +from sqlalchemy import UniqueConstraint from sqlalchemy.orm import deferred from sqlalchemy import ForeignKey from spiffworkflow_backend.models.bpmn_process_definition import BpmnProcessDefinitionModel @@ -9,6 +10,12 @@ from spiffworkflow_backend.models.db import SpiffworkflowBaseDBModel class BpmnProcessDefinitionRelationshipModel(SpiffworkflowBaseDBModel): __tablename__ = "bpmn_process_definition_relationship" + __table_args__ = ( + UniqueConstraint( + "bpmn_process_definition_parent_id", "bpmn_process_definition_child_id", name="bpmn_process_definition_relationship_unique" + ), + ) + id: int = db.Column(db.Integer, primary_key=True) bpmn_process_definition_parent_id: int = db.Column(ForeignKey(BpmnProcessDefinitionModel.id), nullable=False) # type: ignore bpmn_process_definition_child_id: int = db.Column(ForeignKey(BpmnProcessDefinitionModel.id), nullable=False) # type: ignore diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/process_instance.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/process_instance.py index f12cbec7c..4cfb52b7b 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/process_instance.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/process_instance.py @@ -1,5 +1,6 @@ """Process_instance.""" from __future__ import annotations +from spiffworkflow_backend.models.bpmn_process_definition import BpmnProcessDefinitionModel from typing import Any from typing import cast @@ -74,6 +75,9 @@ class ProcessInstanceModel(SpiffworkflowBaseDBModel): ) process_instance_data = relationship("ProcessInstanceDataModel", cascade="delete") + bpmn_process_definition_id: int = db.Column(ForeignKey(BpmnProcessDefinitionModel.id), nullable=True) # type: ignore + bpmn_process_definition = relationship(BpmnProcessDefinitionModel) + active_human_tasks = relationship( "HumanTaskModel", primaryjoin=( diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py b/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py index 172b9542b..aaa8a9c35 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py @@ -1,5 +1,7 @@ """Process_instance_processor.""" -import _strptime # type: ignore +import _strptime +from spiffworkflow_backend.models import serialized_bpmn_definition # type: ignore +from spiffworkflow_backend.models.bpmn_process_definition_relationship import BpmnProcessDefinitionRelationshipModel # noqa: F401 import decimal import json import logging @@ -55,6 +57,7 @@ from SpiffWorkflow.util.deep_merge import DeepMerge # type: ignore from sqlalchemy import text from spiffworkflow_backend.exceptions.api_error import ApiError +from spiffworkflow_backend.models import bpmn_process_definition_relationship from spiffworkflow_backend.models.bpmn_process_definition import BpmnProcessDefinitionModel from spiffworkflow_backend.models.db import db from spiffworkflow_backend.models.file import File @@ -526,9 +529,32 @@ class ProcessInstanceProcessor: def _get_full_bpmn_json(cls, process_instance_model: ProcessInstanceModel) -> dict: if process_instance_model.serialized_bpmn_definition_id is None: return {} - serialized_bpmn_definition = process_instance_model.serialized_bpmn_definition + # serialized_bpmn_definition = process_instance_model.serialized_bpmn_definition + # print(f"serialized_bpmn_definition.static_json: {serialized_bpmn_definition.static_json}") + # loaded_json: dict = json.loads(serialized_bpmn_definition.static_json) # or "{}") + + serialized_bpmn_definition = {} + bpmn_process_definition = BpmnProcessDefinitionModel.query.filter_by(id=process_instance_model.bpmn_process_definition_id).first() + task_definitions = TaskDefinitionModel.query.filter_by(bpmn_process_definition_id=process_instance_model.bpmn_process_definition_id).all() + if bpmn_process_definition is not None: + serialized_bpmn_definition = {"serializer_version": cls.SERIALIZER_VERSION, "spec": {}, "subprocess_specs": {}} + bpmn_process_definition_dict = json.loads(bpmn_process_definition.properties_json) + bpmn_process_definition_dict['task_specs'] = {} + for task_definition in task_definitions: + bpmn_process_definition_dict['task_specs'][task_definition.bpmn_identifier] = json.loads(task_definition.properties_json) + serialized_bpmn_definition['spec'] = bpmn_process_definition_dict + + bpmn_process_subprocess_definitions = BpmnProcessDefinitionRelationshipModel.query.filter_by(bpmn_process_definition_parent_id=bpmn_process_definition.id).all() + for bpmn_process_subprocess_definition in bpmn_process_subprocess_definitions: + subprocess_task_definitions = TaskDefinitionModel.query.filter_by(bpmn_process_definition_id=bpmn_process_subprocess_definition.id).all() + bpmn_process_subprocess_dict = json.loads(bpmn_process_definition.properties_json) + bpmn_process_subprocess_dict['task_specs'] = {} + for subprocess_task_definition in subprocess_task_definitions: + bpmn_process_subprocess_dict['task_specs'][subprocess_task_definition.bpmn_identifier] = json.loads(subprocess_task_definition.properties_json) + serialized_bpmn_definition['subprocess_specs'][bpmn_process_subprocess_definition.bpmn_identifier] = bpmn_process_subprocess_dict + loaded_json: dict = serialized_bpmn_definition + process_instance_data = process_instance_model.process_instance_data - loaded_json: dict = json.loads(serialized_bpmn_definition.static_json or "{}") loaded_json.update(json.loads(process_instance_data.runtime_json)) return loaded_json @@ -909,19 +935,15 @@ class ProcessInstanceProcessor: self.process_instance_model.process_instance_data = process_instance_data - def _store_bpmn_process_definitions(self, process_bpmn_properties: dict) -> None: - # for process_bpmn_identifier, process_bpmn_properties in bpmn_spec_dict.items(): - print(f"process_bpmn_properties: {process_bpmn_properties}") + def _store_bpmn_process_definition(self, process_bpmn_properties: dict, bpmn_process_definition_parent: Optional[BpmnProcessDefinitionModel] = None) -> BpmnProcessDefinitionModel: process_bpmn_identifier = process_bpmn_properties['name'] new_hash_digest = sha256( json.dumps(process_bpmn_properties, sort_keys=True).encode("utf8") ).hexdigest() - bpmn_process_definition = BpmnProcessDefinitionModel.query.filter_by( + bpmn_process_definition: Optional[BpmnProcessDefinitionModel] = BpmnProcessDefinitionModel.query.filter_by( hash=new_hash_digest ).first() if bpmn_process_definition is None: - # print(f"process_bpmn_identifier: {process_bpmn_identifier}") - print(f"process_bpmn_properties: {process_bpmn_properties}") task_specs = process_bpmn_properties.pop("task_specs") bpmn_process_definition = BpmnProcessDefinitionModel( hash=new_hash_digest, bpmn_identifier=process_bpmn_identifier, properties_json=json.dumps(process_bpmn_properties), type="process" @@ -937,6 +959,19 @@ class ProcessInstanceProcessor: ) db.session.add(task_definition) + if bpmn_process_definition_parent: + bpmn_process_definition_relationship = BpmnProcessDefinitionRelationshipModel.query.filter_by( + bpmn_process_definition_parent_id=bpmn_process_definition_parent.id, + bpmn_process_definition_child_id=bpmn_process_definition.id, + ).first() + if bpmn_process_definition_relationship is None: + bpmn_process_definition_relationship = BpmnProcessDefinitionRelationshipModel( + bpmn_process_definition_parent_id=bpmn_process_definition_parent.id, + bpmn_process_definition_child_id=bpmn_process_definition.id, + ) + db.session.add(bpmn_process_definition_relationship) + return bpmn_process_definition + def _add_bpmn_json_records_new(self) -> None: """Adds serialized_bpmn_definition and process_instance_data records to the db session. @@ -952,7 +987,11 @@ class ProcessInstanceProcessor: else: process_instance_data_dict[bpmn_key] = bpmn_dict[bpmn_key] - self._store_bpmn_process_definitions(bpmn_spec_dict['spec']) + bpmn_process_definition_parent = self._store_bpmn_process_definition(bpmn_spec_dict['spec']) + for process_bpmn_properties in bpmn_spec_dict['subprocess_specs'].values(): + self._store_bpmn_process_definition(process_bpmn_properties, bpmn_process_definition_parent) + self.process_instance_model.bpmn_process_definition = bpmn_process_definition_parent + # # FIXME: always save new hash until we get updated Spiff without loopresettask # # if self.process_instance_model.serialized_bpmn_definition_id is None: diff --git a/spiffworkflow-backend/tests/data/dot_notation/diagram.bpmn b/spiffworkflow-backend/tests/data/dot_notation/diagram.bpmn index 96bdce894..5d6829420 100644 --- a/spiffworkflow-backend/tests/data/dot_notation/diagram.bpmn +++ b/spiffworkflow-backend/tests/data/dot_notation/diagram.bpmn @@ -24,14 +24,14 @@ - + Flow_0dbnzbi - - + + Flow_0nt355i - + @@ -53,13 +53,13 @@ - + - + diff --git a/spiffworkflow-backend/tests/spiffworkflow_backend/integration/test_process_api.py b/spiffworkflow-backend/tests/spiffworkflow_backend/integration/test_process_api.py index 3055d4b2c..f450729ce 100644 --- a/spiffworkflow-backend/tests/spiffworkflow_backend/integration/test_process_api.py +++ b/spiffworkflow-backend/tests/spiffworkflow_backend/integration/test_process_api.py @@ -1301,10 +1301,11 @@ class TestProcessApi(BaseTest): assert create_response.json is not None assert create_response.status_code == 201 process_instance_id = create_response.json["id"] - client.post( + run_response = client.post( f"/v1.0/process-instances/{modified_process_model_identifier}/{process_instance_id}/run", headers=self.logged_in_headers(with_super_admin_user), ) + assert run_response.status_code == 200 show_response = client.get( f"/v1.0/process-instances/{modified_process_model_identifier}/{process_instance_id}?process_identifier={spec_reference.identifier}", headers=self.logged_in_headers(with_super_admin_user), diff --git a/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_dot_notation.py b/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_dot_notation.py index 59a0fee8d..c646a7540 100644 --- a/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_dot_notation.py +++ b/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_dot_notation.py @@ -13,9 +13,7 @@ from spiffworkflow_backend.services.process_instance_service import ( class TestDotNotation(BaseTest): - """TestVariousBpmnConstructs.""" - - def test_dot_notation( + def test_dot_notation_in_message_path( self, app: Flask, client: FlaskClient, From e7950456e92af58b6228f551dfcc53f6124cf7d7 Mon Sep 17 00:00:00 2001 From: jasquat Date: Fri, 3 Mar 2023 10:20:26 -0500 Subject: [PATCH 04/25] all tests are passing with new spec tables w/ burnettk --- .../load_database_models.py | 12 +- .../models/bpmn_process.py | 2 +- .../bpmn_process_definition_relationship.py | 22 ++-- .../spiffworkflow_backend/models/json_data.py | 3 - .../models/process_instance.py | 8 +- .../models/spiff_step_details.py | 1 + .../src/spiffworkflow_backend/models/task.py | 9 +- .../models/task_definition.py | 19 ++- .../services/process_instance_processor.py | 110 ++++++++++++------ 9 files changed, 123 insertions(+), 63 deletions(-) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/load_database_models.py b/spiffworkflow-backend/src/spiffworkflow_backend/load_database_models.py index ab518fa54..490cf0b1f 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/load_database_models.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/load_database_models.py @@ -58,10 +58,16 @@ from spiffworkflow_backend.models.process_instance_data import ( ) # noqa: F401 from spiffworkflow_backend.models.bpmn_process import BpmnProcessModel # noqa: F401 -from spiffworkflow_backend.models.bpmn_process_definition import BpmnProcessDefinitionModel # noqa: F401 +from spiffworkflow_backend.models.bpmn_process_definition import ( + BpmnProcessDefinitionModel, +) # noqa: F401 from spiffworkflow_backend.models.task import TaskModel # noqa: F401 -from spiffworkflow_backend.models.task_definition import TaskDefinitionModel # noqa: F401 +from spiffworkflow_backend.models.task_definition import ( + TaskDefinitionModel, +) # noqa: F401 from spiffworkflow_backend.models.json_data import JsonDataModel # noqa: F401 -from spiffworkflow_backend.models.bpmn_process_definition_relationship import BpmnProcessDefinitionRelationshipModel # noqa: F401 +from spiffworkflow_backend.models.bpmn_process_definition_relationship import ( + BpmnProcessDefinitionRelationshipModel, +) # noqa: F401 add_listeners() diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process.py index 05519d4b4..948cdffd1 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process.py @@ -1,6 +1,6 @@ from __future__ import annotations + from sqlalchemy import ForeignKey -from spiffworkflow_backend.models.bpmn_process_definition import BpmnProcessDefinitionModel from spiffworkflow_backend.models.db import db from spiffworkflow_backend.models.db import SpiffworkflowBaseDBModel diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process_definition_relationship.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process_definition_relationship.py index 218d226a7..8f6fdca2e 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process_definition_relationship.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process_definition_relationship.py @@ -1,9 +1,11 @@ from __future__ import annotations -from sqlalchemy import UniqueConstraint -from sqlalchemy.orm import deferred -from sqlalchemy import ForeignKey -from spiffworkflow_backend.models.bpmn_process_definition import BpmnProcessDefinitionModel +from sqlalchemy import ForeignKey +from sqlalchemy import UniqueConstraint + +from spiffworkflow_backend.models.bpmn_process_definition import ( + BpmnProcessDefinitionModel, +) from spiffworkflow_backend.models.db import db from spiffworkflow_backend.models.db import SpiffworkflowBaseDBModel @@ -12,10 +14,16 @@ class BpmnProcessDefinitionRelationshipModel(SpiffworkflowBaseDBModel): __tablename__ = "bpmn_process_definition_relationship" __table_args__ = ( UniqueConstraint( - "bpmn_process_definition_parent_id", "bpmn_process_definition_child_id", name="bpmn_process_definition_relationship_unique" + "bpmn_process_definition_parent_id", + "bpmn_process_definition_child_id", + name="bpmn_process_definition_relationship_unique", ), ) id: int = db.Column(db.Integer, primary_key=True) - bpmn_process_definition_parent_id: int = db.Column(ForeignKey(BpmnProcessDefinitionModel.id), nullable=False) # type: ignore - bpmn_process_definition_child_id: int = db.Column(ForeignKey(BpmnProcessDefinitionModel.id), nullable=False) # type: ignore + bpmn_process_definition_parent_id: int = db.Column( + ForeignKey(BpmnProcessDefinitionModel.id), nullable=False + ) # type: ignore + bpmn_process_definition_child_id: int = db.Column( + ForeignKey(BpmnProcessDefinitionModel.id), nullable=False + ) # type: ignore diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/json_data.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/json_data.py index 1e03b9ff3..103bdc57b 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/json_data.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/json_data.py @@ -1,7 +1,4 @@ from __future__ import annotations -from sqlalchemy.orm import deferred -from sqlalchemy import ForeignKey -from spiffworkflow_backend.models.bpmn_process_definition import BpmnProcessDefinitionModel from spiffworkflow_backend.models.db import db from spiffworkflow_backend.models.db import SpiffworkflowBaseDBModel diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/process_instance.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/process_instance.py index 4cfb52b7b..44cfe8067 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/process_instance.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/process_instance.py @@ -1,6 +1,5 @@ """Process_instance.""" from __future__ import annotations -from spiffworkflow_backend.models.bpmn_process_definition import BpmnProcessDefinitionModel from typing import Any from typing import cast @@ -16,6 +15,9 @@ from sqlalchemy.orm import relationship from sqlalchemy.orm import validates from spiffworkflow_backend.helpers.spiff_enum import SpiffEnum +from spiffworkflow_backend.models.bpmn_process_definition import ( + BpmnProcessDefinitionModel, +) from spiffworkflow_backend.models.db import db from spiffworkflow_backend.models.db import SpiffworkflowBaseDBModel from spiffworkflow_backend.models.process_instance_data import ProcessInstanceDataModel @@ -75,7 +77,9 @@ class ProcessInstanceModel(SpiffworkflowBaseDBModel): ) process_instance_data = relationship("ProcessInstanceDataModel", cascade="delete") - bpmn_process_definition_id: int = db.Column(ForeignKey(BpmnProcessDefinitionModel.id), nullable=True) # type: ignore + bpmn_process_definition_id: int = db.Column( + ForeignKey(BpmnProcessDefinitionModel.id), nullable=True + ) # type: ignore bpmn_process_definition = relationship(BpmnProcessDefinitionModel) active_human_tasks = relationship( diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/spiff_step_details.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/spiff_step_details.py index 503b2a418..6b91bafa1 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/spiff_step_details.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/spiff_step_details.py @@ -149,6 +149,7 @@ from spiffworkflow_backend.models.process_instance import ProcessInstanceModel # "data": {} # }, + @dataclass class SpiffStepDetailsModel(SpiffworkflowBaseDBModel): """SpiffStepDetailsModel.""" diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/task.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/task.py index 326ea5b45..3fda6677f 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/task.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/task.py @@ -1,10 +1,6 @@ """Task.""" import enum from dataclasses import dataclass -from sqlalchemy import ForeignKey -from spiffworkflow_backend.models.bpmn_process import BpmnProcessModel -from spiffworkflow_backend.models.db import db -from spiffworkflow_backend.models.db import SpiffworkflowBaseDBModel from typing import Any from typing import Optional from typing import Union @@ -13,6 +9,11 @@ import marshmallow from marshmallow import Schema from marshmallow_enum import EnumField # type: ignore from SpiffWorkflow.task import TaskStateNames +from sqlalchemy import ForeignKey + +from spiffworkflow_backend.models.bpmn_process import BpmnProcessModel +from spiffworkflow_backend.models.db import db +from spiffworkflow_backend.models.db import SpiffworkflowBaseDBModel from spiffworkflow_backend.models.task_definition import TaskDefinitionModel # type: ignore diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/task_definition.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/task_definition.py index d6e2061d1..7d7c50873 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/task_definition.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/task_definition.py @@ -1,9 +1,12 @@ from __future__ import annotations -from sqlalchemy.orm import relationship -from sqlalchemy import UniqueConstraint -from sqlalchemy import ForeignKey -from spiffworkflow_backend.models.bpmn_process_definition import BpmnProcessDefinitionModel +from sqlalchemy import ForeignKey +from sqlalchemy import UniqueConstraint +from sqlalchemy.orm import relationship + +from spiffworkflow_backend.models.bpmn_process_definition import ( + BpmnProcessDefinitionModel, +) from spiffworkflow_backend.models.db import db from spiffworkflow_backend.models.db import SpiffworkflowBaseDBModel @@ -12,12 +15,16 @@ class TaskDefinitionModel(SpiffworkflowBaseDBModel): __tablename__ = "task_definition" __table_args__ = ( UniqueConstraint( - "bpmn_process_definition_id", "bpmn_identifier", name="task_definition_unique" + "bpmn_process_definition_id", + "bpmn_identifier", + name="task_definition_unique", ), ) id: int = db.Column(db.Integer, primary_key=True) - bpmn_process_definition_id: int = db.Column(ForeignKey(BpmnProcessDefinitionModel.id), nullable=False) # type: ignore + bpmn_process_definition_id: int = db.Column( + ForeignKey(BpmnProcessDefinitionModel.id), nullable=False + ) # type: ignore bpmn_process_definition = relationship(BpmnProcessDefinitionModel) bpmn_identifier: str = db.Column(db.String(255), nullable=False, index=True) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py b/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py index aaa8a9c35..eaf0e433b 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py @@ -1,7 +1,5 @@ """Process_instance_processor.""" import _strptime -from spiffworkflow_backend.models import serialized_bpmn_definition # type: ignore -from spiffworkflow_backend.models.bpmn_process_definition_relationship import BpmnProcessDefinitionRelationshipModel # noqa: F401 import decimal import json import logging @@ -57,8 +55,13 @@ from SpiffWorkflow.util.deep_merge import DeepMerge # type: ignore from sqlalchemy import text from spiffworkflow_backend.exceptions.api_error import ApiError -from spiffworkflow_backend.models import bpmn_process_definition_relationship -from spiffworkflow_backend.models.bpmn_process_definition import BpmnProcessDefinitionModel +from spiffworkflow_backend.models import serialized_bpmn_definition # type: ignore +from spiffworkflow_backend.models.bpmn_process_definition import ( + BpmnProcessDefinitionModel, +) +from spiffworkflow_backend.models.bpmn_process_definition_relationship import ( + BpmnProcessDefinitionRelationshipModel, +) # noqa: F401 from spiffworkflow_backend.models.db import db from spiffworkflow_backend.models.file import File from spiffworkflow_backend.models.file import FileType @@ -525,6 +528,22 @@ class ProcessInstanceProcessor: self.bpmn_process_instance ) + @classmethod + def _get_definition_dict_for_bpmn_process_definition(cls, bpmn_process_definition: BpmnProcessDefinitionModel) -> dict: + task_definitions = TaskDefinitionModel.query.filter_by( + bpmn_process_definition_id=bpmn_process_definition.id + ).all() + bpmn_process_definition_dict: dict = json.loads( + bpmn_process_definition.properties_json + ) + bpmn_process_definition_dict["task_specs"] = {} + for task_definition in task_definitions: + bpmn_process_definition_dict["task_specs"][ + task_definition.bpmn_identifier + ] = json.loads(task_definition.properties_json) + return bpmn_process_definition_dict + + @classmethod def _get_full_bpmn_json(cls, process_instance_model: ProcessInstanceModel) -> dict: if process_instance_model.serialized_bpmn_definition_id is None: @@ -534,24 +553,28 @@ class ProcessInstanceProcessor: # loaded_json: dict = json.loads(serialized_bpmn_definition.static_json) # or "{}") serialized_bpmn_definition = {} - bpmn_process_definition = BpmnProcessDefinitionModel.query.filter_by(id=process_instance_model.bpmn_process_definition_id).first() - task_definitions = TaskDefinitionModel.query.filter_by(bpmn_process_definition_id=process_instance_model.bpmn_process_definition_id).all() + bpmn_process_definition = BpmnProcessDefinitionModel.query.filter_by( + id=process_instance_model.bpmn_process_definition_id + ).first() if bpmn_process_definition is not None: - serialized_bpmn_definition = {"serializer_version": cls.SERIALIZER_VERSION, "spec": {}, "subprocess_specs": {}} - bpmn_process_definition_dict = json.loads(bpmn_process_definition.properties_json) - bpmn_process_definition_dict['task_specs'] = {} - for task_definition in task_definitions: - bpmn_process_definition_dict['task_specs'][task_definition.bpmn_identifier] = json.loads(task_definition.properties_json) - serialized_bpmn_definition['spec'] = bpmn_process_definition_dict + serialized_bpmn_definition = { + "serializer_version": cls.SERIALIZER_VERSION, + "spec": {}, + "subprocess_specs": {}, + } + serialized_bpmn_definition['spec'] = cls._get_definition_dict_for_bpmn_process_definition(bpmn_process_definition) - bpmn_process_subprocess_definitions = BpmnProcessDefinitionRelationshipModel.query.filter_by(bpmn_process_definition_parent_id=bpmn_process_definition.id).all() - for bpmn_process_subprocess_definition in bpmn_process_subprocess_definitions: - subprocess_task_definitions = TaskDefinitionModel.query.filter_by(bpmn_process_definition_id=bpmn_process_subprocess_definition.id).all() - bpmn_process_subprocess_dict = json.loads(bpmn_process_definition.properties_json) - bpmn_process_subprocess_dict['task_specs'] = {} - for subprocess_task_definition in subprocess_task_definitions: - bpmn_process_subprocess_dict['task_specs'][subprocess_task_definition.bpmn_identifier] = json.loads(subprocess_task_definition.properties_json) - serialized_bpmn_definition['subprocess_specs'][bpmn_process_subprocess_definition.bpmn_identifier] = bpmn_process_subprocess_dict + bpmn_process_subprocess_definitions = ( + BpmnProcessDefinitionRelationshipModel.query.filter_by( + bpmn_process_definition_parent_id=bpmn_process_definition.id + ).all() + ) + for ( + bpmn_process_subprocess_definition + ) in bpmn_process_subprocess_definitions: + serialized_bpmn_definition["subprocess_specs"][ + bpmn_process_subprocess_definition.bpmn_identifier + ] = cls._get_definition_dict_for_bpmn_process_definition(bpmn_process_subprocess_definition) loaded_json: dict = serialized_bpmn_definition process_instance_data = process_instance_model.process_instance_data @@ -934,19 +957,25 @@ class ProcessInstanceProcessor: db.session.add(process_instance_data) self.process_instance_model.process_instance_data = process_instance_data - - def _store_bpmn_process_definition(self, process_bpmn_properties: dict, bpmn_process_definition_parent: Optional[BpmnProcessDefinitionModel] = None) -> BpmnProcessDefinitionModel: - process_bpmn_identifier = process_bpmn_properties['name'] + def _store_bpmn_process_definition( + self, + process_bpmn_properties: dict, + bpmn_process_definition_parent: Optional[BpmnProcessDefinitionModel] = None, + ) -> BpmnProcessDefinitionModel: + process_bpmn_identifier = process_bpmn_properties["name"] new_hash_digest = sha256( json.dumps(process_bpmn_properties, sort_keys=True).encode("utf8") ).hexdigest() - bpmn_process_definition: Optional[BpmnProcessDefinitionModel] = BpmnProcessDefinitionModel.query.filter_by( - hash=new_hash_digest - ).first() + bpmn_process_definition: Optional[BpmnProcessDefinitionModel] = ( + BpmnProcessDefinitionModel.query.filter_by(hash=new_hash_digest).first() + ) if bpmn_process_definition is None: task_specs = process_bpmn_properties.pop("task_specs") bpmn_process_definition = BpmnProcessDefinitionModel( - hash=new_hash_digest, bpmn_identifier=process_bpmn_identifier, properties_json=json.dumps(process_bpmn_properties), type="process" + hash=new_hash_digest, + bpmn_identifier=process_bpmn_identifier, + properties_json=json.dumps(process_bpmn_properties), + type="process", ) db.session.add(bpmn_process_definition) @@ -955,15 +984,17 @@ class ProcessInstanceProcessor: bpmn_process_definition=bpmn_process_definition, bpmn_identifier=task_bpmn_identifier, properties_json=json.dumps(task_bpmn_properties), - typename=task_bpmn_properties['typename'], + typename=task_bpmn_properties["typename"], ) db.session.add(task_definition) if bpmn_process_definition_parent: - bpmn_process_definition_relationship = BpmnProcessDefinitionRelationshipModel.query.filter_by( - bpmn_process_definition_parent_id=bpmn_process_definition_parent.id, - bpmn_process_definition_child_id=bpmn_process_definition.id, - ).first() + bpmn_process_definition_relationship = ( + BpmnProcessDefinitionRelationshipModel.query.filter_by( + bpmn_process_definition_parent_id=bpmn_process_definition_parent.id, + bpmn_process_definition_child_id=bpmn_process_definition.id, + ).first() + ) if bpmn_process_definition_relationship is None: bpmn_process_definition_relationship = BpmnProcessDefinitionRelationshipModel( bpmn_process_definition_parent_id=bpmn_process_definition_parent.id, @@ -987,11 +1018,16 @@ class ProcessInstanceProcessor: else: process_instance_data_dict[bpmn_key] = bpmn_dict[bpmn_key] - bpmn_process_definition_parent = self._store_bpmn_process_definition(bpmn_spec_dict['spec']) - for process_bpmn_properties in bpmn_spec_dict['subprocess_specs'].values(): - self._store_bpmn_process_definition(process_bpmn_properties, bpmn_process_definition_parent) - self.process_instance_model.bpmn_process_definition = bpmn_process_definition_parent - + bpmn_process_definition_parent = self._store_bpmn_process_definition( + bpmn_spec_dict["spec"] + ) + for process_bpmn_properties in bpmn_spec_dict["subprocess_specs"].values(): + self._store_bpmn_process_definition( + process_bpmn_properties, bpmn_process_definition_parent + ) + self.process_instance_model.bpmn_process_definition = ( + bpmn_process_definition_parent + ) # # FIXME: always save new hash until we get updated Spiff without loopresettask # # if self.process_instance_model.serialized_bpmn_definition_id is None: From d7f4c8ba957c8c93c26039cbbcff36bf041031fc Mon Sep 17 00:00:00 2001 From: jasquat Date: Fri, 3 Mar 2023 11:24:28 -0500 Subject: [PATCH 05/25] clean up bpmn process definition save code a bit w/ burnettk --- .../migrations/versions/04e43b3c9a50_.py | 36 ++++++++++ .../models/bpmn_process_definition.py | 3 +- .../models/process_instance.py | 6 +- .../models/task_definition.py | 5 +- .../services/process_instance_processor.py | 72 ++++++++----------- .../tests/data/dot_notation/diagram.bpmn | 12 ++-- 6 files changed, 79 insertions(+), 55 deletions(-) create mode 100644 spiffworkflow-backend/migrations/versions/04e43b3c9a50_.py diff --git a/spiffworkflow-backend/migrations/versions/04e43b3c9a50_.py b/spiffworkflow-backend/migrations/versions/04e43b3c9a50_.py new file mode 100644 index 000000000..072f4ba8c --- /dev/null +++ b/spiffworkflow-backend/migrations/versions/04e43b3c9a50_.py @@ -0,0 +1,36 @@ +"""empty message + +Revision ID: 04e43b3c9a50 +Revises: def2cbb0ca6b +Create Date: 2023-03-03 10:31:53.578474 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import mysql + +# revision identifiers, used by Alembic. +revision = '04e43b3c9a50' +down_revision = 'def2cbb0ca6b' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('bpmn_process_definition', 'type', + existing_type=mysql.VARCHAR(length=32), + nullable=True) + op.drop_index('ix_bpmn_process_definition_type', table_name='bpmn_process_definition') + op.add_column('process_instance', sa.Column('spiff_serializer_version', sa.String(length=50), nullable=True)) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('process_instance', 'spiff_serializer_version') + op.create_index('ix_bpmn_process_definition_type', 'bpmn_process_definition', ['type'], unique=False) + op.alter_column('bpmn_process_definition', 'type', + existing_type=mysql.VARCHAR(length=32), + nullable=False) + # ### end Alembic commands ### diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process_definition.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process_definition.py index d60a6c0dd..c78fe4f92 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process_definition.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process_definition.py @@ -22,7 +22,8 @@ class BpmnProcessDefinitionModel(SpiffworkflowBaseDBModel): # process or subprocess # FIXME: will probably ignore for now since we do not strictly need it - type: str = db.Column(db.String(32), nullable=False, index=True) + # make this nullable false and index it once we actually start using it + type: str = db.Column(db.String(32), nullable=True) # TODO: remove these from process_instance bpmn_version_control_type: str = db.Column(db.String(50)) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/process_instance.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/process_instance.py index 44cfe8067..e0fb3f3b2 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/process_instance.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/process_instance.py @@ -78,10 +78,12 @@ class ProcessInstanceModel(SpiffworkflowBaseDBModel): process_instance_data = relationship("ProcessInstanceDataModel", cascade="delete") bpmn_process_definition_id: int = db.Column( - ForeignKey(BpmnProcessDefinitionModel.id), nullable=True - ) # type: ignore + ForeignKey(BpmnProcessDefinitionModel.id), nullable=True # type: ignore + ) bpmn_process_definition = relationship(BpmnProcessDefinitionModel) + spiff_serializer_version = db.Column(db.String(50), nullable=True) + active_human_tasks = relationship( "HumanTaskModel", primaryjoin=( diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/task_definition.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/task_definition.py index 7d7c50873..dab5ae57d 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/task_definition.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/task_definition.py @@ -23,10 +23,11 @@ class TaskDefinitionModel(SpiffworkflowBaseDBModel): id: int = db.Column(db.Integer, primary_key=True) bpmn_process_definition_id: int = db.Column( - ForeignKey(BpmnProcessDefinitionModel.id), nullable=False - ) # type: ignore + ForeignKey(BpmnProcessDefinitionModel.id), nullable=False # type: ignore + ) bpmn_process_definition = relationship(BpmnProcessDefinitionModel) + # bpmn_identifier: str = db.Column(db.String(255, collation='utf8mb4_0900_ai_cs'), nullable=False, index=True) bpmn_identifier: str = db.Column(db.String(255), nullable=False, index=True) properties_json: dict = db.Column(db.JSON, nullable=False) typename: str = db.Column(db.String(255), nullable=False) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py b/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py index eaf0e433b..63e70d55a 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py @@ -552,13 +552,13 @@ class ProcessInstanceProcessor: # print(f"serialized_bpmn_definition.static_json: {serialized_bpmn_definition.static_json}") # loaded_json: dict = json.loads(serialized_bpmn_definition.static_json) # or "{}") - serialized_bpmn_definition = {} + serialized_bpmn_definition: dict = {} bpmn_process_definition = BpmnProcessDefinitionModel.query.filter_by( id=process_instance_model.bpmn_process_definition_id ).first() if bpmn_process_definition is not None: serialized_bpmn_definition = { - "serializer_version": cls.SERIALIZER_VERSION, + "serializer_version": process_instance_model.spiff_serializer_version, "spec": {}, "subprocess_specs": {}, } @@ -570,11 +570,12 @@ class ProcessInstanceProcessor: ).all() ) for ( - bpmn_process_subprocess_definition + bpmn_subprocess_definition ) in bpmn_process_subprocess_definitions: + spec = cls._get_definition_dict_for_bpmn_process_definition(bpmn_subprocess_definition) serialized_bpmn_definition["subprocess_specs"][ - bpmn_process_subprocess_definition.bpmn_identifier - ] = cls._get_definition_dict_for_bpmn_process_definition(bpmn_process_subprocess_definition) + bpmn_subprocess_definition.bpmn_identifier + ] = spec loaded_json: dict = serialized_bpmn_definition process_instance_data = process_instance_model.process_instance_data @@ -975,7 +976,6 @@ class ProcessInstanceProcessor: hash=new_hash_digest, bpmn_identifier=process_bpmn_identifier, properties_json=json.dumps(process_bpmn_properties), - type="process", ) db.session.add(bpmn_process_definition) @@ -988,7 +988,7 @@ class ProcessInstanceProcessor: ) db.session.add(task_definition) - if bpmn_process_definition_parent: + if bpmn_process_definition_parent is not None: bpmn_process_definition_relationship = ( BpmnProcessDefinitionRelationshipModel.query.filter_by( bpmn_process_definition_parent_id=bpmn_process_definition_parent.id, @@ -1003,21 +1003,7 @@ class ProcessInstanceProcessor: db.session.add(bpmn_process_definition_relationship) return bpmn_process_definition - def _add_bpmn_json_records_new(self) -> None: - """Adds serialized_bpmn_definition and process_instance_data records to the db session. - - Expects the save method to commit it. - """ - bpmn_dict = json.loads(self.serialize()) - bpmn_dict_keys = ("spec", "subprocess_specs", "serializer_version") - bpmn_spec_dict = {} - process_instance_data_dict = {} - for bpmn_key in bpmn_dict.keys(): - if bpmn_key in bpmn_dict_keys: - bpmn_spec_dict[bpmn_key] = bpmn_dict[bpmn_key] - else: - process_instance_data_dict[bpmn_key] = bpmn_dict[bpmn_key] - + def _add_bpmn_process_definitions(self, bpmn_spec_dict: dict) -> None: bpmn_process_definition_parent = self._store_bpmn_process_definition( bpmn_spec_dict["spec"] ) @@ -1029,28 +1015,25 @@ class ProcessInstanceProcessor: bpmn_process_definition_parent ) - # # FIXME: always save new hash until we get updated Spiff without loopresettask - # # if self.process_instance_model.serialized_bpmn_definition_id is None: - # new_hash_digest = sha256( - # json.dumps(bpmn_spec_dict, sort_keys=True).encode("utf8") - # ).hexdigest() - # serialized_bpmn_definition = SerializedBpmnDefinitionModel.query.filter_by( - # hash=new_hash_digest - # ).first() - # if serialized_bpmn_definition is None: - # serialized_bpmn_definition = SerializedBpmnDefinitionModel( - # hash=new_hash_digest, static_json=json.dumps(bpmn_spec_dict) - # ) - # db.session.add(serialized_bpmn_definition) - # if ( - # self.process_instance_model.serialized_bpmn_definition_id is None - # or self.process_instance_model.serialized_bpmn_definition.hash - # != new_hash_digest - # ): - # self.process_instance_model.serialized_bpmn_definition = ( - # serialized_bpmn_definition - # ) - # + def _add_bpmn_json_records_new(self) -> None: + """Adds serialized_bpmn_definition and process_instance_data records to the db session. + + Expects the save method to commit it. + """ + bpmn_dict = json.loads(self.serialize()) + bpmn_dict_keys = ("spec", "subprocess_specs", "serializer_version") + process_instance_data_dict = {} + bpmn_spec_dict = {} + for bpmn_key in bpmn_dict.keys(): + if bpmn_key in bpmn_dict_keys: + bpmn_spec_dict[bpmn_key] = bpmn_dict[bpmn_key] + else: + process_instance_data_dict[bpmn_key] = bpmn_dict[bpmn_key] + + # FIXME: always save new hash until we get updated Spiff without loopresettask + # if self.process_instance_model.bpmn_process_definition_id is None: + self._add_bpmn_process_definitions(bpmn_spec_dict) + # process_instance_data = None # if self.process_instance_model.process_instance_data_id is None: # process_instance_data = ProcessInstanceDataModel() @@ -1065,6 +1048,7 @@ class ProcessInstanceProcessor: """Saves the current state of this processor to the database.""" self._add_bpmn_json_records() self._add_bpmn_json_records_new() + self.process_instance_model.spiff_serializer_version = self.SERIALIZER_VERSION complete_states = [TaskState.CANCELLED, TaskState.COMPLETED] user_tasks = list(self.get_all_user_tasks()) diff --git a/spiffworkflow-backend/tests/data/dot_notation/diagram.bpmn b/spiffworkflow-backend/tests/data/dot_notation/diagram.bpmn index 5d6829420..96bdce894 100644 --- a/spiffworkflow-backend/tests/data/dot_notation/diagram.bpmn +++ b/spiffworkflow-backend/tests/data/dot_notation/diagram.bpmn @@ -24,14 +24,14 @@ - + Flow_0dbnzbi - - + + Flow_0nt355i - + @@ -53,13 +53,13 @@ - + - + From 276e6d5f37cc1f9d06ff7d2693738e8f6d1d840f Mon Sep 17 00:00:00 2001 From: jasquat Date: Fri, 3 Mar 2023 11:41:34 -0500 Subject: [PATCH 06/25] recreate migrations to set case sensitive columns for mysql w/ burnettk --- .../migrations/versions/04e43b3c9a50_.py | 36 ------ .../migrations/versions/2fe2830f45e1_.py | 44 -------- .../migrations/versions/317dd5155137_.py | 85 -------------- .../{ac6b60d7fee9_.py => 3ad01a365c17_.py} | 106 +++++++++++++++++- .../migrations/versions/567d22ded3af_.py | 28 ----- .../migrations/versions/6315ff2525b0_.py | 30 ----- .../migrations/versions/7422be14adc4_.py | 51 --------- .../migrations/versions/8930711a75a4_.py | 28 ----- .../migrations/versions/def2cbb0ca6b_.py | 32 ------ .../models/bpmn_process_definition.py | 4 +- .../bpmn_process_definition_relationship.py | 8 +- .../src/spiffworkflow_backend/models/task.py | 4 +- .../models/task_definition.py | 5 +- .../services/process_instance_processor.py | 22 ++-- 14 files changed, 128 insertions(+), 355 deletions(-) delete mode 100644 spiffworkflow-backend/migrations/versions/04e43b3c9a50_.py delete mode 100644 spiffworkflow-backend/migrations/versions/2fe2830f45e1_.py delete mode 100644 spiffworkflow-backend/migrations/versions/317dd5155137_.py rename spiffworkflow-backend/migrations/versions/{ac6b60d7fee9_.py => 3ad01a365c17_.py} (73%) delete mode 100644 spiffworkflow-backend/migrations/versions/567d22ded3af_.py delete mode 100644 spiffworkflow-backend/migrations/versions/6315ff2525b0_.py delete mode 100644 spiffworkflow-backend/migrations/versions/7422be14adc4_.py delete mode 100644 spiffworkflow-backend/migrations/versions/8930711a75a4_.py delete mode 100644 spiffworkflow-backend/migrations/versions/def2cbb0ca6b_.py diff --git a/spiffworkflow-backend/migrations/versions/04e43b3c9a50_.py b/spiffworkflow-backend/migrations/versions/04e43b3c9a50_.py deleted file mode 100644 index 072f4ba8c..000000000 --- a/spiffworkflow-backend/migrations/versions/04e43b3c9a50_.py +++ /dev/null @@ -1,36 +0,0 @@ -"""empty message - -Revision ID: 04e43b3c9a50 -Revises: def2cbb0ca6b -Create Date: 2023-03-03 10:31:53.578474 - -""" -from alembic import op -import sqlalchemy as sa -from sqlalchemy.dialects import mysql - -# revision identifiers, used by Alembic. -revision = '04e43b3c9a50' -down_revision = 'def2cbb0ca6b' -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.alter_column('bpmn_process_definition', 'type', - existing_type=mysql.VARCHAR(length=32), - nullable=True) - op.drop_index('ix_bpmn_process_definition_type', table_name='bpmn_process_definition') - op.add_column('process_instance', sa.Column('spiff_serializer_version', sa.String(length=50), nullable=True)) - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.drop_column('process_instance', 'spiff_serializer_version') - op.create_index('ix_bpmn_process_definition_type', 'bpmn_process_definition', ['type'], unique=False) - op.alter_column('bpmn_process_definition', 'type', - existing_type=mysql.VARCHAR(length=32), - nullable=False) - # ### end Alembic commands ### diff --git a/spiffworkflow-backend/migrations/versions/2fe2830f45e1_.py b/spiffworkflow-backend/migrations/versions/2fe2830f45e1_.py deleted file mode 100644 index 57072697d..000000000 --- a/spiffworkflow-backend/migrations/versions/2fe2830f45e1_.py +++ /dev/null @@ -1,44 +0,0 @@ -"""empty message - -Revision ID: 2fe2830f45e1 -Revises: 317dd5155137 -Create Date: 2023-03-02 17:19:08.535027 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = '2fe2830f45e1' -down_revision = '317dd5155137' -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.create_table('json_data', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('hash', sa.String(length=255), nullable=False), - sa.Column('data', sa.JSON(), nullable=False), - sa.PrimaryKeyConstraint('id') - ) - op.create_index(op.f('ix_json_data_hash'), 'json_data', ['hash'], unique=True) - op.create_table('bpmn_process_definition_relationship', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('bpmn_process_definition_parent_id', sa.Integer(), nullable=False), - sa.Column('bpmn_process_definition_child_id', sa.Integer(), nullable=False), - sa.ForeignKeyConstraint(['bpmn_process_definition_child_id'], ['bpmn_process_definition.id'], ), - sa.ForeignKeyConstraint(['bpmn_process_definition_parent_id'], ['bpmn_process_definition.id'], ), - sa.PrimaryKeyConstraint('id') - ) - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.drop_table('bpmn_process_definition_relationship') - op.drop_index(op.f('ix_json_data_hash'), table_name='json_data') - op.drop_table('json_data') - # ### end Alembic commands ### diff --git a/spiffworkflow-backend/migrations/versions/317dd5155137_.py b/spiffworkflow-backend/migrations/versions/317dd5155137_.py deleted file mode 100644 index e64254c7e..000000000 --- a/spiffworkflow-backend/migrations/versions/317dd5155137_.py +++ /dev/null @@ -1,85 +0,0 @@ -"""empty message - -Revision ID: 317dd5155137 -Revises: 8930711a75a4 -Create Date: 2023-03-02 17:16:15.687837 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = '317dd5155137' -down_revision = '8930711a75a4' -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.create_table('bpmn_process', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('parent_process_id', sa.Integer(), nullable=True), - sa.Column('properties_json', sa.JSON(), nullable=False), - sa.Column('json_data_hash', sa.String(length=255), nullable=False), - sa.Column('process_type', sa.String(length=30), nullable=False), - sa.ForeignKeyConstraint(['parent_process_id'], ['bpmn_process.id'], ), - sa.PrimaryKeyConstraint('id') - ) - op.create_index(op.f('ix_bpmn_process_json_data_hash'), 'bpmn_process', ['json_data_hash'], unique=False) - op.create_table('bpmn_process_definition', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('hash', sa.String(length=255), nullable=False), - sa.Column('bpmn_identifier', sa.String(length=255), nullable=False), - sa.Column('properties_json', sa.JSON(), nullable=False), - sa.Column('type', sa.String(length=32), nullable=False), - sa.Column('bpmn_version_control_type', sa.String(length=50), nullable=True), - sa.Column('bpmn_version_control_identifier', sa.String(length=255), nullable=True), - sa.PrimaryKeyConstraint('id') - ) - op.create_index(op.f('ix_bpmn_process_definition_bpmn_identifier'), 'bpmn_process_definition', ['bpmn_identifier'], unique=False) - op.create_index(op.f('ix_bpmn_process_definition_hash'), 'bpmn_process_definition', ['hash'], unique=True) - op.create_index(op.f('ix_bpmn_process_definition_type'), 'bpmn_process_definition', ['type'], unique=False) - op.create_table('task_definition', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('bpmn_process_definition_id', sa.Integer(), nullable=False), - sa.Column('bpmn_identifier', sa.String(length=255), nullable=False), - sa.Column('properties_json', sa.JSON(), nullable=False), - sa.Column('typename', sa.String(length=255), nullable=False), - sa.ForeignKeyConstraint(['bpmn_process_definition_id'], ['bpmn_process_definition.id'], ), - sa.PrimaryKeyConstraint('id'), - sa.UniqueConstraint('bpmn_process_definition_id', 'bpmn_identifier', name='task_definition_unique') - ) - op.create_index(op.f('ix_task_definition_bpmn_identifier'), 'task_definition', ['bpmn_identifier'], unique=False) - op.create_table('task', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('guid', sa.String(length=36), nullable=False), - sa.Column('bpmn_process_id', sa.Integer(), nullable=False), - sa.Column('task_definition_id', sa.Integer(), nullable=False), - sa.Column('state', sa.String(length=10), nullable=False), - sa.Column('properties_json', sa.JSON(), nullable=False), - sa.Column('json_data_hash', sa.String(length=255), nullable=False), - sa.ForeignKeyConstraint(['bpmn_process_id'], ['bpmn_process.id'], ), - sa.ForeignKeyConstraint(['task_definition_id'], ['task_definition.id'], ), - sa.PrimaryKeyConstraint('id') - ) - op.create_index(op.f('ix_task_guid'), 'task', ['guid'], unique=True) - op.create_index(op.f('ix_task_json_data_hash'), 'task', ['json_data_hash'], unique=False) - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.drop_index(op.f('ix_task_json_data_hash'), table_name='task') - op.drop_index(op.f('ix_task_guid'), table_name='task') - op.drop_table('task') - op.drop_index(op.f('ix_task_definition_bpmn_identifier'), table_name='task_definition') - op.drop_table('task_definition') - op.drop_index(op.f('ix_bpmn_process_definition_type'), table_name='bpmn_process_definition') - op.drop_index(op.f('ix_bpmn_process_definition_hash'), table_name='bpmn_process_definition') - op.drop_index(op.f('ix_bpmn_process_definition_bpmn_identifier'), table_name='bpmn_process_definition') - op.drop_table('bpmn_process_definition') - op.drop_index(op.f('ix_bpmn_process_json_data_hash'), table_name='bpmn_process') - op.drop_table('bpmn_process') - # ### end Alembic commands ### diff --git a/spiffworkflow-backend/migrations/versions/ac6b60d7fee9_.py b/spiffworkflow-backend/migrations/versions/3ad01a365c17_.py similarity index 73% rename from spiffworkflow-backend/migrations/versions/ac6b60d7fee9_.py rename to spiffworkflow-backend/migrations/versions/3ad01a365c17_.py index 520af3804..1378ae022 100644 --- a/spiffworkflow-backend/migrations/versions/ac6b60d7fee9_.py +++ b/spiffworkflow-backend/migrations/versions/3ad01a365c17_.py @@ -1,8 +1,8 @@ """empty message -Revision ID: ac6b60d7fee9 +Revision ID: 3ad01a365c17 Revises: -Create Date: 2023-02-27 22:02:11.465980 +Create Date: 2023-03-03 11:36:56.060834 """ from alembic import op @@ -10,7 +10,7 @@ import sqlalchemy as sa # revision identifiers, used by Alembic. -revision = 'ac6b60d7fee9' +revision = '3ad01a365c17' down_revision = None branch_labels = None depends_on = None @@ -18,6 +18,28 @@ depends_on = None def upgrade(): # ### commands auto generated by Alembic - please adjust! ### + op.create_table('bpmn_process', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('parent_process_id', sa.Integer(), nullable=True), + sa.Column('properties_json', sa.JSON(), nullable=False), + sa.Column('json_data_hash', sa.String(length=255), nullable=False), + sa.Column('process_type', sa.String(length=30), nullable=False), + sa.ForeignKeyConstraint(['parent_process_id'], ['bpmn_process.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_bpmn_process_json_data_hash'), 'bpmn_process', ['json_data_hash'], unique=False) + op.create_table('bpmn_process_definition', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('hash', sa.String(length=255), nullable=False), + sa.Column('bpmn_identifier', sa.String(length=255, collation='utf8mb4_0900_as_cs'), nullable=False), + sa.Column('properties_json', sa.JSON(), nullable=False), + sa.Column('type', sa.String(length=32), nullable=True), + sa.Column('bpmn_version_control_type', sa.String(length=50), nullable=True), + sa.Column('bpmn_version_control_identifier', sa.String(length=255), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_bpmn_process_definition_bpmn_identifier'), 'bpmn_process_definition', ['bpmn_identifier'], unique=False) + op.create_index(op.f('ix_bpmn_process_definition_hash'), 'bpmn_process_definition', ['hash'], unique=True) op.create_table('correlation_property_cache', sa.Column('id', sa.Integer(), nullable=False), sa.Column('name', sa.String(length=50), nullable=False), @@ -32,6 +54,13 @@ def upgrade(): sa.Column('identifier', sa.String(length=255), nullable=True), sa.PrimaryKeyConstraint('id') ) + op.create_table('json_data', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('hash', sa.String(length=255), nullable=False), + sa.Column('data', sa.JSON(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_json_data_hash'), 'json_data', ['hash'], unique=True) op.create_table('message_triggerable_process_model', sa.Column('id', sa.Integer(), nullable=False), sa.Column('message_name', sa.String(length=255), nullable=True), @@ -47,6 +76,18 @@ def upgrade(): sa.PrimaryKeyConstraint('id'), sa.UniqueConstraint('uri') ) + op.create_table('process_instance_data', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('runtime_json', sa.JSON(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('serialized_bpmn_definition', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('hash', sa.String(length=255), nullable=False), + sa.Column('static_json', sa.JSON(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_serialized_bpmn_definition_hash'), 'serialized_bpmn_definition', ['hash'], unique=True) op.create_table('spec_reference_cache', sa.Column('id', sa.Integer(), nullable=False), sa.Column('identifier', sa.String(length=255), nullable=True), @@ -68,6 +109,7 @@ def upgrade(): sa.Column('id', sa.Integer(), nullable=False), sa.Column('process_instance_id', sa.Integer(), nullable=False), sa.Column('bpmn_process_identifier', sa.String(length=255), nullable=False), + sa.Column('bpmn_process_name', sa.String(length=255), nullable=True), sa.Column('bpmn_task_identifier', sa.String(length=255), nullable=False), sa.Column('bpmn_task_name', sa.String(length=255), nullable=True), sa.Column('bpmn_task_type', sa.String(length=255), nullable=True), @@ -94,6 +136,15 @@ def upgrade(): sa.UniqueConstraint('service', 'service_id', name='service_key'), sa.UniqueConstraint('username') ) + op.create_table('bpmn_process_definition_relationship', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('bpmn_process_definition_parent_id', sa.Integer(), nullable=False), + sa.Column('bpmn_process_definition_child_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['bpmn_process_definition_child_id'], ['bpmn_process_definition.id'], ), + sa.ForeignKeyConstraint(['bpmn_process_definition_parent_id'], ['bpmn_process_definition.id'], ), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('bpmn_process_definition_parent_id', 'bpmn_process_definition_child_id', name='bpmn_process_definition_relationship_unique') + ) op.create_table('principal', sa.Column('id', sa.Integer(), nullable=False), sa.Column('user_id', sa.Integer(), nullable=True), @@ -110,6 +161,10 @@ def upgrade(): sa.Column('process_model_identifier', sa.String(length=255), nullable=False), sa.Column('process_model_display_name', sa.String(length=255), nullable=False), sa.Column('process_initiator_id', sa.Integer(), nullable=False), + sa.Column('serialized_bpmn_definition_id', sa.Integer(), nullable=True), + sa.Column('process_instance_data_id', sa.Integer(), nullable=True), + sa.Column('bpmn_process_definition_id', sa.Integer(), nullable=True), + sa.Column('spiff_serializer_version', sa.String(length=50), nullable=True), sa.Column('bpmn_json', sa.JSON(), nullable=True), sa.Column('start_in_seconds', sa.Integer(), nullable=True), sa.Column('end_in_seconds', sa.Integer(), nullable=True), @@ -121,7 +176,10 @@ def upgrade(): sa.Column('spiff_step', sa.Integer(), nullable=True), sa.Column('locked_by', sa.String(length=80), nullable=True), sa.Column('locked_at_in_seconds', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['bpmn_process_definition_id'], ['bpmn_process_definition.id'], ), sa.ForeignKeyConstraint(['process_initiator_id'], ['user.id'], ), + sa.ForeignKeyConstraint(['process_instance_data_id'], ['process_instance_data.id'], ), + sa.ForeignKeyConstraint(['serialized_bpmn_definition_id'], ['serialized_bpmn_definition.id'], ), sa.PrimaryKeyConstraint('id') ) op.create_index(op.f('ix_process_instance_process_model_display_name'), 'process_instance', ['process_model_display_name'], unique=False) @@ -158,6 +216,17 @@ def upgrade(): sa.PrimaryKeyConstraint('id'), sa.UniqueConstraint('key') ) + op.create_table('task_definition', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('bpmn_process_definition_id', sa.Integer(), nullable=False), + sa.Column('bpmn_identifier', sa.String(length=255, collation='utf8mb4_0900_as_cs'), nullable=False), + sa.Column('properties_json', sa.JSON(), nullable=False), + sa.Column('typename', sa.String(length=255), nullable=False), + sa.ForeignKeyConstraint(['bpmn_process_definition_id'], ['bpmn_process_definition.id'], ), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('bpmn_process_definition_id', 'bpmn_identifier', name='task_definition_unique') + ) + op.create_index(op.f('ix_task_definition_bpmn_identifier'), 'task_definition', ['bpmn_identifier'], unique=False) op.create_table('user_group_assignment', sa.Column('id', sa.Integer(), nullable=False), sa.Column('user_id', sa.Integer(), nullable=False), @@ -248,12 +317,27 @@ def upgrade(): sa.Column('task_id', sa.String(length=50), nullable=False), sa.Column('task_state', sa.String(length=50), nullable=False), sa.Column('bpmn_task_identifier', sa.String(length=255), nullable=False), + sa.Column('delta_json', sa.JSON(), nullable=True), sa.Column('start_in_seconds', sa.DECIMAL(precision=17, scale=6), nullable=False), sa.Column('end_in_seconds', sa.DECIMAL(precision=17, scale=6), nullable=True), sa.ForeignKeyConstraint(['process_instance_id'], ['process_instance.id'], ), sa.PrimaryKeyConstraint('id'), sa.UniqueConstraint('process_instance_id', 'spiff_step', name='process_instance_id_spiff_step') ) + op.create_table('task', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('guid', sa.String(length=36), nullable=False), + sa.Column('bpmn_process_id', sa.Integer(), nullable=False), + sa.Column('task_definition_id', sa.Integer(), nullable=False), + sa.Column('state', sa.String(length=10), nullable=False), + sa.Column('properties_json', sa.JSON(), nullable=False), + sa.Column('json_data_hash', sa.String(length=255), nullable=False), + sa.ForeignKeyConstraint(['bpmn_process_id'], ['bpmn_process.id'], ), + sa.ForeignKeyConstraint(['task_definition_id'], ['task_definition.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_task_guid'), 'task', ['guid'], unique=True) + op.create_index(op.f('ix_task_json_data_hash'), 'task', ['json_data_hash'], unique=False) op.create_table('human_task_user', sa.Column('id', sa.Integer(), nullable=False), sa.Column('human_task_id', sa.Integer(), nullable=False), @@ -287,6 +371,9 @@ def downgrade(): op.drop_index(op.f('ix_human_task_user_user_id'), table_name='human_task_user') op.drop_index(op.f('ix_human_task_user_human_task_id'), table_name='human_task_user') op.drop_table('human_task_user') + op.drop_index(op.f('ix_task_json_data_hash'), table_name='task') + op.drop_index(op.f('ix_task_guid'), table_name='task') + op.drop_table('task') op.drop_table('spiff_step_details') op.drop_index(op.f('ix_process_instance_metadata_key'), table_name='process_instance_metadata') op.drop_table('process_instance_metadata') @@ -296,6 +383,8 @@ def downgrade(): op.drop_table('human_task') op.drop_table('user_group_assignment_waiting') op.drop_table('user_group_assignment') + op.drop_index(op.f('ix_task_definition_bpmn_identifier'), table_name='task_definition') + op.drop_table('task_definition') op.drop_table('secret') op.drop_table('refresh_token') op.drop_index(op.f('ix_process_instance_report_identifier'), table_name='process_instance_report') @@ -305,15 +394,26 @@ def downgrade(): op.drop_index(op.f('ix_process_instance_process_model_display_name'), table_name='process_instance') op.drop_table('process_instance') op.drop_table('principal') + op.drop_table('bpmn_process_definition_relationship') op.drop_table('user') op.drop_table('spiff_logging') op.drop_index(op.f('ix_spec_reference_cache_type'), table_name='spec_reference_cache') op.drop_index(op.f('ix_spec_reference_cache_identifier'), table_name='spec_reference_cache') op.drop_index(op.f('ix_spec_reference_cache_display_name'), table_name='spec_reference_cache') op.drop_table('spec_reference_cache') + op.drop_index(op.f('ix_serialized_bpmn_definition_hash'), table_name='serialized_bpmn_definition') + op.drop_table('serialized_bpmn_definition') + op.drop_table('process_instance_data') op.drop_table('permission_target') op.drop_index(op.f('ix_message_triggerable_process_model_process_model_identifier'), table_name='message_triggerable_process_model') op.drop_table('message_triggerable_process_model') + op.drop_index(op.f('ix_json_data_hash'), table_name='json_data') + op.drop_table('json_data') op.drop_table('group') op.drop_table('correlation_property_cache') + op.drop_index(op.f('ix_bpmn_process_definition_hash'), table_name='bpmn_process_definition') + op.drop_index(op.f('ix_bpmn_process_definition_bpmn_identifier'), table_name='bpmn_process_definition') + op.drop_table('bpmn_process_definition') + op.drop_index(op.f('ix_bpmn_process_json_data_hash'), table_name='bpmn_process') + op.drop_table('bpmn_process') # ### end Alembic commands ### diff --git a/spiffworkflow-backend/migrations/versions/567d22ded3af_.py b/spiffworkflow-backend/migrations/versions/567d22ded3af_.py deleted file mode 100644 index 1e41244d2..000000000 --- a/spiffworkflow-backend/migrations/versions/567d22ded3af_.py +++ /dev/null @@ -1,28 +0,0 @@ -"""empty message - -Revision ID: 567d22ded3af -Revises: 2fe2830f45e1 -Create Date: 2023-03-03 08:38:25.855923 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = '567d22ded3af' -down_revision = '2fe2830f45e1' -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.create_unique_constraint('bpmn_process_definition_relationship_unique', 'bpmn_process_definition_relationship', ['bpmn_process_definition_parent_id', 'bpmn_process_definition_child_id']) - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.drop_constraint('bpmn_process_definition_relationship_unique', 'bpmn_process_definition_relationship', type_='unique') - # ### end Alembic commands ### diff --git a/spiffworkflow-backend/migrations/versions/6315ff2525b0_.py b/spiffworkflow-backend/migrations/versions/6315ff2525b0_.py deleted file mode 100644 index d21097a2c..000000000 --- a/spiffworkflow-backend/migrations/versions/6315ff2525b0_.py +++ /dev/null @@ -1,30 +0,0 @@ -"""empty message - -Revision ID: 6315ff2525b0 -Revises: 567d22ded3af -Create Date: 2023-03-03 09:19:28.098057 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = '6315ff2525b0' -down_revision = '567d22ded3af' -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.add_column('process_instance', sa.Column('bpmn_process_definition_id', sa.Integer(), nullable=False)) - op.create_foreign_key(None, 'process_instance', 'bpmn_process_definition', ['bpmn_process_definition_id'], ['id']) - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.drop_constraint(None, 'process_instance', type_='foreignkey') - op.drop_column('process_instance', 'bpmn_process_definition_id') - # ### end Alembic commands ### diff --git a/spiffworkflow-backend/migrations/versions/7422be14adc4_.py b/spiffworkflow-backend/migrations/versions/7422be14adc4_.py deleted file mode 100644 index 6897e0f21..000000000 --- a/spiffworkflow-backend/migrations/versions/7422be14adc4_.py +++ /dev/null @@ -1,51 +0,0 @@ -"""empty message - -Revision ID: 7422be14adc4 -Revises: ac6b60d7fee9 -Create Date: 2023-03-01 15:39:25.731722 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = '7422be14adc4' -down_revision = 'ac6b60d7fee9' -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.create_table('process_instance_data', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('runtime_json', sa.JSON(), nullable=False), - sa.PrimaryKeyConstraint('id') - ) - op.create_table('serialized_bpmn_definition', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('hash', sa.String(length=255), nullable=False), - sa.Column('static_json', sa.JSON(), nullable=False), - sa.PrimaryKeyConstraint('id') - ) - op.create_index(op.f('ix_serialized_bpmn_definition_hash'), 'serialized_bpmn_definition', ['hash'], unique=True) - op.add_column('process_instance', sa.Column('serialized_bpmn_definition_id', sa.Integer(), nullable=True)) - op.add_column('process_instance', sa.Column('process_instance_data_id', sa.Integer(), nullable=True)) - op.create_foreign_key(None, 'process_instance', 'process_instance_data', ['process_instance_data_id'], ['id']) - op.create_foreign_key(None, 'process_instance', 'serialized_bpmn_definition', ['serialized_bpmn_definition_id'], ['id']) - op.add_column('spiff_step_details', sa.Column('delta_json', sa.JSON(), nullable=True)) - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.drop_column('spiff_step_details', 'delta_json') - op.drop_constraint('process_instance_ibfk_3', 'process_instance', type_='foreignkey') - op.drop_constraint('process_instance_ibfk_2', 'process_instance', type_='foreignkey') - op.drop_column('process_instance', 'process_instance_data_id') - op.drop_column('process_instance', 'serialized_bpmn_definition_id') - op.drop_index(op.f('ix_serialized_bpmn_definition_hash'), table_name='serialized_bpmn_definition') - op.drop_table('serialized_bpmn_definition') - op.drop_table('process_instance_data') - # ### end Alembic commands ### diff --git a/spiffworkflow-backend/migrations/versions/8930711a75a4_.py b/spiffworkflow-backend/migrations/versions/8930711a75a4_.py deleted file mode 100644 index 9947a6644..000000000 --- a/spiffworkflow-backend/migrations/versions/8930711a75a4_.py +++ /dev/null @@ -1,28 +0,0 @@ -"""empty message - -Revision ID: 8930711a75a4 -Revises: 7422be14adc4 -Create Date: 2023-03-01 16:16:50.446929 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = '8930711a75a4' -down_revision = '7422be14adc4' -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.add_column('spiff_logging', sa.Column('bpmn_process_name', sa.String(length=255), nullable=True)) - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.drop_column('spiff_logging', 'bpmn_process_name') - # ### end Alembic commands ### diff --git a/spiffworkflow-backend/migrations/versions/def2cbb0ca6b_.py b/spiffworkflow-backend/migrations/versions/def2cbb0ca6b_.py deleted file mode 100644 index 33bb26948..000000000 --- a/spiffworkflow-backend/migrations/versions/def2cbb0ca6b_.py +++ /dev/null @@ -1,32 +0,0 @@ -"""empty message - -Revision ID: def2cbb0ca6b -Revises: 6315ff2525b0 -Create Date: 2023-03-03 09:23:19.480250 - -""" -from alembic import op -import sqlalchemy as sa -from sqlalchemy.dialects import mysql - -# revision identifiers, used by Alembic. -revision = 'def2cbb0ca6b' -down_revision = '6315ff2525b0' -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.alter_column('process_instance', 'bpmn_process_definition_id', - existing_type=mysql.INTEGER(), - nullable=True) - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.alter_column('process_instance', 'bpmn_process_definition_id', - existing_type=mysql.INTEGER(), - nullable=False) - # ### end Alembic commands ### diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process_definition.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process_definition.py index c78fe4f92..916cf7d71 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process_definition.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process_definition.py @@ -16,7 +16,9 @@ class BpmnProcessDefinitionModel(SpiffworkflowBaseDBModel): # this is a sha256 hash of spec and serializer_version hash: str = db.Column(db.String(255), nullable=False, index=True, unique=True) - bpmn_identifier: str = db.Column(db.String(255), nullable=False, index=True) + bpmn_identifier: str = db.Column( + db.String(255, collation="utf8mb4_0900_as_cs"), nullable=False, index=True + ) properties_json: str = db.Column(db.JSON, nullable=False) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process_definition_relationship.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process_definition_relationship.py index 8f6fdca2e..5ab4a7a21 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process_definition_relationship.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process_definition_relationship.py @@ -22,8 +22,8 @@ class BpmnProcessDefinitionRelationshipModel(SpiffworkflowBaseDBModel): id: int = db.Column(db.Integer, primary_key=True) bpmn_process_definition_parent_id: int = db.Column( - ForeignKey(BpmnProcessDefinitionModel.id), nullable=False - ) # type: ignore + ForeignKey(BpmnProcessDefinitionModel.id), nullable=False # type: ignore + ) bpmn_process_definition_child_id: int = db.Column( - ForeignKey(BpmnProcessDefinitionModel.id), nullable=False - ) # type: ignore + ForeignKey(BpmnProcessDefinitionModel.id), nullable=False # type: ignore + ) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/task.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/task.py index 3fda6677f..8038a14be 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/task.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/task.py @@ -8,13 +8,13 @@ from typing import Union import marshmallow from marshmallow import Schema from marshmallow_enum import EnumField # type: ignore -from SpiffWorkflow.task import TaskStateNames +from SpiffWorkflow.task import TaskStateNames # type: ignore from sqlalchemy import ForeignKey from spiffworkflow_backend.models.bpmn_process import BpmnProcessModel from spiffworkflow_backend.models.db import db from spiffworkflow_backend.models.db import SpiffworkflowBaseDBModel -from spiffworkflow_backend.models.task_definition import TaskDefinitionModel # type: ignore +from spiffworkflow_backend.models.task_definition import TaskDefinitionModel class MultiInstanceType(enum.Enum): diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/task_definition.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/task_definition.py index dab5ae57d..8c70bbd04 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/task_definition.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/task_definition.py @@ -27,7 +27,8 @@ class TaskDefinitionModel(SpiffworkflowBaseDBModel): ) bpmn_process_definition = relationship(BpmnProcessDefinitionModel) - # bpmn_identifier: str = db.Column(db.String(255, collation='utf8mb4_0900_ai_cs'), nullable=False, index=True) - bpmn_identifier: str = db.Column(db.String(255), nullable=False, index=True) + bpmn_identifier: str = db.Column( + db.String(255, collation="utf8mb4_0900_as_cs"), nullable=False, index=True + ) properties_json: dict = db.Column(db.JSON, nullable=False) typename: str = db.Column(db.String(255), nullable=False) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py b/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py index 63e70d55a..7ec181659 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py @@ -1,5 +1,5 @@ """Process_instance_processor.""" -import _strptime +import _strptime # type: ignore import decimal import json import logging @@ -55,7 +55,6 @@ from SpiffWorkflow.util.deep_merge import DeepMerge # type: ignore from sqlalchemy import text from spiffworkflow_backend.exceptions.api_error import ApiError -from spiffworkflow_backend.models import serialized_bpmn_definition # type: ignore from spiffworkflow_backend.models.bpmn_process_definition import ( BpmnProcessDefinitionModel, ) @@ -529,7 +528,9 @@ class ProcessInstanceProcessor: ) @classmethod - def _get_definition_dict_for_bpmn_process_definition(cls, bpmn_process_definition: BpmnProcessDefinitionModel) -> dict: + def _get_definition_dict_for_bpmn_process_definition( + cls, bpmn_process_definition: BpmnProcessDefinitionModel + ) -> dict: task_definitions = TaskDefinitionModel.query.filter_by( bpmn_process_definition_id=bpmn_process_definition.id ).all() @@ -543,7 +544,6 @@ class ProcessInstanceProcessor: ] = json.loads(task_definition.properties_json) return bpmn_process_definition_dict - @classmethod def _get_full_bpmn_json(cls, process_instance_model: ProcessInstanceModel) -> dict: if process_instance_model.serialized_bpmn_definition_id is None: @@ -562,17 +562,21 @@ class ProcessInstanceProcessor: "spec": {}, "subprocess_specs": {}, } - serialized_bpmn_definition['spec'] = cls._get_definition_dict_for_bpmn_process_definition(bpmn_process_definition) + serialized_bpmn_definition["spec"] = ( + cls._get_definition_dict_for_bpmn_process_definition( + bpmn_process_definition + ) + ) bpmn_process_subprocess_definitions = ( BpmnProcessDefinitionRelationshipModel.query.filter_by( bpmn_process_definition_parent_id=bpmn_process_definition.id ).all() ) - for ( - bpmn_subprocess_definition - ) in bpmn_process_subprocess_definitions: - spec = cls._get_definition_dict_for_bpmn_process_definition(bpmn_subprocess_definition) + for bpmn_subprocess_definition in bpmn_process_subprocess_definitions: + spec = cls._get_definition_dict_for_bpmn_process_definition( + bpmn_subprocess_definition + ) serialized_bpmn_definition["subprocess_specs"][ bpmn_subprocess_definition.bpmn_identifier ] = spec From 8c60080663114557caeca262a1c0476da1e07e13 Mon Sep 17 00:00:00 2001 From: jasquat Date: Fri, 3 Mar 2023 12:05:12 -0500 Subject: [PATCH 07/25] set the collation on the db directly and not in the models w/ burnettk --- spiffworkflow-backend/migrations/env.py | 2 ++ .../versions/{3ad01a365c17_.py => 25f64177f8ec_.py} | 11 +++++------ .../src/spiffworkflow_backend/models/bpmn_process.py | 2 +- .../models/bpmn_process_definition.py | 3 ++- .../spiffworkflow_backend/models/task_definition.py | 2 +- 5 files changed, 11 insertions(+), 9 deletions(-) rename spiffworkflow-backend/migrations/versions/{3ad01a365c17_.py => 25f64177f8ec_.py} (98%) diff --git a/spiffworkflow-backend/migrations/env.py b/spiffworkflow-backend/migrations/env.py index 630e381ad..68feded2a 100644 --- a/spiffworkflow-backend/migrations/env.py +++ b/spiffworkflow-backend/migrations/env.py @@ -1,3 +1,5 @@ +from __future__ import with_statement + import logging from logging.config import fileConfig diff --git a/spiffworkflow-backend/migrations/versions/3ad01a365c17_.py b/spiffworkflow-backend/migrations/versions/25f64177f8ec_.py similarity index 98% rename from spiffworkflow-backend/migrations/versions/3ad01a365c17_.py rename to spiffworkflow-backend/migrations/versions/25f64177f8ec_.py index 1378ae022..eb7b9295a 100644 --- a/spiffworkflow-backend/migrations/versions/3ad01a365c17_.py +++ b/spiffworkflow-backend/migrations/versions/25f64177f8ec_.py @@ -1,8 +1,8 @@ """empty message -Revision ID: 3ad01a365c17 +Revision ID: 25f64177f8ec Revises: -Create Date: 2023-03-03 11:36:56.060834 +Create Date: 2023-03-03 12:03:28.155345 """ from alembic import op @@ -10,7 +10,7 @@ import sqlalchemy as sa # revision identifiers, used by Alembic. -revision = '3ad01a365c17' +revision = '25f64177f8ec' down_revision = None branch_labels = None depends_on = None @@ -23,7 +23,6 @@ def upgrade(): sa.Column('parent_process_id', sa.Integer(), nullable=True), sa.Column('properties_json', sa.JSON(), nullable=False), sa.Column('json_data_hash', sa.String(length=255), nullable=False), - sa.Column('process_type', sa.String(length=30), nullable=False), sa.ForeignKeyConstraint(['parent_process_id'], ['bpmn_process.id'], ), sa.PrimaryKeyConstraint('id') ) @@ -31,7 +30,7 @@ def upgrade(): op.create_table('bpmn_process_definition', sa.Column('id', sa.Integer(), nullable=False), sa.Column('hash', sa.String(length=255), nullable=False), - sa.Column('bpmn_identifier', sa.String(length=255, collation='utf8mb4_0900_as_cs'), nullable=False), + sa.Column('bpmn_identifier', sa.String(length=255), nullable=False), sa.Column('properties_json', sa.JSON(), nullable=False), sa.Column('type', sa.String(length=32), nullable=True), sa.Column('bpmn_version_control_type', sa.String(length=50), nullable=True), @@ -219,7 +218,7 @@ def upgrade(): op.create_table('task_definition', sa.Column('id', sa.Integer(), nullable=False), sa.Column('bpmn_process_definition_id', sa.Integer(), nullable=False), - sa.Column('bpmn_identifier', sa.String(length=255, collation='utf8mb4_0900_as_cs'), nullable=False), + sa.Column('bpmn_identifier', sa.String(length=255), nullable=False), sa.Column('properties_json', sa.JSON(), nullable=False), sa.Column('typename', sa.String(length=255), nullable=False), sa.ForeignKeyConstraint(['bpmn_process_definition_id'], ['bpmn_process_definition.id'], ), diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process.py index 948cdffd1..0d7e556d9 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process.py @@ -22,4 +22,4 @@ class BpmnProcessModel(SpiffworkflowBaseDBModel): json_data_hash: str = db.Column(db.String(255), nullable=False, index=True) # subprocess or top_level_process - process_type: str = db.Column(db.String(30), nullable=False) + # process_type: str = db.Column(db.String(30), nullable=False) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process_definition.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process_definition.py index 916cf7d71..eec715c37 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process_definition.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process_definition.py @@ -16,8 +16,9 @@ class BpmnProcessDefinitionModel(SpiffworkflowBaseDBModel): # this is a sha256 hash of spec and serializer_version hash: str = db.Column(db.String(255), nullable=False, index=True, unique=True) + bpmn_identifier: str = db.Column( - db.String(255, collation="utf8mb4_0900_as_cs"), nullable=False, index=True + db.String(255), nullable=False, index=True ) properties_json: str = db.Column(db.JSON, nullable=False) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/task_definition.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/task_definition.py index 8c70bbd04..f4f5c997f 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/task_definition.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/task_definition.py @@ -28,7 +28,7 @@ class TaskDefinitionModel(SpiffworkflowBaseDBModel): bpmn_process_definition = relationship(BpmnProcessDefinitionModel) bpmn_identifier: str = db.Column( - db.String(255, collation="utf8mb4_0900_as_cs"), nullable=False, index=True + db.String(255), nullable=False, index=True ) properties_json: dict = db.Column(db.JSON, nullable=False) typename: str = db.Column(db.String(255), nullable=False) From f7c46cd9452af0259d5fcb296caed418f4d5c9e7 Mon Sep 17 00:00:00 2001 From: jasquat Date: Fri, 3 Mar 2023 12:08:36 -0500 Subject: [PATCH 08/25] recreated branch migration w/ burnettk --- spiffworkflow-backend/migrations/env.py | 2 - .../migrations/versions/7422be14adc4_.py | 51 +++++++++ .../migrations/versions/8930711a75a4_.py | 28 +++++ .../{25f64177f8ec_.py => ac6b60d7fee9_.py} | 105 +---------------- .../migrations/versions/d6fd61fd64d7_.py | 107 ++++++++++++++++++ .../models/bpmn_process_definition.py | 4 +- .../models/task_definition.py | 4 +- 7 files changed, 191 insertions(+), 110 deletions(-) create mode 100644 spiffworkflow-backend/migrations/versions/7422be14adc4_.py create mode 100644 spiffworkflow-backend/migrations/versions/8930711a75a4_.py rename spiffworkflow-backend/migrations/versions/{25f64177f8ec_.py => ac6b60d7fee9_.py} (74%) create mode 100644 spiffworkflow-backend/migrations/versions/d6fd61fd64d7_.py diff --git a/spiffworkflow-backend/migrations/env.py b/spiffworkflow-backend/migrations/env.py index 68feded2a..630e381ad 100644 --- a/spiffworkflow-backend/migrations/env.py +++ b/spiffworkflow-backend/migrations/env.py @@ -1,5 +1,3 @@ -from __future__ import with_statement - import logging from logging.config import fileConfig diff --git a/spiffworkflow-backend/migrations/versions/7422be14adc4_.py b/spiffworkflow-backend/migrations/versions/7422be14adc4_.py new file mode 100644 index 000000000..6897e0f21 --- /dev/null +++ b/spiffworkflow-backend/migrations/versions/7422be14adc4_.py @@ -0,0 +1,51 @@ +"""empty message + +Revision ID: 7422be14adc4 +Revises: ac6b60d7fee9 +Create Date: 2023-03-01 15:39:25.731722 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '7422be14adc4' +down_revision = 'ac6b60d7fee9' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('process_instance_data', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('runtime_json', sa.JSON(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('serialized_bpmn_definition', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('hash', sa.String(length=255), nullable=False), + sa.Column('static_json', sa.JSON(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_serialized_bpmn_definition_hash'), 'serialized_bpmn_definition', ['hash'], unique=True) + op.add_column('process_instance', sa.Column('serialized_bpmn_definition_id', sa.Integer(), nullable=True)) + op.add_column('process_instance', sa.Column('process_instance_data_id', sa.Integer(), nullable=True)) + op.create_foreign_key(None, 'process_instance', 'process_instance_data', ['process_instance_data_id'], ['id']) + op.create_foreign_key(None, 'process_instance', 'serialized_bpmn_definition', ['serialized_bpmn_definition_id'], ['id']) + op.add_column('spiff_step_details', sa.Column('delta_json', sa.JSON(), nullable=True)) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('spiff_step_details', 'delta_json') + op.drop_constraint('process_instance_ibfk_3', 'process_instance', type_='foreignkey') + op.drop_constraint('process_instance_ibfk_2', 'process_instance', type_='foreignkey') + op.drop_column('process_instance', 'process_instance_data_id') + op.drop_column('process_instance', 'serialized_bpmn_definition_id') + op.drop_index(op.f('ix_serialized_bpmn_definition_hash'), table_name='serialized_bpmn_definition') + op.drop_table('serialized_bpmn_definition') + op.drop_table('process_instance_data') + # ### end Alembic commands ### diff --git a/spiffworkflow-backend/migrations/versions/8930711a75a4_.py b/spiffworkflow-backend/migrations/versions/8930711a75a4_.py new file mode 100644 index 000000000..9947a6644 --- /dev/null +++ b/spiffworkflow-backend/migrations/versions/8930711a75a4_.py @@ -0,0 +1,28 @@ +"""empty message + +Revision ID: 8930711a75a4 +Revises: 7422be14adc4 +Create Date: 2023-03-01 16:16:50.446929 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '8930711a75a4' +down_revision = '7422be14adc4' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('spiff_logging', sa.Column('bpmn_process_name', sa.String(length=255), nullable=True)) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('spiff_logging', 'bpmn_process_name') + # ### end Alembic commands ### diff --git a/spiffworkflow-backend/migrations/versions/25f64177f8ec_.py b/spiffworkflow-backend/migrations/versions/ac6b60d7fee9_.py similarity index 74% rename from spiffworkflow-backend/migrations/versions/25f64177f8ec_.py rename to spiffworkflow-backend/migrations/versions/ac6b60d7fee9_.py index eb7b9295a..520af3804 100644 --- a/spiffworkflow-backend/migrations/versions/25f64177f8ec_.py +++ b/spiffworkflow-backend/migrations/versions/ac6b60d7fee9_.py @@ -1,8 +1,8 @@ """empty message -Revision ID: 25f64177f8ec +Revision ID: ac6b60d7fee9 Revises: -Create Date: 2023-03-03 12:03:28.155345 +Create Date: 2023-02-27 22:02:11.465980 """ from alembic import op @@ -10,7 +10,7 @@ import sqlalchemy as sa # revision identifiers, used by Alembic. -revision = '25f64177f8ec' +revision = 'ac6b60d7fee9' down_revision = None branch_labels = None depends_on = None @@ -18,27 +18,6 @@ depends_on = None def upgrade(): # ### commands auto generated by Alembic - please adjust! ### - op.create_table('bpmn_process', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('parent_process_id', sa.Integer(), nullable=True), - sa.Column('properties_json', sa.JSON(), nullable=False), - sa.Column('json_data_hash', sa.String(length=255), nullable=False), - sa.ForeignKeyConstraint(['parent_process_id'], ['bpmn_process.id'], ), - sa.PrimaryKeyConstraint('id') - ) - op.create_index(op.f('ix_bpmn_process_json_data_hash'), 'bpmn_process', ['json_data_hash'], unique=False) - op.create_table('bpmn_process_definition', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('hash', sa.String(length=255), nullable=False), - sa.Column('bpmn_identifier', sa.String(length=255), nullable=False), - sa.Column('properties_json', sa.JSON(), nullable=False), - sa.Column('type', sa.String(length=32), nullable=True), - sa.Column('bpmn_version_control_type', sa.String(length=50), nullable=True), - sa.Column('bpmn_version_control_identifier', sa.String(length=255), nullable=True), - sa.PrimaryKeyConstraint('id') - ) - op.create_index(op.f('ix_bpmn_process_definition_bpmn_identifier'), 'bpmn_process_definition', ['bpmn_identifier'], unique=False) - op.create_index(op.f('ix_bpmn_process_definition_hash'), 'bpmn_process_definition', ['hash'], unique=True) op.create_table('correlation_property_cache', sa.Column('id', sa.Integer(), nullable=False), sa.Column('name', sa.String(length=50), nullable=False), @@ -53,13 +32,6 @@ def upgrade(): sa.Column('identifier', sa.String(length=255), nullable=True), sa.PrimaryKeyConstraint('id') ) - op.create_table('json_data', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('hash', sa.String(length=255), nullable=False), - sa.Column('data', sa.JSON(), nullable=False), - sa.PrimaryKeyConstraint('id') - ) - op.create_index(op.f('ix_json_data_hash'), 'json_data', ['hash'], unique=True) op.create_table('message_triggerable_process_model', sa.Column('id', sa.Integer(), nullable=False), sa.Column('message_name', sa.String(length=255), nullable=True), @@ -75,18 +47,6 @@ def upgrade(): sa.PrimaryKeyConstraint('id'), sa.UniqueConstraint('uri') ) - op.create_table('process_instance_data', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('runtime_json', sa.JSON(), nullable=False), - sa.PrimaryKeyConstraint('id') - ) - op.create_table('serialized_bpmn_definition', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('hash', sa.String(length=255), nullable=False), - sa.Column('static_json', sa.JSON(), nullable=False), - sa.PrimaryKeyConstraint('id') - ) - op.create_index(op.f('ix_serialized_bpmn_definition_hash'), 'serialized_bpmn_definition', ['hash'], unique=True) op.create_table('spec_reference_cache', sa.Column('id', sa.Integer(), nullable=False), sa.Column('identifier', sa.String(length=255), nullable=True), @@ -108,7 +68,6 @@ def upgrade(): sa.Column('id', sa.Integer(), nullable=False), sa.Column('process_instance_id', sa.Integer(), nullable=False), sa.Column('bpmn_process_identifier', sa.String(length=255), nullable=False), - sa.Column('bpmn_process_name', sa.String(length=255), nullable=True), sa.Column('bpmn_task_identifier', sa.String(length=255), nullable=False), sa.Column('bpmn_task_name', sa.String(length=255), nullable=True), sa.Column('bpmn_task_type', sa.String(length=255), nullable=True), @@ -135,15 +94,6 @@ def upgrade(): sa.UniqueConstraint('service', 'service_id', name='service_key'), sa.UniqueConstraint('username') ) - op.create_table('bpmn_process_definition_relationship', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('bpmn_process_definition_parent_id', sa.Integer(), nullable=False), - sa.Column('bpmn_process_definition_child_id', sa.Integer(), nullable=False), - sa.ForeignKeyConstraint(['bpmn_process_definition_child_id'], ['bpmn_process_definition.id'], ), - sa.ForeignKeyConstraint(['bpmn_process_definition_parent_id'], ['bpmn_process_definition.id'], ), - sa.PrimaryKeyConstraint('id'), - sa.UniqueConstraint('bpmn_process_definition_parent_id', 'bpmn_process_definition_child_id', name='bpmn_process_definition_relationship_unique') - ) op.create_table('principal', sa.Column('id', sa.Integer(), nullable=False), sa.Column('user_id', sa.Integer(), nullable=True), @@ -160,10 +110,6 @@ def upgrade(): sa.Column('process_model_identifier', sa.String(length=255), nullable=False), sa.Column('process_model_display_name', sa.String(length=255), nullable=False), sa.Column('process_initiator_id', sa.Integer(), nullable=False), - sa.Column('serialized_bpmn_definition_id', sa.Integer(), nullable=True), - sa.Column('process_instance_data_id', sa.Integer(), nullable=True), - sa.Column('bpmn_process_definition_id', sa.Integer(), nullable=True), - sa.Column('spiff_serializer_version', sa.String(length=50), nullable=True), sa.Column('bpmn_json', sa.JSON(), nullable=True), sa.Column('start_in_seconds', sa.Integer(), nullable=True), sa.Column('end_in_seconds', sa.Integer(), nullable=True), @@ -175,10 +121,7 @@ def upgrade(): sa.Column('spiff_step', sa.Integer(), nullable=True), sa.Column('locked_by', sa.String(length=80), nullable=True), sa.Column('locked_at_in_seconds', sa.Integer(), nullable=True), - sa.ForeignKeyConstraint(['bpmn_process_definition_id'], ['bpmn_process_definition.id'], ), sa.ForeignKeyConstraint(['process_initiator_id'], ['user.id'], ), - sa.ForeignKeyConstraint(['process_instance_data_id'], ['process_instance_data.id'], ), - sa.ForeignKeyConstraint(['serialized_bpmn_definition_id'], ['serialized_bpmn_definition.id'], ), sa.PrimaryKeyConstraint('id') ) op.create_index(op.f('ix_process_instance_process_model_display_name'), 'process_instance', ['process_model_display_name'], unique=False) @@ -215,17 +158,6 @@ def upgrade(): sa.PrimaryKeyConstraint('id'), sa.UniqueConstraint('key') ) - op.create_table('task_definition', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('bpmn_process_definition_id', sa.Integer(), nullable=False), - sa.Column('bpmn_identifier', sa.String(length=255), nullable=False), - sa.Column('properties_json', sa.JSON(), nullable=False), - sa.Column('typename', sa.String(length=255), nullable=False), - sa.ForeignKeyConstraint(['bpmn_process_definition_id'], ['bpmn_process_definition.id'], ), - sa.PrimaryKeyConstraint('id'), - sa.UniqueConstraint('bpmn_process_definition_id', 'bpmn_identifier', name='task_definition_unique') - ) - op.create_index(op.f('ix_task_definition_bpmn_identifier'), 'task_definition', ['bpmn_identifier'], unique=False) op.create_table('user_group_assignment', sa.Column('id', sa.Integer(), nullable=False), sa.Column('user_id', sa.Integer(), nullable=False), @@ -316,27 +248,12 @@ def upgrade(): sa.Column('task_id', sa.String(length=50), nullable=False), sa.Column('task_state', sa.String(length=50), nullable=False), sa.Column('bpmn_task_identifier', sa.String(length=255), nullable=False), - sa.Column('delta_json', sa.JSON(), nullable=True), sa.Column('start_in_seconds', sa.DECIMAL(precision=17, scale=6), nullable=False), sa.Column('end_in_seconds', sa.DECIMAL(precision=17, scale=6), nullable=True), sa.ForeignKeyConstraint(['process_instance_id'], ['process_instance.id'], ), sa.PrimaryKeyConstraint('id'), sa.UniqueConstraint('process_instance_id', 'spiff_step', name='process_instance_id_spiff_step') ) - op.create_table('task', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('guid', sa.String(length=36), nullable=False), - sa.Column('bpmn_process_id', sa.Integer(), nullable=False), - sa.Column('task_definition_id', sa.Integer(), nullable=False), - sa.Column('state', sa.String(length=10), nullable=False), - sa.Column('properties_json', sa.JSON(), nullable=False), - sa.Column('json_data_hash', sa.String(length=255), nullable=False), - sa.ForeignKeyConstraint(['bpmn_process_id'], ['bpmn_process.id'], ), - sa.ForeignKeyConstraint(['task_definition_id'], ['task_definition.id'], ), - sa.PrimaryKeyConstraint('id') - ) - op.create_index(op.f('ix_task_guid'), 'task', ['guid'], unique=True) - op.create_index(op.f('ix_task_json_data_hash'), 'task', ['json_data_hash'], unique=False) op.create_table('human_task_user', sa.Column('id', sa.Integer(), nullable=False), sa.Column('human_task_id', sa.Integer(), nullable=False), @@ -370,9 +287,6 @@ def downgrade(): op.drop_index(op.f('ix_human_task_user_user_id'), table_name='human_task_user') op.drop_index(op.f('ix_human_task_user_human_task_id'), table_name='human_task_user') op.drop_table('human_task_user') - op.drop_index(op.f('ix_task_json_data_hash'), table_name='task') - op.drop_index(op.f('ix_task_guid'), table_name='task') - op.drop_table('task') op.drop_table('spiff_step_details') op.drop_index(op.f('ix_process_instance_metadata_key'), table_name='process_instance_metadata') op.drop_table('process_instance_metadata') @@ -382,8 +296,6 @@ def downgrade(): op.drop_table('human_task') op.drop_table('user_group_assignment_waiting') op.drop_table('user_group_assignment') - op.drop_index(op.f('ix_task_definition_bpmn_identifier'), table_name='task_definition') - op.drop_table('task_definition') op.drop_table('secret') op.drop_table('refresh_token') op.drop_index(op.f('ix_process_instance_report_identifier'), table_name='process_instance_report') @@ -393,26 +305,15 @@ def downgrade(): op.drop_index(op.f('ix_process_instance_process_model_display_name'), table_name='process_instance') op.drop_table('process_instance') op.drop_table('principal') - op.drop_table('bpmn_process_definition_relationship') op.drop_table('user') op.drop_table('spiff_logging') op.drop_index(op.f('ix_spec_reference_cache_type'), table_name='spec_reference_cache') op.drop_index(op.f('ix_spec_reference_cache_identifier'), table_name='spec_reference_cache') op.drop_index(op.f('ix_spec_reference_cache_display_name'), table_name='spec_reference_cache') op.drop_table('spec_reference_cache') - op.drop_index(op.f('ix_serialized_bpmn_definition_hash'), table_name='serialized_bpmn_definition') - op.drop_table('serialized_bpmn_definition') - op.drop_table('process_instance_data') op.drop_table('permission_target') op.drop_index(op.f('ix_message_triggerable_process_model_process_model_identifier'), table_name='message_triggerable_process_model') op.drop_table('message_triggerable_process_model') - op.drop_index(op.f('ix_json_data_hash'), table_name='json_data') - op.drop_table('json_data') op.drop_table('group') op.drop_table('correlation_property_cache') - op.drop_index(op.f('ix_bpmn_process_definition_hash'), table_name='bpmn_process_definition') - op.drop_index(op.f('ix_bpmn_process_definition_bpmn_identifier'), table_name='bpmn_process_definition') - op.drop_table('bpmn_process_definition') - op.drop_index(op.f('ix_bpmn_process_json_data_hash'), table_name='bpmn_process') - op.drop_table('bpmn_process') # ### end Alembic commands ### diff --git a/spiffworkflow-backend/migrations/versions/d6fd61fd64d7_.py b/spiffworkflow-backend/migrations/versions/d6fd61fd64d7_.py new file mode 100644 index 000000000..18bf29197 --- /dev/null +++ b/spiffworkflow-backend/migrations/versions/d6fd61fd64d7_.py @@ -0,0 +1,107 @@ +"""empty message + +Revision ID: d6fd61fd64d7 +Revises: 8930711a75a4 +Create Date: 2023-03-03 12:06:39.987943 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'd6fd61fd64d7' +down_revision = '8930711a75a4' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('bpmn_process', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('parent_process_id', sa.Integer(), nullable=True), + sa.Column('properties_json', sa.JSON(), nullable=False), + sa.Column('json_data_hash', sa.String(length=255), nullable=False), + sa.ForeignKeyConstraint(['parent_process_id'], ['bpmn_process.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_bpmn_process_json_data_hash'), 'bpmn_process', ['json_data_hash'], unique=False) + op.create_table('bpmn_process_definition', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('hash', sa.String(length=255), nullable=False), + sa.Column('bpmn_identifier', sa.String(length=255), nullable=False), + sa.Column('properties_json', sa.JSON(), nullable=False), + sa.Column('type', sa.String(length=32), nullable=True), + sa.Column('bpmn_version_control_type', sa.String(length=50), nullable=True), + sa.Column('bpmn_version_control_identifier', sa.String(length=255), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_bpmn_process_definition_bpmn_identifier'), 'bpmn_process_definition', ['bpmn_identifier'], unique=False) + op.create_index(op.f('ix_bpmn_process_definition_hash'), 'bpmn_process_definition', ['hash'], unique=True) + op.create_table('json_data', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('hash', sa.String(length=255), nullable=False), + sa.Column('data', sa.JSON(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_json_data_hash'), 'json_data', ['hash'], unique=True) + op.create_table('bpmn_process_definition_relationship', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('bpmn_process_definition_parent_id', sa.Integer(), nullable=False), + sa.Column('bpmn_process_definition_child_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['bpmn_process_definition_child_id'], ['bpmn_process_definition.id'], ), + sa.ForeignKeyConstraint(['bpmn_process_definition_parent_id'], ['bpmn_process_definition.id'], ), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('bpmn_process_definition_parent_id', 'bpmn_process_definition_child_id', name='bpmn_process_definition_relationship_unique') + ) + op.create_table('task_definition', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('bpmn_process_definition_id', sa.Integer(), nullable=False), + sa.Column('bpmn_identifier', sa.String(length=255), nullable=False), + sa.Column('properties_json', sa.JSON(), nullable=False), + sa.Column('typename', sa.String(length=255), nullable=False), + sa.ForeignKeyConstraint(['bpmn_process_definition_id'], ['bpmn_process_definition.id'], ), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('bpmn_process_definition_id', 'bpmn_identifier', name='task_definition_unique') + ) + op.create_index(op.f('ix_task_definition_bpmn_identifier'), 'task_definition', ['bpmn_identifier'], unique=False) + op.create_table('task', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('guid', sa.String(length=36), nullable=False), + sa.Column('bpmn_process_id', sa.Integer(), nullable=False), + sa.Column('task_definition_id', sa.Integer(), nullable=False), + sa.Column('state', sa.String(length=10), nullable=False), + sa.Column('properties_json', sa.JSON(), nullable=False), + sa.Column('json_data_hash', sa.String(length=255), nullable=False), + sa.ForeignKeyConstraint(['bpmn_process_id'], ['bpmn_process.id'], ), + sa.ForeignKeyConstraint(['task_definition_id'], ['task_definition.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_task_guid'), 'task', ['guid'], unique=True) + op.create_index(op.f('ix_task_json_data_hash'), 'task', ['json_data_hash'], unique=False) + op.add_column('process_instance', sa.Column('bpmn_process_definition_id', sa.Integer(), nullable=True)) + op.add_column('process_instance', sa.Column('spiff_serializer_version', sa.String(length=50), nullable=True)) + op.create_foreign_key(None, 'process_instance', 'bpmn_process_definition', ['bpmn_process_definition_id'], ['id']) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint(None, 'process_instance', type_='foreignkey') + op.drop_column('process_instance', 'spiff_serializer_version') + op.drop_column('process_instance', 'bpmn_process_definition_id') + op.drop_index(op.f('ix_task_json_data_hash'), table_name='task') + op.drop_index(op.f('ix_task_guid'), table_name='task') + op.drop_table('task') + op.drop_index(op.f('ix_task_definition_bpmn_identifier'), table_name='task_definition') + op.drop_table('task_definition') + op.drop_table('bpmn_process_definition_relationship') + op.drop_index(op.f('ix_json_data_hash'), table_name='json_data') + op.drop_table('json_data') + op.drop_index(op.f('ix_bpmn_process_definition_hash'), table_name='bpmn_process_definition') + op.drop_index(op.f('ix_bpmn_process_definition_bpmn_identifier'), table_name='bpmn_process_definition') + op.drop_table('bpmn_process_definition') + op.drop_index(op.f('ix_bpmn_process_json_data_hash'), table_name='bpmn_process') + op.drop_table('bpmn_process') + # ### end Alembic commands ### diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process_definition.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process_definition.py index eec715c37..af655408e 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process_definition.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process_definition.py @@ -17,9 +17,7 @@ class BpmnProcessDefinitionModel(SpiffworkflowBaseDBModel): # this is a sha256 hash of spec and serializer_version hash: str = db.Column(db.String(255), nullable=False, index=True, unique=True) - bpmn_identifier: str = db.Column( - db.String(255), nullable=False, index=True - ) + bpmn_identifier: str = db.Column(db.String(255), nullable=False, index=True) properties_json: str = db.Column(db.JSON, nullable=False) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/task_definition.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/task_definition.py index f4f5c997f..8abbb138b 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/task_definition.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/task_definition.py @@ -27,8 +27,6 @@ class TaskDefinitionModel(SpiffworkflowBaseDBModel): ) bpmn_process_definition = relationship(BpmnProcessDefinitionModel) - bpmn_identifier: str = db.Column( - db.String(255), nullable=False, index=True - ) + bpmn_identifier: str = db.Column(db.String(255), nullable=False, index=True) properties_json: dict = db.Column(db.JSON, nullable=False) typename: str = db.Column(db.String(255), nullable=False) From 4eef5851369c5df90d35f271128396cc81917c36 Mon Sep 17 00:00:00 2001 From: jasquat Date: Fri, 3 Mar 2023 13:08:30 -0500 Subject: [PATCH 09/25] recreated migration from new one in main --- .../versions/{d6fd61fd64d7_.py => a63a61a21398_.py} | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) rename spiffworkflow-backend/migrations/versions/{d6fd61fd64d7_.py => a63a61a21398_.py} (97%) diff --git a/spiffworkflow-backend/migrations/versions/d6fd61fd64d7_.py b/spiffworkflow-backend/migrations/versions/a63a61a21398_.py similarity index 97% rename from spiffworkflow-backend/migrations/versions/d6fd61fd64d7_.py rename to spiffworkflow-backend/migrations/versions/a63a61a21398_.py index 18bf29197..2a4524b15 100644 --- a/spiffworkflow-backend/migrations/versions/d6fd61fd64d7_.py +++ b/spiffworkflow-backend/migrations/versions/a63a61a21398_.py @@ -1,8 +1,8 @@ """empty message -Revision ID: d6fd61fd64d7 -Revises: 8930711a75a4 -Create Date: 2023-03-03 12:06:39.987943 +Revision ID: a63a61a21398 +Revises: 1e8167de6df8 +Create Date: 2023-03-03 13:07:51.665585 """ from alembic import op @@ -10,8 +10,8 @@ import sqlalchemy as sa # revision identifiers, used by Alembic. -revision = 'd6fd61fd64d7' -down_revision = '8930711a75a4' +revision = 'a63a61a21398' +down_revision = '1e8167de6df8' branch_labels = None depends_on = None From 43513de4f57c0d34549a07f11cca3a23752c639e Mon Sep 17 00:00:00 2001 From: jasquat Date: Fri, 3 Mar 2023 13:13:01 -0500 Subject: [PATCH 10/25] use the correct db collation for backend ci --- .github/workflows/backend_tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/backend_tests.yml b/.github/workflows/backend_tests.yml index 99dff06f8..71d1ad8be 100644 --- a/.github/workflows/backend_tests.yml +++ b/.github/workflows/backend_tests.yml @@ -142,6 +142,7 @@ jobs: mysql version: "8.0" mysql database: "spiffworkflow_backend_unit_testing" mysql root password: password + collation server: 'utf8mb4_0900_as_cs' if: matrix.database == 'mysql' - name: Setup Postgres From 2cdacfa03bd99b31a295579f06f0c7ec7ca9e83c Mon Sep 17 00:00:00 2001 From: jasquat Date: Fri, 3 Mar 2023 16:51:24 -0500 Subject: [PATCH 11/25] some stuff is passing but still needs the process_instance_data w/ burnettk --- spiffworkflow-backend/conftest.py | 3 + .../migrations/versions/61cd3e8462f5_.py | 30 ++++ .../migrations/versions/941f7b76d278_.py | 30 ++++ .../migrations/versions/b0b82fab4cf9_.py | 30 ++++ .../models/bpmn_process.py | 4 +- .../models/bpmn_process_definition.py | 5 +- .../models/process_instance.py | 8 +- .../src/spiffworkflow_backend/models/task.py | 6 +- .../services/process_instance_processor.py | 135 ++++++++++++++---- .../unit/test_message_service.py | 1 - 10 files changed, 216 insertions(+), 36 deletions(-) create mode 100644 spiffworkflow-backend/migrations/versions/61cd3e8462f5_.py create mode 100644 spiffworkflow-backend/migrations/versions/941f7b76d278_.py create mode 100644 spiffworkflow-backend/migrations/versions/b0b82fab4cf9_.py diff --git a/spiffworkflow-backend/conftest.py b/spiffworkflow-backend/conftest.py index 9c6c242e4..3e4804a8b 100644 --- a/spiffworkflow-backend/conftest.py +++ b/spiffworkflow-backend/conftest.py @@ -5,6 +5,7 @@ import shutil import pytest from flask.app import Flask from flask.testing import FlaskClient +from spiffworkflow_backend.models.bpmn_process import BpmnProcessModel from tests.spiffworkflow_backend.helpers.base_test import BaseTest from spiffworkflow_backend.models.db import db @@ -46,6 +47,8 @@ def app() -> Flask: def with_db_and_bpmn_file_cleanup() -> None: """Do it cleanly!""" meta = db.metadata + db.session.execute(db.update(BpmnProcessModel, values={"parent_process_id": None})) + for table in reversed(meta.sorted_tables): db.session.execute(table.delete()) db.session.commit() diff --git a/spiffworkflow-backend/migrations/versions/61cd3e8462f5_.py b/spiffworkflow-backend/migrations/versions/61cd3e8462f5_.py new file mode 100644 index 000000000..c6a5f000e --- /dev/null +++ b/spiffworkflow-backend/migrations/versions/61cd3e8462f5_.py @@ -0,0 +1,30 @@ +"""empty message + +Revision ID: 61cd3e8462f5 +Revises: 941f7b76d278 +Create Date: 2023-03-03 16:22:12.449757 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import mysql + +# revision identifiers, used by Alembic. +revision = '61cd3e8462f5' +down_revision = '941f7b76d278' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint('task_ibfk_2', 'task', type_='foreignkey') + op.drop_column('task', 'task_definition_id') + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('task', sa.Column('task_definition_id', mysql.INTEGER(), autoincrement=False, nullable=False)) + op.create_foreign_key('task_ibfk_2', 'task', 'task_definition', ['task_definition_id'], ['id']) + # ### end Alembic commands ### diff --git a/spiffworkflow-backend/migrations/versions/941f7b76d278_.py b/spiffworkflow-backend/migrations/versions/941f7b76d278_.py new file mode 100644 index 000000000..8705894a8 --- /dev/null +++ b/spiffworkflow-backend/migrations/versions/941f7b76d278_.py @@ -0,0 +1,30 @@ +"""empty message + +Revision ID: 941f7b76d278 +Revises: b0b82fab4cf9 +Create Date: 2023-03-03 14:40:46.574985 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '941f7b76d278' +down_revision = 'b0b82fab4cf9' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('bpmn_process', sa.Column('guid', sa.String(length=36), nullable=True)) + op.create_index(op.f('ix_bpmn_process_guid'), 'bpmn_process', ['guid'], unique=True) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f('ix_bpmn_process_guid'), table_name='bpmn_process') + op.drop_column('bpmn_process', 'guid') + # ### end Alembic commands ### diff --git a/spiffworkflow-backend/migrations/versions/b0b82fab4cf9_.py b/spiffworkflow-backend/migrations/versions/b0b82fab4cf9_.py new file mode 100644 index 000000000..61d199450 --- /dev/null +++ b/spiffworkflow-backend/migrations/versions/b0b82fab4cf9_.py @@ -0,0 +1,30 @@ +"""empty message + +Revision ID: b0b82fab4cf9 +Revises: a63a61a21398 +Create Date: 2023-03-03 13:24:08.304492 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'b0b82fab4cf9' +down_revision = 'a63a61a21398' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('process_instance', sa.Column('bpmn_process_id', sa.Integer(), nullable=True)) + op.create_foreign_key(None, 'process_instance', 'bpmn_process', ['bpmn_process_id'], ['id']) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint(None, 'process_instance', type_='foreignkey') + op.drop_column('process_instance', 'bpmn_process_id') + # ### end Alembic commands ### diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process.py index 0d7e556d9..b5a3137e6 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process.py @@ -1,4 +1,5 @@ from __future__ import annotations +from typing import Optional from sqlalchemy import ForeignKey @@ -15,8 +16,9 @@ from spiffworkflow_backend.models.db import SpiffworkflowBaseDBModel class BpmnProcessModel(SpiffworkflowBaseDBModel): __tablename__ = "bpmn_process" id: int = db.Column(db.Integer, primary_key=True) + guid: Optional[str] = db.Column(db.String(36), nullable=True, unique=True, index=True) - parent_process_id: int = db.Column(ForeignKey("bpmn_process.id"), nullable=True) + parent_process_id: Optional[int] = db.Column(ForeignKey("bpmn_process.id"), nullable=True) properties_json: dict = db.Column(db.JSON, nullable=False) json_data_hash: str = db.Column(db.String(255), nullable=False, index=True) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process_definition.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process_definition.py index af655408e..d689a8f2f 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process_definition.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process_definition.py @@ -15,11 +15,14 @@ class BpmnProcessDefinitionModel(SpiffworkflowBaseDBModel): id: int = db.Column(db.Integer, primary_key=True) # this is a sha256 hash of spec and serializer_version + # note that a call activity is its own row in this table, with its own hash, + # and therefore it only gets stored once per version, and can be reused + # by multiple calling processes. hash: str = db.Column(db.String(255), nullable=False, index=True, unique=True) bpmn_identifier: str = db.Column(db.String(255), nullable=False, index=True) - properties_json: str = db.Column(db.JSON, nullable=False) + properties_json: dict = db.Column(db.JSON, nullable=False) # process or subprocess # FIXME: will probably ignore for now since we do not strictly need it diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/process_instance.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/process_instance.py index e0fb3f3b2..5b3733710 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/process_instance.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/process_instance.py @@ -1,5 +1,7 @@ """Process_instance.""" from __future__ import annotations +from typing import Optional +from spiffworkflow_backend.models.bpmn_process import BpmnProcessModel from typing import Any from typing import cast @@ -77,10 +79,14 @@ class ProcessInstanceModel(SpiffworkflowBaseDBModel): ) process_instance_data = relationship("ProcessInstanceDataModel", cascade="delete") - bpmn_process_definition_id: int = db.Column( + bpmn_process_definition_id: Optional[int] = db.Column( ForeignKey(BpmnProcessDefinitionModel.id), nullable=True # type: ignore ) bpmn_process_definition = relationship(BpmnProcessDefinitionModel) + bpmn_process_id: Optional[int] = db.Column( + ForeignKey(BpmnProcessModel.id), nullable=True # type: ignore + ) + bpmn_process = relationship(BpmnProcessModel) spiff_serializer_version = db.Column(db.String(50), nullable=True) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/task.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/task.py index 8038a14be..b1bba7158 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/task.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/task.py @@ -48,9 +48,9 @@ class TaskModel(SpiffworkflowBaseDBModel): ) # find this by looking up the "workflow_name" and "task_spec" from the properties_json - task_definition_id: int = db.Column( - ForeignKey(TaskDefinitionModel.id), nullable=False # type: ignore - ) + # task_definition_id: int = db.Column( + # ForeignKey(TaskDefinitionModel.id), nullable=False # type: ignore + # ) state: str = db.Column(db.String(10), nullable=False) properties_json: dict = db.Column(db.JSON, nullable=False) json_data_hash: str = db.Column(db.String(255), nullable=False, index=True) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py b/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py index 7ec181659..c2db1a463 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py @@ -1,5 +1,7 @@ """Process_instance_processor.""" import _strptime # type: ignore +from SpiffWorkflow.task import TaskStateNames # type: ignore +from spiffworkflow_backend.models.task import TaskModel # noqa: F401 import decimal import json import logging @@ -55,6 +57,7 @@ from SpiffWorkflow.util.deep_merge import DeepMerge # type: ignore from sqlalchemy import text from spiffworkflow_backend.exceptions.api_error import ApiError +from spiffworkflow_backend.models.bpmn_process import BpmnProcessModel from spiffworkflow_backend.models.bpmn_process_definition import ( BpmnProcessDefinitionModel, ) @@ -67,6 +70,7 @@ from spiffworkflow_backend.models.file import FileType from spiffworkflow_backend.models.group import GroupModel from spiffworkflow_backend.models.human_task import HumanTaskModel from spiffworkflow_backend.models.human_task_user import HumanTaskUserModel +from spiffworkflow_backend.models.json_data import JsonDataModel from spiffworkflow_backend.models.message_instance import MessageInstanceModel from spiffworkflow_backend.models.message_instance_correlation import ( MessageInstanceCorrelationRuleModel, @@ -534,16 +538,27 @@ class ProcessInstanceProcessor: task_definitions = TaskDefinitionModel.query.filter_by( bpmn_process_definition_id=bpmn_process_definition.id ).all() - bpmn_process_definition_dict: dict = json.loads( - bpmn_process_definition.properties_json - ) + bpmn_process_definition_dict: dict = bpmn_process_definition.properties_json bpmn_process_definition_dict["task_specs"] = {} for task_definition in task_definitions: bpmn_process_definition_dict["task_specs"][ task_definition.bpmn_identifier - ] = json.loads(task_definition.properties_json) + ] = task_definition.properties_json return bpmn_process_definition_dict + @classmethod + def _get_bpmn_process_dict(cls, bpmn_process: BpmnProcessModel) -> dict: + json_data = JsonDataModel.query.filter_by(hash=bpmn_process.json_data_hash).first() + bpmn_process_dict = {'data': json_data.data, 'tasks': {}} + bpmn_process_dict.update(bpmn_process.properties_json) + tasks = TaskModel.query.filter_by(bpmn_process_id=bpmn_process.id).all() + for task in tasks: + json_data = JsonDataModel.query.filter_by(hash=task.json_data_hash).first() + bpmn_process_dict['tasks'][task.guid] = task.properties_json + bpmn_process_dict['tasks'][task.guid]['data'] = json_data.data + + return bpmn_process_dict + @classmethod def _get_full_bpmn_json(cls, process_instance_model: ProcessInstanceModel) -> dict: if process_instance_model.serialized_bpmn_definition_id is None: @@ -552,16 +567,14 @@ class ProcessInstanceProcessor: # print(f"serialized_bpmn_definition.static_json: {serialized_bpmn_definition.static_json}") # loaded_json: dict = json.loads(serialized_bpmn_definition.static_json) # or "{}") - serialized_bpmn_definition: dict = {} - bpmn_process_definition = BpmnProcessDefinitionModel.query.filter_by( - id=process_instance_model.bpmn_process_definition_id - ).first() + serialized_bpmn_definition: dict = { + "serializer_version": process_instance_model.spiff_serializer_version, + "spec": {}, + "subprocess_specs": {}, + "subprocesses": {} + } + bpmn_process_definition = process_instance_model.bpmn_process_definition if bpmn_process_definition is not None: - serialized_bpmn_definition = { - "serializer_version": process_instance_model.spiff_serializer_version, - "spec": {}, - "subprocess_specs": {}, - } serialized_bpmn_definition["spec"] = ( cls._get_definition_dict_for_bpmn_process_definition( bpmn_process_definition @@ -580,11 +593,21 @@ class ProcessInstanceProcessor: serialized_bpmn_definition["subprocess_specs"][ bpmn_subprocess_definition.bpmn_identifier ] = spec - loaded_json: dict = serialized_bpmn_definition - process_instance_data = process_instance_model.process_instance_data - loaded_json.update(json.loads(process_instance_data.runtime_json)) - return loaded_json + bpmn_process = process_instance_model.bpmn_process + if bpmn_process is not None: + bpmn_process_dict = cls._get_bpmn_process_dict(bpmn_process) + serialized_bpmn_definition.update(bpmn_process_dict) + + bpmn_subprocesses = BpmnProcessModel.query.filter_by(parent_process_id=bpmn_process.id).all() + for bpmn_subprocess in bpmn_subprocesses: + bpmn_process_dict = cls._get_bpmn_process_dict(bpmn_subprocess) + serialized_bpmn_definition['subprocesses'][bpmn_subprocess.guid] = bpmn_process_dict + + # process_instance_data = process_instance_model.process_instance_data + # loaded_json.update(json.loads(process_instance_data.runtime_json)) + + return serialized_bpmn_definition def current_user(self) -> Any: """Current_user.""" @@ -979,7 +1002,7 @@ class ProcessInstanceProcessor: bpmn_process_definition = BpmnProcessDefinitionModel( hash=new_hash_digest, bpmn_identifier=process_bpmn_identifier, - properties_json=json.dumps(process_bpmn_properties), + properties_json=process_bpmn_properties, ) db.session.add(bpmn_process_definition) @@ -987,7 +1010,7 @@ class ProcessInstanceProcessor: task_definition = TaskDefinitionModel( bpmn_process_definition=bpmn_process_definition, bpmn_identifier=task_bpmn_identifier, - properties_json=json.dumps(task_bpmn_properties), + properties_json=task_bpmn_properties, typename=task_bpmn_properties["typename"], ) db.session.add(task_definition) @@ -1019,12 +1042,71 @@ class ProcessInstanceProcessor: bpmn_process_definition_parent ) + def _add_bpmn_process(self, bpmn_process_dict: dict, bpmn_process_parent: Optional[BpmnProcessModel] = None, bpmn_process_guid: Optional[str] = None) -> BpmnProcessModel: + tasks = bpmn_process_dict.pop("tasks") + bpmn_process_data = bpmn_process_dict.pop("data") + + bpmn_process = None + if bpmn_process_parent is not None: + bpmn_process = BpmnProcessModel.query.filter_by(parent_process_id=bpmn_process_parent.id, guid=bpmn_process_guid).first() + elif self.process_instance_model.bpmn_process_id is not None: + bpmn_process = self.process_instance_model.bpmn_process + + if bpmn_process is None: + bpmn_process = BpmnProcessModel() + + bpmn_process.properties_json = bpmn_process_dict + + bpmn_process_data_json = json.dumps(bpmn_process_data, sort_keys=True).encode("utf8") + bpmn_process_data_hash = sha256(bpmn_process_data_json).hexdigest() + if bpmn_process.json_data_hash != bpmn_process_data_hash: + json_data = db.session.query(JsonDataModel.id).filter_by(hash=bpmn_process_data_hash).first() + if json_data is None: + json_data = JsonDataModel(hash=bpmn_process_data_hash, data=bpmn_process_data) + db.session.add(json_data) + bpmn_process.json_data_hash = bpmn_process_data_hash + + if bpmn_process_parent is None: + self.process_instance_model.bpmn_process = bpmn_process + elif bpmn_process.parent_process_id is None: + bpmn_process.parent_process_id = bpmn_process_parent.id + db.session.add(bpmn_process) + + for task_id, task_properties in tasks.items(): + task_data_dict = task_properties.pop('data') + state_int = task_properties['state'] + + task = TaskModel.query.filter_by(guid=task_id).first() + if task is None: + # bpmn_process_identifier = task_properties['workflow_name'] + # bpmn_identifier = task_properties['task_spec'] + # + # task_definition = TaskDefinitionModel.query.filter_by(bpmn_identifier=bpmn_identifier).join(BpmnProcessDefinitionModel).filter(BpmnProcessDefinitionModel.bpmn_identifier==bpmn_process_identifier).first() + # if task_definition is None: + # subprocess_task = TaskModel.query.filter_by(guid=bpmn_process.guid) + task = TaskModel(guid=task_id, bpmn_process_id=bpmn_process.id) + task.state = TaskStateNames[state_int] + task.properties_json = task_properties + + task_data_json = json.dumps(task_data_dict, sort_keys=True).encode("utf8") + task_data_hash = sha256(task_data_json).hexdigest() + if task.json_data_hash != task_data_hash: + json_data = db.session.query(JsonDataModel.id).filter_by(hash=task_data_hash).first() + if json_data is None: + json_data = JsonDataModel(hash=task_data_hash, data=task_data_dict) + db.session.add(json_data) + task.json_data_hash = task_data_hash + db.session.add(task) + + return bpmn_process + def _add_bpmn_json_records_new(self) -> None: """Adds serialized_bpmn_definition and process_instance_data records to the db session. Expects the save method to commit it. """ bpmn_dict = json.loads(self.serialize()) + # with open('tmp2.json', 'w') as f: f.write(json.dumps(bpmn_dict) bpmn_dict_keys = ("spec", "subprocess_specs", "serializer_version") process_instance_data_dict = {} bpmn_spec_dict = {} @@ -1038,19 +1120,14 @@ class ProcessInstanceProcessor: # if self.process_instance_model.bpmn_process_definition_id is None: self._add_bpmn_process_definitions(bpmn_spec_dict) - # process_instance_data = None - # if self.process_instance_model.process_instance_data_id is None: - # process_instance_data = ProcessInstanceDataModel() - # else: - # process_instance_data = self.process_instance_model.process_instance_data - # - # process_instance_data.runtime_json = json.dumps(process_instance_data_dict) - # db.session.add(process_instance_data) - # self.process_instance_model.process_instance_data = process_instance_data + subprocesses = process_instance_data_dict.pop("subprocesses") + bpmn_process_parent = self._add_bpmn_process(process_instance_data_dict) + for _subprocess_task_id, subprocess_properties in subprocesses.items(): + self._add_bpmn_process(subprocess_properties, bpmn_process_parent) def save(self) -> None: """Saves the current state of this processor to the database.""" - self._add_bpmn_json_records() + # self._add_bpmn_json_records() self._add_bpmn_json_records_new() self.process_instance_model.spiff_serializer_version = self.SERIALIZER_VERSION diff --git a/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_message_service.py b/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_message_service.py index e30d9dcdb..c0898a042 100644 --- a/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_message_service.py +++ b/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_message_service.py @@ -71,7 +71,6 @@ class TestMessageService(BaseTest): .all() ) assert len(waiting_messages) == 0 - # The process has completed assert self.process_instance.status == "complete" From 6acd47e67e19d54e4da79ec3abcd73d5d8074943 Mon Sep 17 00:00:00 2001 From: jasquat Date: Mon, 6 Mar 2023 11:06:03 -0500 Subject: [PATCH 12/25] tests are passing with new db tables w/ burnettk --- .../routes/process_instances_controller.py | 19 ++--- .../services/process_instance_processor.py | 82 ++++++++----------- 2 files changed, 44 insertions(+), 57 deletions(-) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/routes/process_instances_controller.py b/spiffworkflow-backend/src/spiffworkflow_backend/routes/process_instances_controller.py index 918d1749e..d33a1c000 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/routes/process_instances_controller.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/routes/process_instances_controller.py @@ -549,13 +549,11 @@ def process_instance_task_list( step_details = step_detail_query.all() - process_instance_data = process_instance.process_instance_data - process_instance_data_json = ( - "{}" if process_instance_data is None else process_instance_data.runtime_json - ) - process_instance_data_dict = json.loads(process_instance_data_json) - tasks = process_instance_data_dict["tasks"] - subprocesses = process_instance_data_dict["subprocesses"] + processor = ProcessInstanceProcessor(process_instance) + full_bpmn_process_dict = processor.full_bpmn_process_dict + + tasks = full_bpmn_process_dict["tasks"] + subprocesses = full_bpmn_process_dict["subprocesses"] steps_by_id = {step_detail.task_id: step_detail for step_detail in step_details} @@ -588,18 +586,17 @@ def process_instance_task_list( spiff_task_id, TaskState.FUTURE ) - process_instance_data.runtime_json = json.dumps(process_instance_data_dict) + bpmn_process_instance = ProcessInstanceProcessor._serializer.workflow_from_dict(full_bpmn_process_dict) - processor = ProcessInstanceProcessor(process_instance) spiff_task = processor.__class__.get_task_by_bpmn_identifier( - step_details[-1].bpmn_task_identifier, processor.bpmn_process_instance + step_details[-1].bpmn_task_identifier, bpmn_process_instance ) if spiff_task is not None and spiff_task.state != TaskState.READY: spiff_task.complete() spiff_tasks = None if all_tasks: - spiff_tasks = processor.bpmn_process_instance.get_tasks(TaskState.ANY_MASK) + spiff_tasks = bpmn_process_instance.get_tasks(TaskState.ANY_MASK) else: spiff_tasks = processor.get_all_user_tasks() diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py b/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py index c2db1a463..6b41b44a7 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py @@ -453,8 +453,9 @@ class ProcessInstanceProcessor: self.process_instance_model = process_instance_model self.process_model_service = ProcessModelService() bpmn_process_spec = None + self.full_bpmn_process_dict = {} subprocesses: Optional[IdToBpmnProcessSpecMapping] = None - if process_instance_model.serialized_bpmn_definition_id is None: + if process_instance_model.bpmn_process_definition_id is None: ( bpmn_process_spec, subprocesses, @@ -468,7 +469,7 @@ class ProcessInstanceProcessor: ) try: - self.bpmn_process_instance = self.__get_bpmn_process_instance( + (self.bpmn_process_instance, self.full_bpmn_process_dict) = self.__get_bpmn_process_instance( process_instance_model, bpmn_process_spec, validate_only, @@ -546,6 +547,23 @@ class ProcessInstanceProcessor: ] = task_definition.properties_json return bpmn_process_definition_dict + @classmethod + def _set_definition_dict_for_bpmn_subprocess_definitions( + cls, bpmn_process_definition: BpmnProcessDefinitionModel, spiff_bpmn_process_dict: dict + ) -> None: + bpmn_process_subprocess_definitions = ( + BpmnProcessDefinitionRelationshipModel.query.filter_by( + bpmn_process_definition_parent_id=bpmn_process_definition.id + ).all() + ) + for bpmn_subprocess_definition in bpmn_process_subprocess_definitions: + spec = cls._set_definition_dict_for_bpmn_subprocess_definitions( + bpmn_subprocess_definition, spiff_bpmn_process_dict + ) + spiff_bpmn_process_dict["subprocess_specs"][ + bpmn_subprocess_definition.bpmn_identifier + ] = spec + @classmethod def _get_bpmn_process_dict(cls, bpmn_process: BpmnProcessModel) -> dict: json_data = JsonDataModel.query.filter_by(hash=bpmn_process.json_data_hash).first() @@ -560,14 +578,11 @@ class ProcessInstanceProcessor: return bpmn_process_dict @classmethod - def _get_full_bpmn_json(cls, process_instance_model: ProcessInstanceModel) -> dict: - if process_instance_model.serialized_bpmn_definition_id is None: + def _get_full_bpmn_process_dict(cls, process_instance_model: ProcessInstanceModel) -> dict: + if process_instance_model.bpmn_process_definition_id is None: return {} - # serialized_bpmn_definition = process_instance_model.serialized_bpmn_definition - # print(f"serialized_bpmn_definition.static_json: {serialized_bpmn_definition.static_json}") - # loaded_json: dict = json.loads(serialized_bpmn_definition.static_json) # or "{}") - serialized_bpmn_definition: dict = { + spiff_bpmn_process_dict: dict = { "serializer_version": process_instance_model.spiff_serializer_version, "spec": {}, "subprocess_specs": {}, @@ -575,39 +590,24 @@ class ProcessInstanceProcessor: } bpmn_process_definition = process_instance_model.bpmn_process_definition if bpmn_process_definition is not None: - serialized_bpmn_definition["spec"] = ( + spiff_bpmn_process_dict["spec"] = ( cls._get_definition_dict_for_bpmn_process_definition( bpmn_process_definition ) ) - - bpmn_process_subprocess_definitions = ( - BpmnProcessDefinitionRelationshipModel.query.filter_by( - bpmn_process_definition_parent_id=bpmn_process_definition.id - ).all() - ) - for bpmn_subprocess_definition in bpmn_process_subprocess_definitions: - spec = cls._get_definition_dict_for_bpmn_process_definition( - bpmn_subprocess_definition - ) - serialized_bpmn_definition["subprocess_specs"][ - bpmn_subprocess_definition.bpmn_identifier - ] = spec + cls._set_definition_dict_for_bpmn_subprocess_definitions(bpmn_process_definition, spiff_bpmn_process_dict) bpmn_process = process_instance_model.bpmn_process if bpmn_process is not None: bpmn_process_dict = cls._get_bpmn_process_dict(bpmn_process) - serialized_bpmn_definition.update(bpmn_process_dict) + spiff_bpmn_process_dict.update(bpmn_process_dict) bpmn_subprocesses = BpmnProcessModel.query.filter_by(parent_process_id=bpmn_process.id).all() for bpmn_subprocess in bpmn_subprocesses: bpmn_process_dict = cls._get_bpmn_process_dict(bpmn_subprocess) - serialized_bpmn_definition['subprocesses'][bpmn_subprocess.guid] = bpmn_process_dict + spiff_bpmn_process_dict['subprocesses'][bpmn_subprocess.guid] = bpmn_process_dict - # process_instance_data = process_instance_model.process_instance_data - # loaded_json.update(json.loads(process_instance_data.runtime_json)) - - return serialized_bpmn_definition + return spiff_bpmn_process_dict def current_user(self) -> Any: """Current_user.""" @@ -643,22 +643,18 @@ class ProcessInstanceProcessor: validate_only: bool = False, subprocesses: Optional[IdToBpmnProcessSpecMapping] = None, ) -> BpmnWorkflow: - """__get_bpmn_process_instance.""" - if process_instance_model.serialized_bpmn_definition_id is not None: + full_bpmn_process_dict = {} + if process_instance_model.bpmn_process_definition_id is not None: # turn off logging to avoid duplicated spiff logs spiff_logger = logging.getLogger("spiff") original_spiff_logger_log_level = spiff_logger.level spiff_logger.setLevel(logging.WARNING) try: - full_bpmn_json = ProcessInstanceProcessor._get_full_bpmn_json( + full_bpmn_process_dict = ProcessInstanceProcessor._get_full_bpmn_process_dict( process_instance_model ) - bpmn_process_instance = ( - ProcessInstanceProcessor._serializer.deserialize_json( - json.dumps(full_bpmn_json) - ) - ) + bpmn_process_instance = ProcessInstanceProcessor._serializer.workflow_from_dict(full_bpmn_process_dict) except Exception as err: raise err finally: @@ -674,7 +670,7 @@ class ProcessInstanceProcessor: bpmn_process_instance.data[ ProcessInstanceProcessor.VALIDATION_PROCESS_KEY ] = validate_only - return bpmn_process_instance + return (bpmn_process_instance, full_bpmn_process_dict) def slam_in_data(self, data: dict) -> None: """Slam_in_data.""" @@ -814,9 +810,7 @@ class ProcessInstanceProcessor: Rerturns: {process_name: [task_1, task_2, ...], ...} """ - bpmn_definition_dict = json.loads( - self.process_instance_model.serialized_bpmn_definition.static_json or "{}" - ) + bpmn_definition_dict = self.full_bpmn_process_dict processes: dict[str, list[str]] = {bpmn_definition_dict["spec"]["name"]: []} for task_name, _task_spec in bpmn_definition_dict["spec"]["task_specs"].items(): processes[bpmn_definition_dict["spec"]["name"]].append(task_name) @@ -864,9 +858,7 @@ class ProcessInstanceProcessor: NOTE: this may not fully work for tasks that are NOT call activities since their task_name may not be unique but in our current use case we only care about the call activities here. """ - bpmn_definition_dict = json.loads( - self.process_instance_model.serialized_bpmn_definition.static_json or "{}" - ) + bpmn_definition_dict = self.full_bpmn_process_dict spiff_task_json = bpmn_definition_dict["spec"]["task_specs"] or {} if "subprocess_specs" in bpmn_definition_dict: for _subprocess_name, subprocess_details in bpmn_definition_dict[ @@ -891,9 +883,7 @@ class ProcessInstanceProcessor: Also note that subprocess_task_id might in fact be a call activity, because spiff treats call activities like subprocesses in terms of the serialization. """ - process_instance_data_dict = json.loads( - self.process_instance_model.process_instance_data.runtime_json or "{}" - ) + process_instance_data_dict = self.full_bpmn_process_dict spiff_task_json = self.get_all_task_specs() subprocesses_by_child_task_ids = {} From e55485bd4a9c20227e888c301f6861f25279ecac Mon Sep 17 00:00:00 2001 From: jasquat Date: Mon, 6 Mar 2023 11:13:53 -0500 Subject: [PATCH 13/25] removed process instance data and serialized bpmn definition models w/ burnettk --- spiffworkflow-backend/migrations/env.py | 2 + .../migrations/versions/1e8167de6df8_.py | 43 ------- .../{ac6b60d7fee9_.py => 55b76c4528c5_.py} | 106 ++++++++++++++++- .../migrations/versions/61cd3e8462f5_.py | 30 ----- .../migrations/versions/7422be14adc4_.py | 51 --------- .../migrations/versions/8930711a75a4_.py | 28 ----- .../migrations/versions/941f7b76d278_.py | 30 ----- .../migrations/versions/a63a61a21398_.py | 107 ------------------ .../migrations/versions/b0b82fab4cf9_.py | 30 ----- .../load_database_models.py | 6 - .../models/process_instance.py | 14 --- .../models/process_instance_data.py | 22 ---- .../models/serialized_bpmn_definition.py | 49 -------- .../services/process_instance_processor.py | 56 +-------- 14 files changed, 107 insertions(+), 467 deletions(-) delete mode 100644 spiffworkflow-backend/migrations/versions/1e8167de6df8_.py rename spiffworkflow-backend/migrations/versions/{ac6b60d7fee9_.py => 55b76c4528c5_.py} (73%) delete mode 100644 spiffworkflow-backend/migrations/versions/61cd3e8462f5_.py delete mode 100644 spiffworkflow-backend/migrations/versions/7422be14adc4_.py delete mode 100644 spiffworkflow-backend/migrations/versions/8930711a75a4_.py delete mode 100644 spiffworkflow-backend/migrations/versions/941f7b76d278_.py delete mode 100644 spiffworkflow-backend/migrations/versions/a63a61a21398_.py delete mode 100644 spiffworkflow-backend/migrations/versions/b0b82fab4cf9_.py delete mode 100644 spiffworkflow-backend/src/spiffworkflow_backend/models/process_instance_data.py delete mode 100644 spiffworkflow-backend/src/spiffworkflow_backend/models/serialized_bpmn_definition.py diff --git a/spiffworkflow-backend/migrations/env.py b/spiffworkflow-backend/migrations/env.py index 630e381ad..68feded2a 100644 --- a/spiffworkflow-backend/migrations/env.py +++ b/spiffworkflow-backend/migrations/env.py @@ -1,3 +1,5 @@ +from __future__ import with_statement + import logging from logging.config import fileConfig diff --git a/spiffworkflow-backend/migrations/versions/1e8167de6df8_.py b/spiffworkflow-backend/migrations/versions/1e8167de6df8_.py deleted file mode 100644 index 18668aed1..000000000 --- a/spiffworkflow-backend/migrations/versions/1e8167de6df8_.py +++ /dev/null @@ -1,43 +0,0 @@ -"""empty message - -Revision ID: 1e8167de6df8 -Revises: 8930711a75a4 -Create Date: 2023-03-02 15:18:19.515864 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = '1e8167de6df8' -down_revision = '8930711a75a4' -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.create_table('process_instance_file_data', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('process_instance_id', sa.Integer(), nullable=False), - sa.Column('identifier', sa.String(length=255), nullable=False), - sa.Column('list_index', sa.Integer(), nullable=True), - sa.Column('mimetype', sa.String(length=255), nullable=False), - sa.Column('filename', sa.String(length=255), nullable=False), - sa.Column('contents', sa.LargeBinary(), nullable=False), - sa.Column('digest', sa.String(length=64), nullable=False), - sa.Column('updated_at_in_seconds', sa.Integer(), nullable=False), - sa.Column('created_at_in_seconds', sa.Integer(), nullable=False), - sa.ForeignKeyConstraint(['process_instance_id'], ['process_instance.id'], ), - sa.PrimaryKeyConstraint('id') - ) - op.create_index(op.f('ix_process_instance_file_data_digest'), 'process_instance_file_data', ['digest'], unique=False) - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.drop_index(op.f('ix_process_instance_file_data_digest'), table_name='process_instance_file_data') - op.drop_table('process_instance_file_data') - # ### end Alembic commands ### diff --git a/spiffworkflow-backend/migrations/versions/ac6b60d7fee9_.py b/spiffworkflow-backend/migrations/versions/55b76c4528c5_.py similarity index 73% rename from spiffworkflow-backend/migrations/versions/ac6b60d7fee9_.py rename to spiffworkflow-backend/migrations/versions/55b76c4528c5_.py index 520af3804..efe373eb4 100644 --- a/spiffworkflow-backend/migrations/versions/ac6b60d7fee9_.py +++ b/spiffworkflow-backend/migrations/versions/55b76c4528c5_.py @@ -1,8 +1,8 @@ """empty message -Revision ID: ac6b60d7fee9 +Revision ID: 55b76c4528c5 Revises: -Create Date: 2023-02-27 22:02:11.465980 +Create Date: 2023-03-06 11:11:55.431564 """ from alembic import op @@ -10,7 +10,7 @@ import sqlalchemy as sa # revision identifiers, used by Alembic. -revision = 'ac6b60d7fee9' +revision = '55b76c4528c5' down_revision = None branch_labels = None depends_on = None @@ -18,6 +18,29 @@ depends_on = None def upgrade(): # ### commands auto generated by Alembic - please adjust! ### + op.create_table('bpmn_process', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('guid', sa.String(length=36), nullable=True), + sa.Column('parent_process_id', sa.Integer(), nullable=True), + sa.Column('properties_json', sa.JSON(), nullable=False), + sa.Column('json_data_hash', sa.String(length=255), nullable=False), + sa.ForeignKeyConstraint(['parent_process_id'], ['bpmn_process.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_bpmn_process_guid'), 'bpmn_process', ['guid'], unique=True) + op.create_index(op.f('ix_bpmn_process_json_data_hash'), 'bpmn_process', ['json_data_hash'], unique=False) + op.create_table('bpmn_process_definition', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('hash', sa.String(length=255), nullable=False), + sa.Column('bpmn_identifier', sa.String(length=255), nullable=False), + sa.Column('properties_json', sa.JSON(), nullable=False), + sa.Column('type', sa.String(length=32), nullable=True), + sa.Column('bpmn_version_control_type', sa.String(length=50), nullable=True), + sa.Column('bpmn_version_control_identifier', sa.String(length=255), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_bpmn_process_definition_bpmn_identifier'), 'bpmn_process_definition', ['bpmn_identifier'], unique=False) + op.create_index(op.f('ix_bpmn_process_definition_hash'), 'bpmn_process_definition', ['hash'], unique=True) op.create_table('correlation_property_cache', sa.Column('id', sa.Integer(), nullable=False), sa.Column('name', sa.String(length=50), nullable=False), @@ -32,6 +55,13 @@ def upgrade(): sa.Column('identifier', sa.String(length=255), nullable=True), sa.PrimaryKeyConstraint('id') ) + op.create_table('json_data', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('hash', sa.String(length=255), nullable=False), + sa.Column('data', sa.JSON(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_json_data_hash'), 'json_data', ['hash'], unique=True) op.create_table('message_triggerable_process_model', sa.Column('id', sa.Integer(), nullable=False), sa.Column('message_name', sa.String(length=255), nullable=True), @@ -68,6 +98,7 @@ def upgrade(): sa.Column('id', sa.Integer(), nullable=False), sa.Column('process_instance_id', sa.Integer(), nullable=False), sa.Column('bpmn_process_identifier', sa.String(length=255), nullable=False), + sa.Column('bpmn_process_name', sa.String(length=255), nullable=True), sa.Column('bpmn_task_identifier', sa.String(length=255), nullable=False), sa.Column('bpmn_task_name', sa.String(length=255), nullable=True), sa.Column('bpmn_task_type', sa.String(length=255), nullable=True), @@ -94,6 +125,15 @@ def upgrade(): sa.UniqueConstraint('service', 'service_id', name='service_key'), sa.UniqueConstraint('username') ) + op.create_table('bpmn_process_definition_relationship', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('bpmn_process_definition_parent_id', sa.Integer(), nullable=False), + sa.Column('bpmn_process_definition_child_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['bpmn_process_definition_child_id'], ['bpmn_process_definition.id'], ), + sa.ForeignKeyConstraint(['bpmn_process_definition_parent_id'], ['bpmn_process_definition.id'], ), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('bpmn_process_definition_parent_id', 'bpmn_process_definition_child_id', name='bpmn_process_definition_relationship_unique') + ) op.create_table('principal', sa.Column('id', sa.Integer(), nullable=False), sa.Column('user_id', sa.Integer(), nullable=True), @@ -110,6 +150,9 @@ def upgrade(): sa.Column('process_model_identifier', sa.String(length=255), nullable=False), sa.Column('process_model_display_name', sa.String(length=255), nullable=False), sa.Column('process_initiator_id', sa.Integer(), nullable=False), + sa.Column('bpmn_process_definition_id', sa.Integer(), nullable=True), + sa.Column('bpmn_process_id', sa.Integer(), nullable=True), + sa.Column('spiff_serializer_version', sa.String(length=50), nullable=True), sa.Column('bpmn_json', sa.JSON(), nullable=True), sa.Column('start_in_seconds', sa.Integer(), nullable=True), sa.Column('end_in_seconds', sa.Integer(), nullable=True), @@ -121,6 +164,8 @@ def upgrade(): sa.Column('spiff_step', sa.Integer(), nullable=True), sa.Column('locked_by', sa.String(length=80), nullable=True), sa.Column('locked_at_in_seconds', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['bpmn_process_definition_id'], ['bpmn_process_definition.id'], ), + sa.ForeignKeyConstraint(['bpmn_process_id'], ['bpmn_process.id'], ), sa.ForeignKeyConstraint(['process_initiator_id'], ['user.id'], ), sa.PrimaryKeyConstraint('id') ) @@ -158,6 +203,29 @@ def upgrade(): sa.PrimaryKeyConstraint('id'), sa.UniqueConstraint('key') ) + op.create_table('task', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('guid', sa.String(length=36), nullable=False), + sa.Column('bpmn_process_id', sa.Integer(), nullable=False), + sa.Column('state', sa.String(length=10), nullable=False), + sa.Column('properties_json', sa.JSON(), nullable=False), + sa.Column('json_data_hash', sa.String(length=255), nullable=False), + sa.ForeignKeyConstraint(['bpmn_process_id'], ['bpmn_process.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_task_guid'), 'task', ['guid'], unique=True) + op.create_index(op.f('ix_task_json_data_hash'), 'task', ['json_data_hash'], unique=False) + op.create_table('task_definition', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('bpmn_process_definition_id', sa.Integer(), nullable=False), + sa.Column('bpmn_identifier', sa.String(length=255), nullable=False), + sa.Column('properties_json', sa.JSON(), nullable=False), + sa.Column('typename', sa.String(length=255), nullable=False), + sa.ForeignKeyConstraint(['bpmn_process_definition_id'], ['bpmn_process_definition.id'], ), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('bpmn_process_definition_id', 'bpmn_identifier', name='task_definition_unique') + ) + op.create_index(op.f('ix_task_definition_bpmn_identifier'), 'task_definition', ['bpmn_identifier'], unique=False) op.create_table('user_group_assignment', sa.Column('id', sa.Integer(), nullable=False), sa.Column('user_id', sa.Integer(), nullable=False), @@ -228,6 +296,21 @@ def upgrade(): sa.PrimaryKeyConstraint('id'), sa.UniqueConstraint('principal_id', 'permission_target_id', 'permission', name='permission_assignment_uniq') ) + op.create_table('process_instance_file_data', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('process_instance_id', sa.Integer(), nullable=False), + sa.Column('identifier', sa.String(length=255), nullable=False), + sa.Column('list_index', sa.Integer(), nullable=True), + sa.Column('mimetype', sa.String(length=255), nullable=False), + sa.Column('filename', sa.String(length=255), nullable=False), + sa.Column('contents', sa.LargeBinary(), nullable=False), + sa.Column('digest', sa.String(length=64), nullable=False), + sa.Column('updated_at_in_seconds', sa.Integer(), nullable=False), + sa.Column('created_at_in_seconds', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['process_instance_id'], ['process_instance.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_process_instance_file_data_digest'), 'process_instance_file_data', ['digest'], unique=False) op.create_table('process_instance_metadata', sa.Column('id', sa.Integer(), nullable=False), sa.Column('process_instance_id', sa.Integer(), nullable=False), @@ -248,6 +331,7 @@ def upgrade(): sa.Column('task_id', sa.String(length=50), nullable=False), sa.Column('task_state', sa.String(length=50), nullable=False), sa.Column('bpmn_task_identifier', sa.String(length=255), nullable=False), + sa.Column('delta_json', sa.JSON(), nullable=True), sa.Column('start_in_seconds', sa.DECIMAL(precision=17, scale=6), nullable=False), sa.Column('end_in_seconds', sa.DECIMAL(precision=17, scale=6), nullable=True), sa.ForeignKeyConstraint(['process_instance_id'], ['process_instance.id'], ), @@ -290,12 +374,19 @@ def downgrade(): op.drop_table('spiff_step_details') op.drop_index(op.f('ix_process_instance_metadata_key'), table_name='process_instance_metadata') op.drop_table('process_instance_metadata') + op.drop_index(op.f('ix_process_instance_file_data_digest'), table_name='process_instance_file_data') + op.drop_table('process_instance_file_data') op.drop_table('permission_assignment') op.drop_table('message_instance') op.drop_index(op.f('ix_human_task_completed'), table_name='human_task') op.drop_table('human_task') op.drop_table('user_group_assignment_waiting') op.drop_table('user_group_assignment') + op.drop_index(op.f('ix_task_definition_bpmn_identifier'), table_name='task_definition') + op.drop_table('task_definition') + op.drop_index(op.f('ix_task_json_data_hash'), table_name='task') + op.drop_index(op.f('ix_task_guid'), table_name='task') + op.drop_table('task') op.drop_table('secret') op.drop_table('refresh_token') op.drop_index(op.f('ix_process_instance_report_identifier'), table_name='process_instance_report') @@ -305,6 +396,7 @@ def downgrade(): op.drop_index(op.f('ix_process_instance_process_model_display_name'), table_name='process_instance') op.drop_table('process_instance') op.drop_table('principal') + op.drop_table('bpmn_process_definition_relationship') op.drop_table('user') op.drop_table('spiff_logging') op.drop_index(op.f('ix_spec_reference_cache_type'), table_name='spec_reference_cache') @@ -314,6 +406,14 @@ def downgrade(): op.drop_table('permission_target') op.drop_index(op.f('ix_message_triggerable_process_model_process_model_identifier'), table_name='message_triggerable_process_model') op.drop_table('message_triggerable_process_model') + op.drop_index(op.f('ix_json_data_hash'), table_name='json_data') + op.drop_table('json_data') op.drop_table('group') op.drop_table('correlation_property_cache') + op.drop_index(op.f('ix_bpmn_process_definition_hash'), table_name='bpmn_process_definition') + op.drop_index(op.f('ix_bpmn_process_definition_bpmn_identifier'), table_name='bpmn_process_definition') + op.drop_table('bpmn_process_definition') + op.drop_index(op.f('ix_bpmn_process_json_data_hash'), table_name='bpmn_process') + op.drop_index(op.f('ix_bpmn_process_guid'), table_name='bpmn_process') + op.drop_table('bpmn_process') # ### end Alembic commands ### diff --git a/spiffworkflow-backend/migrations/versions/61cd3e8462f5_.py b/spiffworkflow-backend/migrations/versions/61cd3e8462f5_.py deleted file mode 100644 index c6a5f000e..000000000 --- a/spiffworkflow-backend/migrations/versions/61cd3e8462f5_.py +++ /dev/null @@ -1,30 +0,0 @@ -"""empty message - -Revision ID: 61cd3e8462f5 -Revises: 941f7b76d278 -Create Date: 2023-03-03 16:22:12.449757 - -""" -from alembic import op -import sqlalchemy as sa -from sqlalchemy.dialects import mysql - -# revision identifiers, used by Alembic. -revision = '61cd3e8462f5' -down_revision = '941f7b76d278' -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.drop_constraint('task_ibfk_2', 'task', type_='foreignkey') - op.drop_column('task', 'task_definition_id') - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.add_column('task', sa.Column('task_definition_id', mysql.INTEGER(), autoincrement=False, nullable=False)) - op.create_foreign_key('task_ibfk_2', 'task', 'task_definition', ['task_definition_id'], ['id']) - # ### end Alembic commands ### diff --git a/spiffworkflow-backend/migrations/versions/7422be14adc4_.py b/spiffworkflow-backend/migrations/versions/7422be14adc4_.py deleted file mode 100644 index 6897e0f21..000000000 --- a/spiffworkflow-backend/migrations/versions/7422be14adc4_.py +++ /dev/null @@ -1,51 +0,0 @@ -"""empty message - -Revision ID: 7422be14adc4 -Revises: ac6b60d7fee9 -Create Date: 2023-03-01 15:39:25.731722 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = '7422be14adc4' -down_revision = 'ac6b60d7fee9' -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.create_table('process_instance_data', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('runtime_json', sa.JSON(), nullable=False), - sa.PrimaryKeyConstraint('id') - ) - op.create_table('serialized_bpmn_definition', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('hash', sa.String(length=255), nullable=False), - sa.Column('static_json', sa.JSON(), nullable=False), - sa.PrimaryKeyConstraint('id') - ) - op.create_index(op.f('ix_serialized_bpmn_definition_hash'), 'serialized_bpmn_definition', ['hash'], unique=True) - op.add_column('process_instance', sa.Column('serialized_bpmn_definition_id', sa.Integer(), nullable=True)) - op.add_column('process_instance', sa.Column('process_instance_data_id', sa.Integer(), nullable=True)) - op.create_foreign_key(None, 'process_instance', 'process_instance_data', ['process_instance_data_id'], ['id']) - op.create_foreign_key(None, 'process_instance', 'serialized_bpmn_definition', ['serialized_bpmn_definition_id'], ['id']) - op.add_column('spiff_step_details', sa.Column('delta_json', sa.JSON(), nullable=True)) - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.drop_column('spiff_step_details', 'delta_json') - op.drop_constraint('process_instance_ibfk_3', 'process_instance', type_='foreignkey') - op.drop_constraint('process_instance_ibfk_2', 'process_instance', type_='foreignkey') - op.drop_column('process_instance', 'process_instance_data_id') - op.drop_column('process_instance', 'serialized_bpmn_definition_id') - op.drop_index(op.f('ix_serialized_bpmn_definition_hash'), table_name='serialized_bpmn_definition') - op.drop_table('serialized_bpmn_definition') - op.drop_table('process_instance_data') - # ### end Alembic commands ### diff --git a/spiffworkflow-backend/migrations/versions/8930711a75a4_.py b/spiffworkflow-backend/migrations/versions/8930711a75a4_.py deleted file mode 100644 index 9947a6644..000000000 --- a/spiffworkflow-backend/migrations/versions/8930711a75a4_.py +++ /dev/null @@ -1,28 +0,0 @@ -"""empty message - -Revision ID: 8930711a75a4 -Revises: 7422be14adc4 -Create Date: 2023-03-01 16:16:50.446929 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = '8930711a75a4' -down_revision = '7422be14adc4' -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.add_column('spiff_logging', sa.Column('bpmn_process_name', sa.String(length=255), nullable=True)) - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.drop_column('spiff_logging', 'bpmn_process_name') - # ### end Alembic commands ### diff --git a/spiffworkflow-backend/migrations/versions/941f7b76d278_.py b/spiffworkflow-backend/migrations/versions/941f7b76d278_.py deleted file mode 100644 index 8705894a8..000000000 --- a/spiffworkflow-backend/migrations/versions/941f7b76d278_.py +++ /dev/null @@ -1,30 +0,0 @@ -"""empty message - -Revision ID: 941f7b76d278 -Revises: b0b82fab4cf9 -Create Date: 2023-03-03 14:40:46.574985 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = '941f7b76d278' -down_revision = 'b0b82fab4cf9' -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.add_column('bpmn_process', sa.Column('guid', sa.String(length=36), nullable=True)) - op.create_index(op.f('ix_bpmn_process_guid'), 'bpmn_process', ['guid'], unique=True) - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.drop_index(op.f('ix_bpmn_process_guid'), table_name='bpmn_process') - op.drop_column('bpmn_process', 'guid') - # ### end Alembic commands ### diff --git a/spiffworkflow-backend/migrations/versions/a63a61a21398_.py b/spiffworkflow-backend/migrations/versions/a63a61a21398_.py deleted file mode 100644 index 2a4524b15..000000000 --- a/spiffworkflow-backend/migrations/versions/a63a61a21398_.py +++ /dev/null @@ -1,107 +0,0 @@ -"""empty message - -Revision ID: a63a61a21398 -Revises: 1e8167de6df8 -Create Date: 2023-03-03 13:07:51.665585 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = 'a63a61a21398' -down_revision = '1e8167de6df8' -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.create_table('bpmn_process', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('parent_process_id', sa.Integer(), nullable=True), - sa.Column('properties_json', sa.JSON(), nullable=False), - sa.Column('json_data_hash', sa.String(length=255), nullable=False), - sa.ForeignKeyConstraint(['parent_process_id'], ['bpmn_process.id'], ), - sa.PrimaryKeyConstraint('id') - ) - op.create_index(op.f('ix_bpmn_process_json_data_hash'), 'bpmn_process', ['json_data_hash'], unique=False) - op.create_table('bpmn_process_definition', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('hash', sa.String(length=255), nullable=False), - sa.Column('bpmn_identifier', sa.String(length=255), nullable=False), - sa.Column('properties_json', sa.JSON(), nullable=False), - sa.Column('type', sa.String(length=32), nullable=True), - sa.Column('bpmn_version_control_type', sa.String(length=50), nullable=True), - sa.Column('bpmn_version_control_identifier', sa.String(length=255), nullable=True), - sa.PrimaryKeyConstraint('id') - ) - op.create_index(op.f('ix_bpmn_process_definition_bpmn_identifier'), 'bpmn_process_definition', ['bpmn_identifier'], unique=False) - op.create_index(op.f('ix_bpmn_process_definition_hash'), 'bpmn_process_definition', ['hash'], unique=True) - op.create_table('json_data', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('hash', sa.String(length=255), nullable=False), - sa.Column('data', sa.JSON(), nullable=False), - sa.PrimaryKeyConstraint('id') - ) - op.create_index(op.f('ix_json_data_hash'), 'json_data', ['hash'], unique=True) - op.create_table('bpmn_process_definition_relationship', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('bpmn_process_definition_parent_id', sa.Integer(), nullable=False), - sa.Column('bpmn_process_definition_child_id', sa.Integer(), nullable=False), - sa.ForeignKeyConstraint(['bpmn_process_definition_child_id'], ['bpmn_process_definition.id'], ), - sa.ForeignKeyConstraint(['bpmn_process_definition_parent_id'], ['bpmn_process_definition.id'], ), - sa.PrimaryKeyConstraint('id'), - sa.UniqueConstraint('bpmn_process_definition_parent_id', 'bpmn_process_definition_child_id', name='bpmn_process_definition_relationship_unique') - ) - op.create_table('task_definition', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('bpmn_process_definition_id', sa.Integer(), nullable=False), - sa.Column('bpmn_identifier', sa.String(length=255), nullable=False), - sa.Column('properties_json', sa.JSON(), nullable=False), - sa.Column('typename', sa.String(length=255), nullable=False), - sa.ForeignKeyConstraint(['bpmn_process_definition_id'], ['bpmn_process_definition.id'], ), - sa.PrimaryKeyConstraint('id'), - sa.UniqueConstraint('bpmn_process_definition_id', 'bpmn_identifier', name='task_definition_unique') - ) - op.create_index(op.f('ix_task_definition_bpmn_identifier'), 'task_definition', ['bpmn_identifier'], unique=False) - op.create_table('task', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('guid', sa.String(length=36), nullable=False), - sa.Column('bpmn_process_id', sa.Integer(), nullable=False), - sa.Column('task_definition_id', sa.Integer(), nullable=False), - sa.Column('state', sa.String(length=10), nullable=False), - sa.Column('properties_json', sa.JSON(), nullable=False), - sa.Column('json_data_hash', sa.String(length=255), nullable=False), - sa.ForeignKeyConstraint(['bpmn_process_id'], ['bpmn_process.id'], ), - sa.ForeignKeyConstraint(['task_definition_id'], ['task_definition.id'], ), - sa.PrimaryKeyConstraint('id') - ) - op.create_index(op.f('ix_task_guid'), 'task', ['guid'], unique=True) - op.create_index(op.f('ix_task_json_data_hash'), 'task', ['json_data_hash'], unique=False) - op.add_column('process_instance', sa.Column('bpmn_process_definition_id', sa.Integer(), nullable=True)) - op.add_column('process_instance', sa.Column('spiff_serializer_version', sa.String(length=50), nullable=True)) - op.create_foreign_key(None, 'process_instance', 'bpmn_process_definition', ['bpmn_process_definition_id'], ['id']) - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.drop_constraint(None, 'process_instance', type_='foreignkey') - op.drop_column('process_instance', 'spiff_serializer_version') - op.drop_column('process_instance', 'bpmn_process_definition_id') - op.drop_index(op.f('ix_task_json_data_hash'), table_name='task') - op.drop_index(op.f('ix_task_guid'), table_name='task') - op.drop_table('task') - op.drop_index(op.f('ix_task_definition_bpmn_identifier'), table_name='task_definition') - op.drop_table('task_definition') - op.drop_table('bpmn_process_definition_relationship') - op.drop_index(op.f('ix_json_data_hash'), table_name='json_data') - op.drop_table('json_data') - op.drop_index(op.f('ix_bpmn_process_definition_hash'), table_name='bpmn_process_definition') - op.drop_index(op.f('ix_bpmn_process_definition_bpmn_identifier'), table_name='bpmn_process_definition') - op.drop_table('bpmn_process_definition') - op.drop_index(op.f('ix_bpmn_process_json_data_hash'), table_name='bpmn_process') - op.drop_table('bpmn_process') - # ### end Alembic commands ### diff --git a/spiffworkflow-backend/migrations/versions/b0b82fab4cf9_.py b/spiffworkflow-backend/migrations/versions/b0b82fab4cf9_.py deleted file mode 100644 index 61d199450..000000000 --- a/spiffworkflow-backend/migrations/versions/b0b82fab4cf9_.py +++ /dev/null @@ -1,30 +0,0 @@ -"""empty message - -Revision ID: b0b82fab4cf9 -Revises: a63a61a21398 -Create Date: 2023-03-03 13:24:08.304492 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = 'b0b82fab4cf9' -down_revision = 'a63a61a21398' -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.add_column('process_instance', sa.Column('bpmn_process_id', sa.Integer(), nullable=True)) - op.create_foreign_key(None, 'process_instance', 'bpmn_process', ['bpmn_process_id'], ['id']) - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.drop_constraint(None, 'process_instance', type_='foreignkey') - op.drop_column('process_instance', 'bpmn_process_id') - # ### end Alembic commands ### diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/load_database_models.py b/spiffworkflow-backend/src/spiffworkflow_backend/load_database_models.py index d51abb494..376083cf2 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/load_database_models.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/load_database_models.py @@ -50,12 +50,6 @@ from spiffworkflow_backend.models.group import GroupModel # noqa: F401 from spiffworkflow_backend.models.process_instance_metadata import ( ProcessInstanceMetadataModel, ) # noqa: F401 -from spiffworkflow_backend.models.serialized_bpmn_definition import ( - SerializedBpmnDefinitionModel, -) # noqa: F401 -from spiffworkflow_backend.models.process_instance_data import ( - ProcessInstanceDataModel, -) # noqa: F401 from spiffworkflow_backend.models.process_instance_file_data import ( ProcessInstanceFileDataModel, ) # noqa: F401 diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/process_instance.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/process_instance.py index 5b3733710..ea7f45623 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/process_instance.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/process_instance.py @@ -22,10 +22,6 @@ from spiffworkflow_backend.models.bpmn_process_definition import ( ) from spiffworkflow_backend.models.db import db from spiffworkflow_backend.models.db import SpiffworkflowBaseDBModel -from spiffworkflow_backend.models.process_instance_data import ProcessInstanceDataModel -from spiffworkflow_backend.models.serialized_bpmn_definition import ( - SerializedBpmnDefinitionModel, -) # noqa: F401 from spiffworkflow_backend.models.task import Task from spiffworkflow_backend.models.task import TaskSchema from spiffworkflow_backend.models.user import UserModel @@ -69,16 +65,6 @@ class ProcessInstanceModel(SpiffworkflowBaseDBModel): process_initiator_id: int = db.Column(ForeignKey(UserModel.id), nullable=False) # type: ignore process_initiator = relationship("UserModel") - serialized_bpmn_definition_id: int | None = db.Column( - ForeignKey(SerializedBpmnDefinitionModel.id), nullable=True # type: ignore - ) - serialized_bpmn_definition = relationship("SerializedBpmnDefinitionModel") - - process_instance_data_id: int | None = db.Column( - ForeignKey(ProcessInstanceDataModel.id), nullable=True # type: ignore - ) - process_instance_data = relationship("ProcessInstanceDataModel", cascade="delete") - bpmn_process_definition_id: Optional[int] = db.Column( ForeignKey(BpmnProcessDefinitionModel.id), nullable=True # type: ignore ) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/process_instance_data.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/process_instance_data.py deleted file mode 100644 index 01f07e33c..000000000 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/process_instance_data.py +++ /dev/null @@ -1,22 +0,0 @@ -"""Process_instance.""" -from __future__ import annotations - -from spiffworkflow_backend.models.db import db -from spiffworkflow_backend.models.db import SpiffworkflowBaseDBModel - - -# the last three here should maybe become columns on process instance someday -# runtime_json -# "bpmn_messages", -# "correlations", -# "data", -# "subprocesses", -# "tasks", -# "last_task", # guid generated by spiff -# "root", # guid generated by spiff -# "success", # boolean -class ProcessInstanceDataModel(SpiffworkflowBaseDBModel): - __tablename__ = "process_instance_data" - id: int = db.Column(db.Integer, primary_key=True) - # this is not deferred because there is no reason to query this model if you do not want the runtime_json - runtime_json: str = db.Column(db.JSON, nullable=False) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/serialized_bpmn_definition.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/serialized_bpmn_definition.py deleted file mode 100644 index 5ccb71466..000000000 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/serialized_bpmn_definition.py +++ /dev/null @@ -1,49 +0,0 @@ -"""Process_instance.""" -from __future__ import annotations - -from spiffworkflow_backend.models.db import db -from spiffworkflow_backend.models.db import SpiffworkflowBaseDBModel - - -# top level serialized keys -# -# static -# "subprocess_specs", -# "spec", -# "serializer_version", -# -# runtime -# "bpmn_messages", -# "correlations", -# "subprocesses", -# -# also in subprocesses -# "data", -# "tasks" -# "last_task", # guid generated by spiff -# "root", # guid generated by spiff -# "success", # boolean -# -# new columns on process_instance -# -# delta algorithm: -# a = {"hey": { "hey2": 2, "hey3": 3, "hey6": 7 }, "hey30": 3, "hey40": 4} -# b = {"hey": { "hey2": 4, "hey5": 3 }, "hey20": 2, "hey30": 3} -# a_keys = set(a.keys()) -# b_keys = set(b.keys()) -# removed = a_keys - b_keys -# added_keys = b_keys - a_keys -# keys_present_in_both = a_keys & b_keys -# changed = {} -# for key_in_both in keys_present_in_both: -# if a[key_in_both] != b[key_in_both]: -# changed[key_in_both] = b[key_in_both] -# added = {} -# for added_key in added_keys: -# added[added_key] = b[added_key] -# final_tuple = [added, removed, changed] -class SerializedBpmnDefinitionModel(SpiffworkflowBaseDBModel): - __tablename__ = "serialized_bpmn_definition" - id: int = db.Column(db.Integer, primary_key=True) - hash: str = db.Column(db.String(255), nullable=False, index=True, unique=True) - static_json: str = db.Column(db.JSON, nullable=False) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py b/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py index 6b41b44a7..f58cf3034 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py @@ -77,7 +77,6 @@ from spiffworkflow_backend.models.message_instance_correlation import ( ) from spiffworkflow_backend.models.process_instance import ProcessInstanceModel from spiffworkflow_backend.models.process_instance import ProcessInstanceStatus -from spiffworkflow_backend.models.process_instance_data import ProcessInstanceDataModel from spiffworkflow_backend.models.process_instance_metadata import ( ProcessInstanceMetadataModel, ) @@ -85,9 +84,6 @@ from spiffworkflow_backend.models.process_model import ProcessModelInfo from spiffworkflow_backend.models.script_attributes_context import ( ScriptAttributesContext, ) -from spiffworkflow_backend.models.serialized_bpmn_definition import ( - SerializedBpmnDefinitionModel, -) # noqa: F401 from spiffworkflow_backend.models.spec_reference import SpecReferenceCache from spiffworkflow_backend.models.spiff_step_details import SpiffStepDetailsModel from spiffworkflow_backend.models.task_definition import TaskDefinitionModel @@ -928,53 +924,6 @@ class ProcessInstanceProcessor: ) return subprocesses_by_child_task_ids - def _add_bpmn_json_records(self) -> None: - """Adds serialized_bpmn_definition and process_instance_data records to the db session. - - Expects the save method to commit it. - """ - bpmn_dict = json.loads(self.serialize()) - bpmn_dict_keys = ("spec", "subprocess_specs", "serializer_version") - bpmn_spec_dict = {} - process_instance_data_dict = {} - for bpmn_key in bpmn_dict.keys(): - if bpmn_key in bpmn_dict_keys: - bpmn_spec_dict[bpmn_key] = bpmn_dict[bpmn_key] - else: - process_instance_data_dict[bpmn_key] = bpmn_dict[bpmn_key] - - # FIXME: always save new hash until we get updated Spiff without loopresettask - # if self.process_instance_model.serialized_bpmn_definition_id is None: - new_hash_digest = sha256( - json.dumps(bpmn_spec_dict, sort_keys=True).encode("utf8") - ).hexdigest() - serialized_bpmn_definition = SerializedBpmnDefinitionModel.query.filter_by( - hash=new_hash_digest - ).first() - if serialized_bpmn_definition is None: - serialized_bpmn_definition = SerializedBpmnDefinitionModel( - hash=new_hash_digest, static_json=json.dumps(bpmn_spec_dict) - ) - db.session.add(serialized_bpmn_definition) - if ( - self.process_instance_model.serialized_bpmn_definition_id is None - or self.process_instance_model.serialized_bpmn_definition.hash - != new_hash_digest - ): - self.process_instance_model.serialized_bpmn_definition = ( - serialized_bpmn_definition - ) - - process_instance_data = None - if self.process_instance_model.process_instance_data_id is None: - process_instance_data = ProcessInstanceDataModel() - else: - process_instance_data = self.process_instance_model.process_instance_data - - process_instance_data.runtime_json = json.dumps(process_instance_data_dict) - db.session.add(process_instance_data) - self.process_instance_model.process_instance_data = process_instance_data - def _store_bpmn_process_definition( self, process_bpmn_properties: dict, @@ -1090,7 +1039,7 @@ class ProcessInstanceProcessor: return bpmn_process - def _add_bpmn_json_records_new(self) -> None: + def _add_bpmn_json_records(self) -> None: """Adds serialized_bpmn_definition and process_instance_data records to the db session. Expects the save method to commit it. @@ -1117,8 +1066,7 @@ class ProcessInstanceProcessor: def save(self) -> None: """Saves the current state of this processor to the database.""" - # self._add_bpmn_json_records() - self._add_bpmn_json_records_new() + self._add_bpmn_json_records() self.process_instance_model.spiff_serializer_version = self.SERIALIZER_VERSION complete_states = [TaskState.CANCELLED, TaskState.COMPLETED] From 4ee7b299f2dd62acedb8c2be5dd5438cb03e4e34 Mon Sep 17 00:00:00 2001 From: jasquat Date: Mon, 6 Mar 2023 11:59:33 -0500 Subject: [PATCH 14/25] pyl and fixed cypress tests w/ burnettk --- spiffworkflow-backend/conftest.py | 2 +- spiffworkflow-backend/migrations/env.py | 2 - .../models/bpmn_process.py | 7 +- .../models/process_instance.py | 7 +- .../routes/process_instances_controller.py | 4 +- .../services/process_instance_processor.py | 97 +++++++++++++------ .../cypress/e2e/process_instances.cy.js | 5 +- .../cypress/pilot/pp1.cy.js | 2 +- .../cypress/support/commands.js | 8 +- 9 files changed, 86 insertions(+), 48 deletions(-) diff --git a/spiffworkflow-backend/conftest.py b/spiffworkflow-backend/conftest.py index 3e4804a8b..3adf8725e 100644 --- a/spiffworkflow-backend/conftest.py +++ b/spiffworkflow-backend/conftest.py @@ -5,9 +5,9 @@ import shutil import pytest from flask.app import Flask from flask.testing import FlaskClient -from spiffworkflow_backend.models.bpmn_process import BpmnProcessModel from tests.spiffworkflow_backend.helpers.base_test import BaseTest +from spiffworkflow_backend.models.bpmn_process import BpmnProcessModel from spiffworkflow_backend.models.db import db from spiffworkflow_backend.models.process_instance import ProcessInstanceModel from spiffworkflow_backend.models.user import UserModel diff --git a/spiffworkflow-backend/migrations/env.py b/spiffworkflow-backend/migrations/env.py index 68feded2a..630e381ad 100644 --- a/spiffworkflow-backend/migrations/env.py +++ b/spiffworkflow-backend/migrations/env.py @@ -1,5 +1,3 @@ -from __future__ import with_statement - import logging from logging.config import fileConfig diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process.py index b5a3137e6..e1c146d4f 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process.py @@ -1,5 +1,4 @@ from __future__ import annotations -from typing import Optional from sqlalchemy import ForeignKey @@ -16,9 +15,11 @@ from spiffworkflow_backend.models.db import SpiffworkflowBaseDBModel class BpmnProcessModel(SpiffworkflowBaseDBModel): __tablename__ = "bpmn_process" id: int = db.Column(db.Integer, primary_key=True) - guid: Optional[str] = db.Column(db.String(36), nullable=True, unique=True, index=True) + guid: str | None = db.Column(db.String(36), nullable=True, unique=True, index=True) - parent_process_id: Optional[int] = db.Column(ForeignKey("bpmn_process.id"), nullable=True) + parent_process_id: int | None = db.Column( + ForeignKey("bpmn_process.id"), nullable=True + ) properties_json: dict = db.Column(db.JSON, nullable=False) json_data_hash: str = db.Column(db.String(255), nullable=False, index=True) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/process_instance.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/process_instance.py index ea7f45623..cbbceaba9 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/process_instance.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/process_instance.py @@ -1,7 +1,5 @@ """Process_instance.""" from __future__ import annotations -from typing import Optional -from spiffworkflow_backend.models.bpmn_process import BpmnProcessModel from typing import Any from typing import cast @@ -17,6 +15,7 @@ from sqlalchemy.orm import relationship from sqlalchemy.orm import validates from spiffworkflow_backend.helpers.spiff_enum import SpiffEnum +from spiffworkflow_backend.models.bpmn_process import BpmnProcessModel from spiffworkflow_backend.models.bpmn_process_definition import ( BpmnProcessDefinitionModel, ) @@ -65,11 +64,11 @@ class ProcessInstanceModel(SpiffworkflowBaseDBModel): process_initiator_id: int = db.Column(ForeignKey(UserModel.id), nullable=False) # type: ignore process_initiator = relationship("UserModel") - bpmn_process_definition_id: Optional[int] = db.Column( + bpmn_process_definition_id: int | None = db.Column( ForeignKey(BpmnProcessDefinitionModel.id), nullable=True # type: ignore ) bpmn_process_definition = relationship(BpmnProcessDefinitionModel) - bpmn_process_id: Optional[int] = db.Column( + bpmn_process_id: int | None = db.Column( ForeignKey(BpmnProcessModel.id), nullable=True # type: ignore ) bpmn_process = relationship(BpmnProcessModel) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/routes/process_instances_controller.py b/spiffworkflow-backend/src/spiffworkflow_backend/routes/process_instances_controller.py index d33a1c000..fbf33ad38 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/routes/process_instances_controller.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/routes/process_instances_controller.py @@ -586,7 +586,9 @@ def process_instance_task_list( spiff_task_id, TaskState.FUTURE ) - bpmn_process_instance = ProcessInstanceProcessor._serializer.workflow_from_dict(full_bpmn_process_dict) + bpmn_process_instance = ProcessInstanceProcessor._serializer.workflow_from_dict( + full_bpmn_process_dict + ) spiff_task = processor.__class__.get_task_by_bpmn_identifier( step_details[-1].bpmn_task_identifier, bpmn_process_instance diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py b/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py index 24e6a3187..e90acd77a 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py @@ -1,7 +1,5 @@ """Process_instance_processor.""" import _strptime # type: ignore -from SpiffWorkflow.task import TaskStateNames # type: ignore -from spiffworkflow_backend.models.task import TaskModel # noqa: F401 import decimal import json import logging @@ -53,6 +51,7 @@ from SpiffWorkflow.serializer.exceptions import MissingSpecError # type: ignore from SpiffWorkflow.spiff.serializer.config import SPIFF_SPEC_CONFIG # type: ignore from SpiffWorkflow.task import Task as SpiffTask # type: ignore from SpiffWorkflow.task import TaskState +from SpiffWorkflow.task import TaskStateNames from SpiffWorkflow.util.deep_merge import DeepMerge # type: ignore from sqlalchemy import text @@ -86,6 +85,7 @@ from spiffworkflow_backend.models.script_attributes_context import ( ) from spiffworkflow_backend.models.spec_reference import SpecReferenceCache from spiffworkflow_backend.models.spiff_step_details import SpiffStepDetailsModel +from spiffworkflow_backend.models.task import TaskModel # noqa: F401 from spiffworkflow_backend.models.task_definition import TaskDefinitionModel from spiffworkflow_backend.models.user import UserModel from spiffworkflow_backend.scripts.script import Script @@ -465,11 +465,13 @@ class ProcessInstanceProcessor: ) try: - (self.bpmn_process_instance, self.full_bpmn_process_dict) = self.__get_bpmn_process_instance( - process_instance_model, - bpmn_process_spec, - validate_only, - subprocesses=subprocesses, + (self.bpmn_process_instance, self.full_bpmn_process_dict) = ( + self.__get_bpmn_process_instance( + process_instance_model, + bpmn_process_spec, + validate_only, + subprocesses=subprocesses, + ) ) self.set_script_engine(self.bpmn_process_instance) @@ -545,7 +547,9 @@ class ProcessInstanceProcessor: @classmethod def _set_definition_dict_for_bpmn_subprocess_definitions( - cls, bpmn_process_definition: BpmnProcessDefinitionModel, spiff_bpmn_process_dict: dict + cls, + bpmn_process_definition: BpmnProcessDefinitionModel, + spiff_bpmn_process_dict: dict, ) -> None: bpmn_process_subprocess_definitions = ( BpmnProcessDefinitionRelationshipModel.query.filter_by( @@ -562,19 +566,23 @@ class ProcessInstanceProcessor: @classmethod def _get_bpmn_process_dict(cls, bpmn_process: BpmnProcessModel) -> dict: - json_data = JsonDataModel.query.filter_by(hash=bpmn_process.json_data_hash).first() - bpmn_process_dict = {'data': json_data.data, 'tasks': {}} + json_data = JsonDataModel.query.filter_by( + hash=bpmn_process.json_data_hash + ).first() + bpmn_process_dict = {"data": json_data.data, "tasks": {}} bpmn_process_dict.update(bpmn_process.properties_json) tasks = TaskModel.query.filter_by(bpmn_process_id=bpmn_process.id).all() for task in tasks: json_data = JsonDataModel.query.filter_by(hash=task.json_data_hash).first() - bpmn_process_dict['tasks'][task.guid] = task.properties_json - bpmn_process_dict['tasks'][task.guid]['data'] = json_data.data + bpmn_process_dict["tasks"][task.guid] = task.properties_json + bpmn_process_dict["tasks"][task.guid]["data"] = json_data.data return bpmn_process_dict @classmethod - def _get_full_bpmn_process_dict(cls, process_instance_model: ProcessInstanceModel) -> dict: + def _get_full_bpmn_process_dict( + cls, process_instance_model: ProcessInstanceModel + ) -> dict: if process_instance_model.bpmn_process_definition_id is None: return {} @@ -582,7 +590,7 @@ class ProcessInstanceProcessor: "serializer_version": process_instance_model.spiff_serializer_version, "spec": {}, "subprocess_specs": {}, - "subprocesses": {} + "subprocesses": {}, } bpmn_process_definition = process_instance_model.bpmn_process_definition if bpmn_process_definition is not None: @@ -591,17 +599,23 @@ class ProcessInstanceProcessor: bpmn_process_definition ) ) - cls._set_definition_dict_for_bpmn_subprocess_definitions(bpmn_process_definition, spiff_bpmn_process_dict) + cls._set_definition_dict_for_bpmn_subprocess_definitions( + bpmn_process_definition, spiff_bpmn_process_dict + ) bpmn_process = process_instance_model.bpmn_process if bpmn_process is not None: bpmn_process_dict = cls._get_bpmn_process_dict(bpmn_process) spiff_bpmn_process_dict.update(bpmn_process_dict) - bpmn_subprocesses = BpmnProcessModel.query.filter_by(parent_process_id=bpmn_process.id).all() + bpmn_subprocesses = BpmnProcessModel.query.filter_by( + parent_process_id=bpmn_process.id + ).all() for bpmn_subprocess in bpmn_subprocesses: bpmn_process_dict = cls._get_bpmn_process_dict(bpmn_subprocess) - spiff_bpmn_process_dict['subprocesses'][bpmn_subprocess.guid] = bpmn_process_dict + spiff_bpmn_process_dict["subprocesses"][ + bpmn_subprocess.guid + ] = bpmn_process_dict return spiff_bpmn_process_dict @@ -647,10 +661,16 @@ class ProcessInstanceProcessor: spiff_logger.setLevel(logging.WARNING) try: - full_bpmn_process_dict = ProcessInstanceProcessor._get_full_bpmn_process_dict( - process_instance_model + full_bpmn_process_dict = ( + ProcessInstanceProcessor._get_full_bpmn_process_dict( + process_instance_model + ) + ) + bpmn_process_instance = ( + ProcessInstanceProcessor._serializer.workflow_from_dict( + full_bpmn_process_dict + ) ) - bpmn_process_instance = ProcessInstanceProcessor._serializer.workflow_from_dict(full_bpmn_process_dict) except Exception as err: raise err finally: @@ -981,14 +1001,20 @@ class ProcessInstanceProcessor: bpmn_process_definition_parent ) - def _add_bpmn_process(self, bpmn_process_dict: dict, bpmn_process_parent: Optional[BpmnProcessModel] = None, bpmn_process_guid: Optional[str] = None) -> BpmnProcessModel: + def _add_bpmn_process( + self, + bpmn_process_dict: dict, + bpmn_process_parent: Optional[BpmnProcessModel] = None, + bpmn_process_guid: Optional[str] = None, + ) -> BpmnProcessModel: tasks = bpmn_process_dict.pop("tasks") bpmn_process_data = bpmn_process_dict.pop("data") bpmn_process = None if bpmn_process_parent is not None: bpmn_process = BpmnProcessModel.query.filter_by( - parent_process_id=bpmn_process_parent.id, guid=bpmn_process_guid).first() + parent_process_id=bpmn_process_parent.id, guid=bpmn_process_guid + ).first() elif self.process_instance_model.bpmn_process_id is not None: bpmn_process = self.process_instance_model.bpmn_process @@ -997,12 +1023,20 @@ class ProcessInstanceProcessor: bpmn_process.properties_json = bpmn_process_dict - bpmn_process_data_json = json.dumps(bpmn_process_data, sort_keys=True).encode("utf8") + bpmn_process_data_json = json.dumps(bpmn_process_data, sort_keys=True).encode( + "utf8" + ) bpmn_process_data_hash = sha256(bpmn_process_data_json).hexdigest() if bpmn_process.json_data_hash != bpmn_process_data_hash: - json_data = db.session.query(JsonDataModel.id).filter_by(hash=bpmn_process_data_hash).first() + json_data = ( + db.session.query(JsonDataModel.id) + .filter_by(hash=bpmn_process_data_hash) + .first() + ) if json_data is None: - json_data = JsonDataModel(hash=bpmn_process_data_hash, data=bpmn_process_data) + json_data = JsonDataModel( + hash=bpmn_process_data_hash, data=bpmn_process_data + ) db.session.add(json_data) bpmn_process.json_data_hash = bpmn_process_data_hash @@ -1013,15 +1047,16 @@ class ProcessInstanceProcessor: db.session.add(bpmn_process) for task_id, task_properties in tasks.items(): - task_data_dict = task_properties.pop('data') - state_int = task_properties['state'] + task_data_dict = task_properties.pop("data") + state_int = task_properties["state"] task = TaskModel.query.filter_by(guid=task_id).first() if task is None: # bpmn_process_identifier = task_properties['workflow_name'] # bpmn_identifier = task_properties['task_spec'] # - # task_definition = TaskDefinitionModel.query.filter_by(bpmn_identifier=bpmn_identifier).join(BpmnProcessDefinitionModel).filter(BpmnProcessDefinitionModel.bpmn_identifier==bpmn_process_identifier).first() + # task_definition = TaskDefinitionModel.query.filter_by(bpmn_identifier=bpmn_identifier) + # .join(BpmnProcessDefinitionModel).filter(BpmnProcessDefinitionModel.bpmn_identifier==bpmn_process_identifier).first() # if task_definition is None: # subprocess_task = TaskModel.query.filter_by(guid=bpmn_process.guid) task = TaskModel(guid=task_id, bpmn_process_id=bpmn_process.id) @@ -1031,7 +1066,11 @@ class ProcessInstanceProcessor: task_data_json = json.dumps(task_data_dict, sort_keys=True).encode("utf8") task_data_hash = sha256(task_data_json).hexdigest() if task.json_data_hash != task_data_hash: - json_data = db.session.query(JsonDataModel.id).filter_by(hash=task_data_hash).first() + json_data = ( + db.session.query(JsonDataModel.id) + .filter_by(hash=task_data_hash) + .first() + ) if json_data is None: json_data = JsonDataModel(hash=task_data_hash, data=task_data_dict) db.session.add(json_data) diff --git a/spiffworkflow-frontend/cypress/e2e/process_instances.cy.js b/spiffworkflow-frontend/cypress/e2e/process_instances.cy.js index e582dcbba..846d41627 100644 --- a/spiffworkflow-frontend/cypress/e2e/process_instances.cy.js +++ b/spiffworkflow-frontend/cypress/e2e/process_instances.cy.js @@ -179,13 +179,14 @@ describe('process-instances', () => { cy.get(statusSelect).contains(processStatus).click(); cy.get(statusSelect).click(); cy.getBySel('filter-button').click(); - // FIXME: wait a little bit for the useEffects to be able to fully set processInstanceFilters - cy.wait(1000); cy.url().should('include', `status=${processStatus}`); cy.assertAtLeastOneItemInPaginatedResults(); cy.getBySel(`process-instance-status-${processStatus}`); // there should really only be one, but in CI there are sometimes more cy.get('div[aria-label="Clear all selected items"]:first').click(); + cy.get('div[aria-label="Clear all selected items"]').should( + 'not.exist' + ); } }); diff --git a/spiffworkflow-frontend/cypress/pilot/pp1.cy.js b/spiffworkflow-frontend/cypress/pilot/pp1.cy.js index 47dbbf30a..c34315b83 100644 --- a/spiffworkflow-frontend/cypress/pilot/pp1.cy.js +++ b/spiffworkflow-frontend/cypress/pilot/pp1.cy.js @@ -31,7 +31,7 @@ const approveWithUser = ( }; describe('pp1', () => { - it('can run PP1', () => { + it.skip('can run PP1', () => { cy.login('core5.contributor', 'core5.contributor'); cy.visit('/'); cy.contains('Start New +').click(); diff --git a/spiffworkflow-frontend/cypress/support/commands.js b/spiffworkflow-frontend/cypress/support/commands.js index a878fbf46..5e786a0dc 100644 --- a/spiffworkflow-frontend/cypress/support/commands.js +++ b/spiffworkflow-frontend/cypress/support/commands.js @@ -42,13 +42,11 @@ Cypress.Commands.add('navigateToAdmin', () => { }); Cypress.Commands.add('login', (username, password) => { - // Cypress.Commands.add('login', (selector, ...args) => { cy.visit('/admin'); + console.log('username', username); if (!username) { - const username = - Cypress.env('SPIFFWORKFLOW_FRONTEND_USERNAME') || 'ciadmin1'; - const password = - Cypress.env('SPIFFWORKFLOW_FRONTEND_PASSWORD') || 'ciadmin1'; + username = Cypress.env('SPIFFWORKFLOW_FRONTEND_USERNAME') || 'ciadmin1'; + password = Cypress.env('SPIFFWORKFLOW_FRONTEND_PASSWORD') || 'ciadmin1'; } cy.get('#username').type(username); cy.get('#password').type(password); From 628dc14d7ce95863d93814173ed0b9337fe43787 Mon Sep 17 00:00:00 2001 From: jasquat Date: Mon, 6 Mar 2023 13:03:18 -0500 Subject: [PATCH 15/25] make sure we set the subprocesses correctly in the bpmn dict w/ burnettk --- .../services/process_instance_processor.py | 28 ++++++++++++----- .../unit/test_process_instance_processor.py | 31 +++++++++++++++++++ 2 files changed, 51 insertions(+), 8 deletions(-) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py b/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py index e90acd77a..774feec47 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py @@ -551,18 +551,26 @@ class ProcessInstanceProcessor: bpmn_process_definition: BpmnProcessDefinitionModel, spiff_bpmn_process_dict: dict, ) -> None: + # find all child subprocesses of a process bpmn_process_subprocess_definitions = ( - BpmnProcessDefinitionRelationshipModel.query.filter_by( - bpmn_process_definition_parent_id=bpmn_process_definition.id - ).all() + BpmnProcessDefinitionModel.query.join( + BpmnProcessDefinitionRelationshipModel, + BpmnProcessDefinitionModel.id + == BpmnProcessDefinitionRelationshipModel.bpmn_process_definition_child_id, + ) + .filter_by(bpmn_process_definition_parent_id=bpmn_process_definition.id) + .all() ) for bpmn_subprocess_definition in bpmn_process_subprocess_definitions: - spec = cls._set_definition_dict_for_bpmn_subprocess_definitions( - bpmn_subprocess_definition, spiff_bpmn_process_dict + spec = cls._get_definition_dict_for_bpmn_process_definition( + bpmn_subprocess_definition ) spiff_bpmn_process_dict["subprocess_specs"][ bpmn_subprocess_definition.bpmn_identifier ] = spec + cls._set_definition_dict_for_bpmn_subprocess_definitions( + bpmn_subprocess_definition, spiff_bpmn_process_dict + ) @classmethod def _get_bpmn_process_dict(cls, bpmn_process: BpmnProcessModel) -> dict: @@ -1019,7 +1027,7 @@ class ProcessInstanceProcessor: bpmn_process = self.process_instance_model.bpmn_process if bpmn_process is None: - bpmn_process = BpmnProcessModel() + bpmn_process = BpmnProcessModel(guid=bpmn_process_guid) bpmn_process.properties_json = bpmn_process_dict @@ -1101,8 +1109,12 @@ class ProcessInstanceProcessor: subprocesses = process_instance_data_dict.pop("subprocesses") bpmn_process_parent = self._add_bpmn_process(process_instance_data_dict) - for _subprocess_task_id, subprocess_properties in subprocesses.items(): - self._add_bpmn_process(subprocess_properties, bpmn_process_parent) + for subprocess_task_id, subprocess_properties in subprocesses.items(): + self._add_bpmn_process( + subprocess_properties, + bpmn_process_parent, + bpmn_process_guid=subprocess_task_id, + ) def save(self) -> None: """Saves the current state of this processor to the database.""" diff --git a/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_process_instance_processor.py b/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_process_instance_processor.py index b46ba1e13..75eb31464 100644 --- a/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_process_instance_processor.py +++ b/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_process_instance_processor.py @@ -3,6 +3,7 @@ import pytest from flask import g from flask.app import Flask from flask.testing import FlaskClient +from SpiffWorkflow.task import TaskState # type: ignore from tests.spiffworkflow_backend.helpers.base_test import BaseTest from tests.spiffworkflow_backend.helpers.test_data import load_test_spec @@ -261,6 +262,36 @@ class TestProcessInstanceProcessor(BaseTest): assert process_instance.status == ProcessInstanceStatus.complete.value + def test_can_load_up_processor_after_running_model_with_call_activities( + self, + app: Flask, + client: FlaskClient, + with_db_and_bpmn_file_cleanup: None, + with_super_admin_user: UserModel, + ) -> None: + """Test_does_not_recreate_human_tasks_on_multiple_saves.""" + initiator_user = self.find_or_create_user("initiator_user") + + process_model = load_test_spec( + process_model_id="test_group/call_activity_nested", + process_model_source_directory="call_activity_nested", + ) + process_instance = self.create_process_instance_from_process_model( + process_model=process_model, user=initiator_user + ) + processor = ProcessInstanceProcessor(process_instance) + processor.do_engine_steps(save=True) + + # ensure this does not raise + processor = ProcessInstanceProcessor(process_instance) + + # this task will be found within subprocesses + spiff_task = processor.__class__.get_task_by_bpmn_identifier( + "do_nothing", processor.bpmn_process_instance + ) + assert spiff_task is not None + assert spiff_task.state == TaskState.COMPLETED + def test_does_not_recreate_human_tasks_on_multiple_saves( self, app: Flask, From 24d2dff5e361b4f39a491b3ecd9816407e719c18 Mon Sep 17 00:00:00 2001 From: jasquat Date: Mon, 6 Mar 2023 14:22:59 -0500 Subject: [PATCH 16/25] pin to specific version of poetry --- .github/workflows/backend_tests.yml | 2 +- spiffworkflow-backend/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/backend_tests.yml b/.github/workflows/backend_tests.yml index 71d1ad8be..fff033ed3 100644 --- a/.github/workflows/backend_tests.yml +++ b/.github/workflows/backend_tests.yml @@ -193,7 +193,7 @@ jobs: python-version: "3.11" - name: Install Poetry run: | - pipx install poetry + pipx install --pip-args=--constraint=spiffworkflow-backend/.github/workflows/constraints.txt poetry poetry --version - name: Poetry Install run: poetry install diff --git a/spiffworkflow-backend/Dockerfile b/spiffworkflow-backend/Dockerfile index ef915bde3..9c27426d8 100644 --- a/spiffworkflow-backend/Dockerfile +++ b/spiffworkflow-backend/Dockerfile @@ -19,7 +19,7 @@ RUN apt-get update \ # Setup image for installing Python dependencies. FROM base AS setup -RUN pip install poetry +RUN pip install poetry=1.2.2 RUN useradd _gunicorn --no-create-home --user-group RUN apt-get update \ From af7f58dd1ea70a28dc992da679ae478a6e980aea Mon Sep 17 00:00:00 2001 From: jasquat Date: Mon, 6 Mar 2023 14:29:55 -0500 Subject: [PATCH 17/25] fixed pip install command --- spiffworkflow-backend/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spiffworkflow-backend/Dockerfile b/spiffworkflow-backend/Dockerfile index 9c27426d8..452261717 100644 --- a/spiffworkflow-backend/Dockerfile +++ b/spiffworkflow-backend/Dockerfile @@ -19,7 +19,7 @@ RUN apt-get update \ # Setup image for installing Python dependencies. FROM base AS setup -RUN pip install poetry=1.2.2 +RUN pip install poetry==1.2.2 RUN useradd _gunicorn --no-create-home --user-group RUN apt-get update \ From 9705244f6c103a27807d82139b7806039866e4df Mon Sep 17 00:00:00 2001 From: jasquat Date: Mon, 6 Mar 2023 14:41:03 -0500 Subject: [PATCH 18/25] use poetry 1.3.2 --- spiffworkflow-backend/.github/workflows/constraints.txt | 2 +- spiffworkflow-backend/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/spiffworkflow-backend/.github/workflows/constraints.txt b/spiffworkflow-backend/.github/workflows/constraints.txt index 7ccc87119..55013fd65 100644 --- a/spiffworkflow-backend/.github/workflows/constraints.txt +++ b/spiffworkflow-backend/.github/workflows/constraints.txt @@ -1,5 +1,5 @@ pip==22.2.2 nox==2022.11.21 nox-poetry==1.0.2 -poetry==1.2.2 +poetry==1.3.2 virtualenv==20.16.5 diff --git a/spiffworkflow-backend/Dockerfile b/spiffworkflow-backend/Dockerfile index 452261717..42b0fee2f 100644 --- a/spiffworkflow-backend/Dockerfile +++ b/spiffworkflow-backend/Dockerfile @@ -19,7 +19,7 @@ RUN apt-get update \ # Setup image for installing Python dependencies. FROM base AS setup -RUN pip install poetry==1.2.2 +RUN pip install poetry==1.3.2 RUN useradd _gunicorn --no-create-home --user-group RUN apt-get update \ From 94776a78b5330826db456314fdccea288d6b3fe7 Mon Sep 17 00:00:00 2001 From: jasquat Date: Mon, 6 Mar 2023 16:16:55 -0500 Subject: [PATCH 19/25] some minor tweaks and turn off debug logging for spiff step details w/ burnettk --- spiffworkflow-backend/Dockerfile | 5 +++++ .../services/process_instance_processor.py | 6 +++--- spiffworkflow-frontend/cypress/pilot/pp1.cy.js | 12 ++++++++---- spiffworkflow-frontend/cypress/support/commands.js | 2 +- 4 files changed, 17 insertions(+), 8 deletions(-) diff --git a/spiffworkflow-backend/Dockerfile b/spiffworkflow-backend/Dockerfile index 42b0fee2f..f42b8a5b6 100644 --- a/spiffworkflow-backend/Dockerfile +++ b/spiffworkflow-backend/Dockerfile @@ -19,6 +19,11 @@ RUN apt-get update \ # Setup image for installing Python dependencies. FROM base AS setup +# poetry 1.4 seems to cause an issue where it errors with +# This error originates from the build backend, and is likely not a +# problem with poetry but with lazy-object-proxy (1.7.1) not supporting PEP 517 builds. +# You can verify this by running 'pip wheel --use-pep517 "lazy-object-proxy (==1.7.1) ; python_version >= "3.6""'. +# Pinnning to 1.3.2 to attempt to avoid it. RUN pip install poetry==1.3.2 RUN useradd _gunicorn --no-create-home --user-group diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py b/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py index 774feec47..e73e7ff09 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py @@ -1218,7 +1218,7 @@ class ProcessInstanceProcessor: ) db.session.add(spiff_step_detail) db.session.commit() - self.log_spiff_step_details(spiff_step_detail_mapping) + # self.log_spiff_step_details(spiff_step_detail_mapping) if len(human_tasks) > 0: for at in human_tasks: @@ -1255,7 +1255,7 @@ class ProcessInstanceProcessor: spiff_step_detail = SpiffStepDetailsModel(**step) db.session.add(spiff_step_detail) db.session.commit() - self.log_spiff_step_details(step) + # self.log_spiff_step_details(step) def manual_complete_task(self, task_id: str, execute: bool) -> None: """Mark the task complete optionally executing it.""" @@ -1720,7 +1720,7 @@ class ProcessInstanceProcessor: raise ApiError.from_workflow_exception("task_error", str(swe), swe) from swe finally: - self.log_spiff_step_details(step_details) + # self.log_spiff_step_details(step_details) db.session.bulk_insert_mappings(SpiffStepDetailsModel, step_details) spiff_logger = logging.getLogger("spiff") for handler in spiff_logger.handlers: diff --git a/spiffworkflow-frontend/cypress/pilot/pp1.cy.js b/spiffworkflow-frontend/cypress/pilot/pp1.cy.js index c34315b83..196559b6b 100644 --- a/spiffworkflow-frontend/cypress/pilot/pp1.cy.js +++ b/spiffworkflow-frontend/cypress/pilot/pp1.cy.js @@ -31,14 +31,17 @@ const approveWithUser = ( }; describe('pp1', () => { - it.skip('can run PP1', () => { + it('can run PP1', () => { cy.login('core5.contributor', 'core5.contributor'); cy.visit('/'); cy.contains('Start New +').click(); cy.contains('Raise New Demand Request'); cy.runPrimaryBpmnFile(true); - cy.contains('Procurement').click(); - // cy.contains('Submit').click(); + cy.contains('Please select the type of request to Start the process.'); + // wait a second to ensure we can click the radio button + cy.wait(1000); + cy.get('input#root-procurement').click(); + cy.wait(1000); cy.get('button') .contains(/^Submit$/) .click(); @@ -77,7 +80,8 @@ describe('pp1', () => { .click(); cy.contains( - 'Review and provide any supporting information or files for your request.' + 'Review and provide any supporting information or files for your request.', + { timeout: 20000 } ); cy.contains('Submit the Request').click(); cy.get('input[value="Submit the Request"]').click(); diff --git a/spiffworkflow-frontend/cypress/support/commands.js b/spiffworkflow-frontend/cypress/support/commands.js index 5e786a0dc..5fe791d10 100644 --- a/spiffworkflow-frontend/cypress/support/commands.js +++ b/spiffworkflow-frontend/cypress/support/commands.js @@ -109,7 +109,7 @@ Cypress.Commands.add( if (expectAutoRedirectToHumanTask) { // the url changes immediately, so also make sure we get some content from the next page, "Task:", or else when we try to interact with the page, it'll re-render and we'll get an error with cypress. cy.url().should('include', `/tasks/`); - cy.contains('Task: '); + cy.contains('Task: ', { timeout: 10000 }); } else { cy.contains(/Process Instance.*[kK]icked [oO]ff/); cy.reload(true); From 240c2372d2a70efdf1b0c20513d99305db050cc0 Mon Sep 17 00:00:00 2001 From: jasquat Date: Mon, 6 Mar 2023 17:24:01 -0500 Subject: [PATCH 20/25] avoid calling task_definition and bpmn_process_definition more than we need to w/ burnettk --- .../services/process_instance_processor.py | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py b/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py index e73e7ff09..e48597bf4 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py @@ -561,16 +561,24 @@ class ProcessInstanceProcessor: .filter_by(bpmn_process_definition_parent_id=bpmn_process_definition.id) .all() ) + + bpmn_subprocess_definition_bpmn_identifiers = {} for bpmn_subprocess_definition in bpmn_process_subprocess_definitions: - spec = cls._get_definition_dict_for_bpmn_process_definition( - bpmn_subprocess_definition - ) + bpmn_process_definition_dict: dict = bpmn_subprocess_definition.properties_json spiff_bpmn_process_dict["subprocess_specs"][ bpmn_subprocess_definition.bpmn_identifier - ] = spec - cls._set_definition_dict_for_bpmn_subprocess_definitions( - bpmn_subprocess_definition, spiff_bpmn_process_dict - ) + ] = bpmn_process_definition_dict + spiff_bpmn_process_dict["subprocess_specs"][ + bpmn_subprocess_definition.bpmn_identifier + ]['task_specs'] = {} + bpmn_subprocess_definition_bpmn_identifiers[bpmn_subprocess_definition.id] = bpmn_subprocess_definition.bpmn_identifier + + task_definitions = TaskDefinitionModel.query.filter( + TaskDefinitionModel.bpmn_process_definition_id.in_(bpmn_subprocess_definition_bpmn_identifiers.keys()) # type: ignore + ).all() + for task_definition in task_definitions: + bpmn_subprocess_definition_bpmn_identifier = bpmn_subprocess_definition_bpmn_identifiers[task_definition.bpmn_process_definition_id] + spiff_bpmn_process_dict["subprocess_specs"][bpmn_subprocess_definition_bpmn_identifier]['task_specs'][task_definition.bpmn_identifier] = task_definition.properties_json @classmethod def _get_bpmn_process_dict(cls, bpmn_process: BpmnProcessModel) -> dict: From 3a961753db985615654dcd7506ea383371612367 Mon Sep 17 00:00:00 2001 From: jasquat Date: Mon, 6 Mar 2023 17:30:35 -0500 Subject: [PATCH 21/25] pyl w/ burnettk --- .../services/process_instance_processor.py | 26 ++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py b/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py index e48597bf4..0305a865c 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py @@ -564,21 +564,35 @@ class ProcessInstanceProcessor: bpmn_subprocess_definition_bpmn_identifiers = {} for bpmn_subprocess_definition in bpmn_process_subprocess_definitions: - bpmn_process_definition_dict: dict = bpmn_subprocess_definition.properties_json + bpmn_process_definition_dict: dict = ( + bpmn_subprocess_definition.properties_json + ) spiff_bpmn_process_dict["subprocess_specs"][ bpmn_subprocess_definition.bpmn_identifier ] = bpmn_process_definition_dict spiff_bpmn_process_dict["subprocess_specs"][ bpmn_subprocess_definition.bpmn_identifier - ]['task_specs'] = {} - bpmn_subprocess_definition_bpmn_identifiers[bpmn_subprocess_definition.id] = bpmn_subprocess_definition.bpmn_identifier + ]["task_specs"] = {} + bpmn_subprocess_definition_bpmn_identifiers[ + bpmn_subprocess_definition.id + ] = bpmn_subprocess_definition.bpmn_identifier task_definitions = TaskDefinitionModel.query.filter( - TaskDefinitionModel.bpmn_process_definition_id.in_(bpmn_subprocess_definition_bpmn_identifiers.keys()) # type: ignore + TaskDefinitionModel.bpmn_process_definition_id.in_( # type: ignore + bpmn_subprocess_definition_bpmn_identifiers.keys() + ) ).all() for task_definition in task_definitions: - bpmn_subprocess_definition_bpmn_identifier = bpmn_subprocess_definition_bpmn_identifiers[task_definition.bpmn_process_definition_id] - spiff_bpmn_process_dict["subprocess_specs"][bpmn_subprocess_definition_bpmn_identifier]['task_specs'][task_definition.bpmn_identifier] = task_definition.properties_json + bpmn_subprocess_definition_bpmn_identifier = ( + bpmn_subprocess_definition_bpmn_identifiers[ + task_definition.bpmn_process_definition_id + ] + ) + spiff_bpmn_process_dict["subprocess_specs"][ + bpmn_subprocess_definition_bpmn_identifier + ]["task_specs"][ + task_definition.bpmn_identifier + ] = task_definition.properties_json @classmethod def _get_bpmn_process_dict(cls, bpmn_process: BpmnProcessModel) -> dict: From c329a69602b5bbef36afb7b2ef77daa512139642 Mon Sep 17 00:00:00 2001 From: burnettk Date: Mon, 6 Mar 2023 21:41:27 -0500 Subject: [PATCH 22/25] add cypress pilot script --- spiffworkflow-frontend/bin/cypress_pilot | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100755 spiffworkflow-frontend/bin/cypress_pilot diff --git a/spiffworkflow-frontend/bin/cypress_pilot b/spiffworkflow-frontend/bin/cypress_pilot new file mode 100755 index 000000000..6476d0562 --- /dev/null +++ b/spiffworkflow-frontend/bin/cypress_pilot @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +function error_handler() { + >&2 echo "Exited with BAD EXIT CODE '${2}' in ${0} script at line: ${1}." + exit "$2" +} +trap 'error_handler ${LINENO} $?' ERR +set -o errtrace -o errexit -o nounset -o pipefail + +# see also: npx cypress run --env grep="can filter",grepFilterSpecs=true +# https://github.com/cypress-io/cypress/tree/develop/npm/grep#pre-filter-specs-grepfilterspecs + +command="${1:-}" +if [[ -z "$command" ]]; then + command=open +else + shift +fi + +if [[ -z "${CYPRESS_SPIFFWORKFLOW_FRONTEND_AUTH_WITH_KEYCLOAK:-}" ]]; then + export CYPRESS_SPIFFWORKFLOW_FRONTEND_AUTH_WITH_KEYCLOAK=true +fi + +./node_modules/.bin/cypress "$command" -c specPattern="cypress/pilot/**/*.cy.{js,jsx,ts,tsx}" --e2e --browser chrome "$@" From 285b128879dd054e1ecb74bd36d8f947656f5811 Mon Sep 17 00:00:00 2001 From: burnettk Date: Mon, 6 Mar 2023 21:53:00 -0500 Subject: [PATCH 23/25] use a1 automated test users --- spiffworkflow-frontend/cypress/pilot/pp1.cy.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/spiffworkflow-frontend/cypress/pilot/pp1.cy.js b/spiffworkflow-frontend/cypress/pilot/pp1.cy.js index 196559b6b..aac3dbc8b 100644 --- a/spiffworkflow-frontend/cypress/pilot/pp1.cy.js +++ b/spiffworkflow-frontend/cypress/pilot/pp1.cy.js @@ -32,7 +32,7 @@ const approveWithUser = ( describe('pp1', () => { it('can run PP1', () => { - cy.login('core5.contributor', 'core5.contributor'); + cy.login('core-a1.contributor', 'core-a1.contributor'); cy.visit('/'); cy.contains('Start New +').click(); cy.contains('Raise New Demand Request'); @@ -95,14 +95,14 @@ describe('pp1', () => { processInstanceId, 'Task: Reminder: Request Additional Budget' ); - approveWithUser('ppg.ba.sme', processInstanceId); - approveWithUser('security.sme', processInstanceId); + approveWithUser('ppg.ba-a1.sme', processInstanceId); + approveWithUser('security-a1.sme', processInstanceId); approveWithUser( - 'infra.sme', + 'infra-a1.sme', processInstanceId, 'Task: Update Application Landscape' ); - approveWithUser('legal.sme', processInstanceId); + approveWithUser('legal-a1.sme', processInstanceId); }); }); }); From adcb5134b66297f096aea01e26a1ab8d980befb3 Mon Sep 17 00:00:00 2001 From: jasquat Date: Tue, 7 Mar 2023 09:28:38 -0500 Subject: [PATCH 24/25] a little cleanup of comments --- .../spiffworkflow_backend/models/json_data.py | 16 ++ .../models/spiff_step_details.py | 139 ------------------ 2 files changed, 16 insertions(+), 139 deletions(-) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/json_data.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/json_data.py index 103bdc57b..0723a50a0 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/json_data.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/json_data.py @@ -4,6 +4,22 @@ from spiffworkflow_backend.models.db import db from spiffworkflow_backend.models.db import SpiffworkflowBaseDBModel +# delta algorithm <- just to save it for when we want to try to implement it: +# a = {"hey": { "hey2": 2, "hey3": 3, "hey6": 7 }, "hey30": 3, "hey40": 4} +# b = {"hey": { "hey2": 4, "hey5": 3 }, "hey20": 2, "hey30": 3} +# a_keys = set(a.keys()) +# b_keys = set(b.keys()) +# removed = a_keys - b_keys +# added_keys = b_keys - a_keys +# keys_present_in_both = a_keys & b_keys +# changed = {} +# for key_in_both in keys_present_in_both: +# if a[key_in_both] != b[key_in_both]: +# changed[key_in_both] = b[key_in_both] +# added = {} +# for added_key in added_keys: +# added[added_key] = b[added_key] +# final_tuple = [added, removed, changed] class JsonDataModel(SpiffworkflowBaseDBModel): __tablename__ = "json_data" id: int = db.Column(db.Integer, primary_key=True) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/spiff_step_details.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/spiff_step_details.py index 6b91bafa1..713bd3cd8 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/spiff_step_details.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/spiff_step_details.py @@ -10,145 +10,6 @@ from spiffworkflow_backend.models.db import db from spiffworkflow_backend.models.db import SpiffworkflowBaseDBModel from spiffworkflow_backend.models.process_instance import ProcessInstanceModel -# process_instance: -# process_model_definition_id -# process_id - - -# "bpmn_messages", -# "correlations", -# "subprocesses", <-- omit from json -# -# also in subprocesses -# "data", -# "tasks" <-- omit from json -# "last_task", # guid generated by spiff -# "root", # guid generated by spiff -# "success", # boolean -# -# runtime_json: -# "last_task", # guid generated by spiff -# "root", # guid generated by spiff -# "success", # boolean -# "bpmn_messages", # if top-level process -# "correlations", # if top-level process -# process: -# id -# parent_process_id -# runtime_json <-- minus tasks and subproceses -# type <-- subprocess, top_level_process -# data - -# runtime_json: -# "id": "a56e1403-2838-4f03-a31f-f99afe16f38d", -# "parent": null, -# "children": [ -# "af6ba340-71e7-46d7-b2d4-e3db1751785d" -# ], -# "last_state_change": 1677775475.18116, -# "state": 32, -# "task_spec": "Root", -# "triggered": false, -# "workflow_name": "Process_category_number_one_call_activity_call_activity_test_bd2e724", -# "internal_data": {}, -# "data": {} -# task: -# id -# guid -# process_id -# task_definition_id -# state <-- store string value -# runtime_json -# data - -# ### workflow -# "io_specification": null, -# "data_objects": {}, -# "correlation_keys": {}, -# "typename": "BpmnProcessSpec" -# "name": "Process_category_number_one_call_activity_call_activity_test_bd2e724", -# "description": "Process_category_number_one_call_activity_call_activity_test_bd2e724", -# "file": "call_activity_test.bpmn", -# "task_specs": {} - -# ### definition -# "Root": { -# "id": "Process_category_number_one_call_activity_call_activity_test_bd2e724_8", -# "name": "Root", -# "description": "", -# "manual": false, -# "internal": false, -# "lookahead": 2, -# "inputs": [], -# "outputs": [], -# "typename": "Simple" -# } -# ## runtime -# "a56e1403-2838-4f03-a31f-f99afe16f38d": { -# "id": "a56e1403-2838-4f03-a31f-f99afe16f38d", -# "parent": null, -# "children": [ -# "af6ba340-71e7-46d7-b2d4-e3db1751785d" -# ], -# "last_state_change": 1677775475.18116, -# "state": 32, -# "task_spec": "Root", -# "triggered": false, -# "workflow_name": "Process_category_number_one_call_activity_call_activity_test_bd2e724", -# "internal_data": {}, -# "data": {} -# }, - - -# ### definition -# "StartEvent_1": { -# "id": "Process_category_number_one_call_activity_call_activity_test_bd2e724_4", -# "name": "StartEvent_1", -# "description": null, -# "manual": false, -# "internal": false, -# "lookahead": 2, -# "inputs": [ -# "Start" -# ], -# "outputs": [ -# "same_process_model" -# ], -# "lane": null, -# "documentation": null, -# "position": { -# "x": 179, -# "y": 159 -# }, -# "data_input_associations": [], -# "data_output_associations": [], -# "io_specification": null, -# "event_definition": { -# "internal": false, -# "external": false, -# "typename": "NoneEventDefinition" -# }, -# "typename": "StartEvent", -# "extensions": {} -# }, -# ## runtime -# "b86b5552-c541-4afe-b200-db0190439f38": { -# "id": "b86b5552-c541-4afe-b200-db0190439f38", -# "parent": "af6ba340-71e7-46d7-b2d4-e3db1751785d", -# "children": [ -# "e9525f55-794f-450d-b5da-bee1951f99fc" -# ], -# "last_state_change": 1677775475.1963174, -# "state": 32, -# "task_spec": "StartEvent_1", -# "triggered": false, -# "workflow_name": "Process_category_number_one_call_activity_call_activity_test_bd2e724", -# "internal_data": { -# "event_fired": true -# }, -# "data": {} -# }, - @dataclass class SpiffStepDetailsModel(SpiffworkflowBaseDBModel): From 87bacbdaaf15d874bcc804ce96364defbb642f28 Mon Sep 17 00:00:00 2001 From: jasquat Date: Tue, 7 Mar 2023 10:41:54 -0500 Subject: [PATCH 25/25] added some timestamp fields to bpmn process tables w/ burnettk --- .../{ede2ae7d3c80_.py => 389800c352ee_.py} | 14 +++++++++++--- .../spiffworkflow_backend/models/bpmn_process.py | 4 ++++ .../models/bpmn_process_definition.py | 3 +++ .../src/spiffworkflow_backend/models/task.py | 3 +++ .../models/task_definition.py | 3 +++ .../services/process_instance_processor.py | 3 +++ 6 files changed, 27 insertions(+), 3 deletions(-) rename spiffworkflow-backend/migrations/versions/{ede2ae7d3c80_.py => 389800c352ee_.py} (97%) diff --git a/spiffworkflow-backend/migrations/versions/ede2ae7d3c80_.py b/spiffworkflow-backend/migrations/versions/389800c352ee_.py similarity index 97% rename from spiffworkflow-backend/migrations/versions/ede2ae7d3c80_.py rename to spiffworkflow-backend/migrations/versions/389800c352ee_.py index 13a1560ad..bfcf5da8c 100644 --- a/spiffworkflow-backend/migrations/versions/ede2ae7d3c80_.py +++ b/spiffworkflow-backend/migrations/versions/389800c352ee_.py @@ -1,8 +1,8 @@ """empty message -Revision ID: ede2ae7d3c80 +Revision ID: 389800c352ee Revises: -Create Date: 2023-03-06 11:14:40.739641 +Create Date: 2023-03-07 10:40:43.709777 """ from alembic import op @@ -10,7 +10,7 @@ import sqlalchemy as sa from sqlalchemy.dialects import mysql # revision identifiers, used by Alembic. -revision = 'ede2ae7d3c80' +revision = '389800c352ee' down_revision = None branch_labels = None depends_on = None @@ -24,6 +24,8 @@ def upgrade(): sa.Column('parent_process_id', sa.Integer(), nullable=True), sa.Column('properties_json', sa.JSON(), nullable=False), sa.Column('json_data_hash', sa.String(length=255), nullable=False), + sa.Column('start_in_seconds', sa.DECIMAL(precision=17, scale=6), nullable=True), + sa.Column('end_in_seconds', sa.DECIMAL(precision=17, scale=6), nullable=True), sa.ForeignKeyConstraint(['parent_process_id'], ['bpmn_process.id'], ), sa.PrimaryKeyConstraint('id') ) @@ -37,6 +39,8 @@ def upgrade(): sa.Column('type', sa.String(length=32), nullable=True), sa.Column('bpmn_version_control_type', sa.String(length=50), nullable=True), sa.Column('bpmn_version_control_identifier', sa.String(length=255), nullable=True), + sa.Column('updated_at_in_seconds', sa.Integer(), nullable=True), + sa.Column('created_at_in_seconds', sa.Integer(), nullable=True), sa.PrimaryKeyConstraint('id') ) op.create_index(op.f('ix_bpmn_process_definition_bpmn_identifier'), 'bpmn_process_definition', ['bpmn_identifier'], unique=False) @@ -210,6 +214,8 @@ def upgrade(): sa.Column('state', sa.String(length=10), nullable=False), sa.Column('properties_json', sa.JSON(), nullable=False), sa.Column('json_data_hash', sa.String(length=255), nullable=False), + sa.Column('start_in_seconds', sa.DECIMAL(precision=17, scale=6), nullable=True), + sa.Column('end_in_seconds', sa.DECIMAL(precision=17, scale=6), nullable=True), sa.ForeignKeyConstraint(['bpmn_process_id'], ['bpmn_process.id'], ), sa.PrimaryKeyConstraint('id') ) @@ -221,6 +227,8 @@ def upgrade(): sa.Column('bpmn_identifier', sa.String(length=255), nullable=False), sa.Column('properties_json', sa.JSON(), nullable=False), sa.Column('typename', sa.String(length=255), nullable=False), + sa.Column('updated_at_in_seconds', sa.Integer(), nullable=True), + sa.Column('created_at_in_seconds', sa.Integer(), nullable=True), sa.ForeignKeyConstraint(['bpmn_process_definition_id'], ['bpmn_process_definition.id'], ), sa.PrimaryKeyConstraint('id'), sa.UniqueConstraint('bpmn_process_definition_id', 'bpmn_identifier', name='task_definition_unique') diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process.py index e1c146d4f..67e295e98 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process.py @@ -26,3 +26,7 @@ class BpmnProcessModel(SpiffworkflowBaseDBModel): # subprocess or top_level_process # process_type: str = db.Column(db.String(30), nullable=False) + + # FIXME: find out how to set this but it'd be cool + start_in_seconds: float = db.Column(db.DECIMAL(17, 6)) + end_in_seconds: float | None = db.Column(db.DECIMAL(17, 6)) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process_definition.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process_definition.py index d689a8f2f..4e7744efb 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process_definition.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/bpmn_process_definition.py @@ -32,3 +32,6 @@ class BpmnProcessDefinitionModel(SpiffworkflowBaseDBModel): # TODO: remove these from process_instance bpmn_version_control_type: str = db.Column(db.String(50)) bpmn_version_control_identifier: str = db.Column(db.String(255)) + + updated_at_in_seconds: int = db.Column(db.Integer) + created_at_in_seconds: int = db.Column(db.Integer) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/task.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/task.py index 254c7d56f..302df25ac 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/task.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/task.py @@ -54,6 +54,9 @@ class TaskModel(SpiffworkflowBaseDBModel): properties_json: dict = db.Column(db.JSON, nullable=False) json_data_hash: str = db.Column(db.String(255), nullable=False, index=True) + start_in_seconds: float = db.Column(db.DECIMAL(17, 6)) + end_in_seconds: float | None = db.Column(db.DECIMAL(17, 6)) + class Task: """Task.""" diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/task_definition.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/task_definition.py index 8abbb138b..358b7c1ce 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/task_definition.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/task_definition.py @@ -30,3 +30,6 @@ class TaskDefinitionModel(SpiffworkflowBaseDBModel): bpmn_identifier: str = db.Column(db.String(255), nullable=False, index=True) properties_json: dict = db.Column(db.JSON, nullable=False) typename: str = db.Column(db.String(255), nullable=False) + + updated_at_in_seconds: int = db.Column(db.Integer) + created_at_in_seconds: int = db.Column(db.Integer) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py b/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py index 7da47bfa6..67c786b13 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py @@ -1124,6 +1124,9 @@ class ProcessInstanceProcessor: # if self.process_instance_model.bpmn_process_definition_id is None: self._add_bpmn_process_definitions(bpmn_spec_dict) + # FIXME: Update tasks in the did_complete_task instead to set the final info. + # We will need to somehow cache all tasks initially though before each task is run. + # Maybe always do this for first run - just need to know it's the first run. subprocesses = process_instance_data_dict.pop("subprocesses") bpmn_process_parent = self._add_bpmn_process(process_instance_data_dict) for subprocess_task_id, subprocess_properties in subprocesses.items():