diff --git a/spiffworkflow-backend/.flake8 b/spiffworkflow-backend/.flake8 index 456b4422..1f7217bc 100644 --- a/spiffworkflow-backend/.flake8 +++ b/spiffworkflow-backend/.flake8 @@ -17,10 +17,10 @@ per-file-ignores = # THEN, test_hey.py will NOT be excluding D103 # asserts are ok in tests - tests/*:S101,D102,D103 + tests/*:S101,D101,D102,D103 # prefer naming functions descriptively rather than forcing comments - src/*:D102,D103 + src/*:D101,D102,D103 bin/keycloak_test_server.py:B950,D conftest.py:S105 diff --git a/spiffworkflow-backend/migrations/versions/fdf522e4c48a_.py b/spiffworkflow-backend/migrations/versions/fdf522e4c48a_.py new file mode 100644 index 00000000..051710fa --- /dev/null +++ b/spiffworkflow-backend/migrations/versions/fdf522e4c48a_.py @@ -0,0 +1,33 @@ +"""empty message + +Revision ID: fdf522e4c48a +Revises: ac6b60d7fee9 +Create Date: 2023-02-28 14:50:26.030983 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'fdf522e4c48a' +down_revision = 'ac6b60d7fee9' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('serialized_bpmn_definition', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('hash', sa.String(length=255), nullable=False), + sa.Column('static_json', sa.JSON(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('serialized_bpmn_definition') + # ### end Alembic commands ### diff --git a/spiffworkflow-backend/poetry.lock b/spiffworkflow-backend/poetry.lock index 00504090..1f9bf39d 100644 --- a/spiffworkflow-backend/poetry.lock +++ b/spiffworkflow-backend/poetry.lock @@ -901,6 +901,14 @@ MarkupSafe = ">=2.0" [package.extras] i18n = ["Babel (>=2.7)"] +[[package]] +name = "json-delta" +version = "2.0.2" +description = "A diff/patch pair for JSON-serialized data structures." +category = "main" +optional = false +python-versions = "*" + [[package]] name = "jsonschema" version = "4.16.0" @@ -1817,7 +1825,7 @@ lxml = "*" type = "git" url = "https://github.com/sartography/SpiffWorkflow" reference = "main" -resolved_reference = "b3235fad598ee3c4680a23f26adb09cdc8f2807b" +resolved_reference = "161cb7a4509a3d0e0574f3e2a98157862c053bad" [[package]] name = "SQLAlchemy" @@ -2196,7 +2204,7 @@ testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools" [metadata] lock-version = "1.1" python-versions = ">=3.9,<3.12" -content-hash = "3876acb4e3d947787a3ba8e831844ca0b06bde34dc038be46cabc00aa2a4defe" +content-hash = "af711e2941c42b837da47ca8d647b1ae44657bf6805353bc216bb49cb3cbbfae" [metadata.files] alabaster = [ @@ -2602,6 +2610,10 @@ Jinja2 = [ {file = "Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, {file = "Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"}, ] +json-delta = [ + {file = "json_delta-2.0.2-py2.py3-none-any.whl", hash = "sha256:12bc798354ea722fa04fae21ea06879321c47b0887572c27384accd6ef28efbf"}, + {file = "json_delta-2.0.2.tar.gz", hash = "sha256:95ea3ff9908fc7d634c27ffec11db8fd8d935aa3e895d7302915d394b10e0321"}, +] jsonschema = [ {file = "jsonschema-4.16.0-py3-none-any.whl", hash = "sha256:9e74b8f9738d6a946d70705dc692b74b5429cd0960d58e79ffecfc43b2221eb9"}, {file = "jsonschema-4.16.0.tar.gz", hash = "sha256:165059f076eff6971bae5b742fc029a7b4ef3f9bcf04c14e4776a7605de14b23"}, @@ -2650,6 +2662,7 @@ lazy-object-proxy = [ {file = "lazy_object_proxy-1.7.1-pp37.pp38-none-any.whl", hash = "sha256:d66906d5785da8e0be7360912e99c9188b70f52c422f9fc18223347235691a84"}, ] livereload = [ + {file = "livereload-2.6.3-py2.py3-none-any.whl", hash = "sha256:ad4ac6f53b2d62bb6ce1a5e6e96f1f00976a32348afedcb4b6d68df2a1d346e4"}, {file = "livereload-2.6.3.tar.gz", hash = "sha256:776f2f865e59fde56490a56bcc6773b6917366bce0c267c60ee8aaf1a0959869"}, ] lxml = [ @@ -3149,6 +3162,7 @@ ruamel-yaml-clib = [ {file = "ruamel.yaml.clib-0.2.7-cp310-cp310-win_amd64.whl", hash = "sha256:d000f258cf42fec2b1bbf2863c61d7b8918d31ffee905da62dede869254d3b8a"}, {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:045e0626baf1c52e5527bd5db361bc83180faaba2ff586e763d3d5982a876a9e"}, {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-macosx_12_6_arm64.whl", hash = "sha256:721bc4ba4525f53f6a611ec0967bdcee61b31df5a56801281027a3a6d1c2daf5"}, + {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:41d0f1fa4c6830176eef5b276af04c89320ea616655d01327d5ce65e50575c94"}, {file = "ruamel.yaml.clib-0.2.7-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:4b3a93bb9bc662fc1f99c5c3ea8e623d8b23ad22f861eb6fce9377ac07ad6072"}, {file = "ruamel.yaml.clib-0.2.7-cp36-cp36m-macosx_12_0_arm64.whl", hash = "sha256:a234a20ae07e8469da311e182e70ef6b199d0fbeb6c6cc2901204dd87fb867e8"}, {file = "ruamel.yaml.clib-0.2.7-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:15910ef4f3e537eea7fe45f8a5d19997479940d9196f357152a09031c5be59f3"}, diff --git a/spiffworkflow-backend/pyproject.toml b/spiffworkflow-backend/pyproject.toml index 84373861..fcfa810d 100644 --- a/spiffworkflow-backend/pyproject.toml +++ b/spiffworkflow-backend/pyproject.toml @@ -72,6 +72,7 @@ dateparser = "^1.1.2" types-dateparser = "^1.1.4.1" flask-jwt-extended = "^4.4.4" pylint = "^2.15.10" +json-delta = "^2.0.2" [tool.poetry.dev-dependencies] diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/load_database_models.py b/spiffworkflow-backend/src/spiffworkflow_backend/load_database_models.py index 8e79e013..16648b93 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/load_database_models.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/load_database_models.py @@ -50,5 +50,8 @@ from spiffworkflow_backend.models.group import GroupModel # noqa: F401 from spiffworkflow_backend.models.process_instance_metadata import ( ProcessInstanceMetadataModel, ) # noqa: F401 +from spiffworkflow_backend.models.serialized_bpmn_definition import SerializedBpmnDefinitionModel # noqa: F401 + + add_listeners() diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/process_instance.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/process_instance.py index 62f5af3f..24f1aff4 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/process_instance.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/process_instance.py @@ -79,6 +79,21 @@ class ProcessInstanceModel(SpiffworkflowBaseDBModel): cascade="delete", ) # type: ignore + # static + # "subprocess_specs", + # "spec", + # "serializer_version", + + # runtime + # "bpmn_messages", + # "correlations", + # "data", + # "last_task", + # "root", + # "subprocesses", + # "success", + # "tasks" + bpmn_json: str | None = deferred(db.Column(db.JSON)) # type: ignore start_in_seconds: int | None = db.Column(db.Integer) end_in_seconds: int | None = db.Column(db.Integer) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/serialized_bpmn_definition.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/serialized_bpmn_definition.py new file mode 100644 index 00000000..6003e24b --- /dev/null +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/serialized_bpmn_definition.py @@ -0,0 +1,50 @@ +"""Process_instance.""" +from __future__ import annotations + +from sqlalchemy.orm import deferred + +from spiffworkflow_backend.models.db import db +from spiffworkflow_backend.models.db import SpiffworkflowBaseDBModel + + +# top level serialized keys +# +# static +# "subprocess_specs", +# "spec", +# "serializer_version", +# +# runtime +# "bpmn_messages", +# "correlations", +# "data", +# "subprocesses", +# "tasks" +# +# new columns on process_instance +# "last_task", # guid generated by spiff +# "root", # guid generated by spiff +# "success", # boolean +# +# delta algorithm: +# a = {"hey": { "hey2": 2, "hey3": 3, "hey6": 7 }, "hey30": 3, "hey40": 4} +# b = {"hey": { "hey2": 4, "hey5": 3 }, "hey20": 2, "hey30": 3} +# a_keys = set(a.keys()) +# b_keys = set(b.keys()) +# removed = a_keys - b_keys +# added_keys = b_keys - a_keys +# keys_present_in_both = a_keys & b_keys +# changed = {} +# for key_in_both in keys_present_in_both: +# if a[key_in_both] != b[key_in_both]: +# changed[key_in_both] = b[key_in_both] +# added = {} +# for added_key in added_keys: +# added[added_key] = b[added_key] +# final_tuple = [added, removed, changed] +class SerializedBpmnDefinitionModel(SpiffworkflowBaseDBModel): + + __tablename__ = "serialized_bpmn_definition" + id: int = db.Column(db.Integer, primary_key=True) + hash: str = db.Column(db.String(255), nullable=False) + static_json: str | None = deferred(db.Column(db.JSON)) # type: ignore diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/spiff_step_details.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/spiff_step_details.py index 9fd1b296..79715185 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/spiff_step_details.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/spiff_step_details.py @@ -31,6 +31,7 @@ class SpiffStepDetailsModel(SpiffworkflowBaseDBModel): task_id: str = db.Column(db.String(50), nullable=False) task_state: str = db.Column(db.String(50), nullable=False) bpmn_task_identifier: str = db.Column(db.String(255), nullable=False) + delta_json: list = deferred(db.Column(db.JSON, nullable=False)) # type: ignore start_in_seconds: float = db.Column(db.DECIMAL(17, 6), nullable=False) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py b/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py index 71055d9a..c2248b62 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py @@ -204,6 +204,7 @@ class NonTaskDataBasedScriptEngineEnvironment(BasePythonScriptEngineEnvironment) external_methods: Optional[Dict[str, Any]] = None, ) -> None: # TODO: once integrated look at the tests that fail without Box + # context is task.data Box.convert_to_box(context) self.state.update(self.globals) self.state.update(external_methods or {}) diff --git a/spiffworkflow-backend/tests/spiffworkflow_backend/integration/test_process_api.py b/spiffworkflow-backend/tests/spiffworkflow_backend/integration/test_process_api.py index 881f11ca..a7ea5068 100644 --- a/spiffworkflow-backend/tests/spiffworkflow_backend/integration/test_process_api.py +++ b/spiffworkflow-backend/tests/spiffworkflow_backend/integration/test_process_api.py @@ -1,8 +1,10 @@ """Test Process Api Blueprint.""" import io +from typing import Set import json import os import time +import json_delta from typing import Any from typing import Dict