added new table and some notes on how to get a delta w/ burnettk jbirddog

spiffworkflow-backend/.flake8

@@ -17,10 +17,10 @@ per-file-ignores =
     # THEN, test_hey.py will NOT be excluding D103
 
     # asserts are ok in tests
-    tests/*:S101,D102,D103
+    tests/*:S101,D101,D102,D103
 
     # prefer naming functions descriptively rather than forcing comments
-    src/*:D102,D103
+    src/*:D101,D102,D103
 
     bin/keycloak_test_server.py:B950,D
     conftest.py:S105

spiffworkflow-backend/migrations/versions/fdf522e4c48a_.py

@@ -0,0 +1,33 @@
+"""empty message
+
+Revision ID: fdf522e4c48a
+Revises: ac6b60d7fee9
+Create Date: 2023-02-28 14:50:26.030983
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'fdf522e4c48a'
+down_revision = 'ac6b60d7fee9'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('serialized_bpmn_definition',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('hash', sa.String(length=255), nullable=False),
+    sa.Column('static_json', sa.JSON(), nullable=True),
+    sa.PrimaryKeyConstraint('id')
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('serialized_bpmn_definition')
+    # ### end Alembic commands ###

spiffworkflow-backend/poetry.lock

@@ -901,6 +901,14 @@ MarkupSafe = ">=2.0"
 [package.extras]
 i18n = ["Babel (>=2.7)"]
 
+[[package]]
+name = "json-delta"
+version = "2.0.2"
+description = "A diff/patch pair for JSON-serialized data structures."
+category = "main"
+optional = false
+python-versions = "*"
+
 [[package]]
 name = "jsonschema"
 version = "4.16.0"
@@ -1817,7 +1825,7 @@ lxml = "*"
 type = "git"
 url = "https://github.com/sartography/SpiffWorkflow"
 reference = "main"
-resolved_reference = "b3235fad598ee3c4680a23f26adb09cdc8f2807b"
+resolved_reference = "161cb7a4509a3d0e0574f3e2a98157862c053bad"
 
 [[package]]
 name = "SQLAlchemy"
@@ -2196,7 +2204,7 @@ testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools"
 [metadata]
 lock-version = "1.1"
 python-versions = ">=3.9,<3.12"
-content-hash = "3876acb4e3d947787a3ba8e831844ca0b06bde34dc038be46cabc00aa2a4defe"
+content-hash = "af711e2941c42b837da47ca8d647b1ae44657bf6805353bc216bb49cb3cbbfae"
 
 [metadata.files]
 alabaster = [
@@ -2602,6 +2610,10 @@ Jinja2 = [
     {file = "Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"},
     {file = "Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"},
 ]
+json-delta = [
+    {file = "json_delta-2.0.2-py2.py3-none-any.whl", hash = "sha256:12bc798354ea722fa04fae21ea06879321c47b0887572c27384accd6ef28efbf"},
+    {file = "json_delta-2.0.2.tar.gz", hash = "sha256:95ea3ff9908fc7d634c27ffec11db8fd8d935aa3e895d7302915d394b10e0321"},
+]
 jsonschema = [
     {file = "jsonschema-4.16.0-py3-none-any.whl", hash = "sha256:9e74b8f9738d6a946d70705dc692b74b5429cd0960d58e79ffecfc43b2221eb9"},
     {file = "jsonschema-4.16.0.tar.gz", hash = "sha256:165059f076eff6971bae5b742fc029a7b4ef3f9bcf04c14e4776a7605de14b23"},
@@ -2650,6 +2662,7 @@ lazy-object-proxy = [
     {file = "lazy_object_proxy-1.7.1-pp37.pp38-none-any.whl", hash = "sha256:d66906d5785da8e0be7360912e99c9188b70f52c422f9fc18223347235691a84"},
 ]
 livereload = [
+    {file = "livereload-2.6.3-py2.py3-none-any.whl", hash = "sha256:ad4ac6f53b2d62bb6ce1a5e6e96f1f00976a32348afedcb4b6d68df2a1d346e4"},
     {file = "livereload-2.6.3.tar.gz", hash = "sha256:776f2f865e59fde56490a56bcc6773b6917366bce0c267c60ee8aaf1a0959869"},
 ]
 lxml = [
@@ -3149,6 +3162,7 @@ ruamel-yaml-clib = [
     {file = "ruamel.yaml.clib-0.2.7-cp310-cp310-win_amd64.whl", hash = "sha256:d000f258cf42fec2b1bbf2863c61d7b8918d31ffee905da62dede869254d3b8a"},
     {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:045e0626baf1c52e5527bd5db361bc83180faaba2ff586e763d3d5982a876a9e"},
     {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-macosx_12_6_arm64.whl", hash = "sha256:721bc4ba4525f53f6a611ec0967bdcee61b31df5a56801281027a3a6d1c2daf5"},
+    {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:41d0f1fa4c6830176eef5b276af04c89320ea616655d01327d5ce65e50575c94"},
     {file = "ruamel.yaml.clib-0.2.7-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:4b3a93bb9bc662fc1f99c5c3ea8e623d8b23ad22f861eb6fce9377ac07ad6072"},
     {file = "ruamel.yaml.clib-0.2.7-cp36-cp36m-macosx_12_0_arm64.whl", hash = "sha256:a234a20ae07e8469da311e182e70ef6b199d0fbeb6c6cc2901204dd87fb867e8"},
     {file = "ruamel.yaml.clib-0.2.7-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:15910ef4f3e537eea7fe45f8a5d19997479940d9196f357152a09031c5be59f3"},

spiffworkflow-backend/pyproject.toml

@@ -72,6 +72,7 @@ dateparser = "^1.1.2"
 types-dateparser = "^1.1.4.1"
 flask-jwt-extended = "^4.4.4"
 pylint = "^2.15.10"
+json-delta = "^2.0.2"
 
 
 [tool.poetry.dev-dependencies]

spiffworkflow-backend/src/spiffworkflow_backend/load_database_models.py

@@ -50,5 +50,8 @@ from spiffworkflow_backend.models.group import GroupModel  # noqa: F401
 from spiffworkflow_backend.models.process_instance_metadata import (
     ProcessInstanceMetadataModel,
 )  # noqa: F401
+from spiffworkflow_backend.models.serialized_bpmn_definition import SerializedBpmnDefinitionModel  # noqa: F401
+
+
 
 add_listeners()

spiffworkflow-backend/src/spiffworkflow_backend/models/process_instance.py

@@ -79,6 +79,21 @@ class ProcessInstanceModel(SpiffworkflowBaseDBModel):
         cascade="delete",
     )  # type: ignore
 
+    # static
+    #   "subprocess_specs",
+    #   "spec",
+    #   "serializer_version",
+
+    # runtime
+    #   "bpmn_messages",
+    #   "correlations",
+    #   "data",
+    #   "last_task",
+    #   "root",
+    #   "subprocesses",
+    #   "success",
+    #   "tasks"
+
     bpmn_json: str | None = deferred(db.Column(db.JSON))  # type: ignore
     start_in_seconds: int | None = db.Column(db.Integer)
     end_in_seconds: int | None = db.Column(db.Integer)

spiffworkflow-backend/src/spiffworkflow_backend/models/serialized_bpmn_definition.py

@@ -0,0 +1,50 @@
+"""Process_instance."""
+from __future__ import annotations
+
+from sqlalchemy.orm import deferred
+
+from spiffworkflow_backend.models.db import db
+from spiffworkflow_backend.models.db import SpiffworkflowBaseDBModel
+
+
+# top level serialized keys
+#
+# static
+#   "subprocess_specs",
+#   "spec",
+#   "serializer_version",
+#
+# runtime
+#   "bpmn_messages",
+#   "correlations",
+#   "data",
+#   "subprocesses",
+#   "tasks"
+#
+# new columns on process_instance
+#   "last_task", # guid generated by spiff
+#   "root", # guid generated by spiff
+#   "success", # boolean
+#
+# delta algorithm:
+#   a = {"hey": { "hey2": 2, "hey3": 3, "hey6": 7 }, "hey30": 3, "hey40": 4}
+#   b = {"hey": { "hey2": 4, "hey5": 3 }, "hey20": 2, "hey30": 3}
+#   a_keys = set(a.keys())
+#   b_keys = set(b.keys())
+#   removed = a_keys - b_keys
+#   added_keys = b_keys - a_keys
+#   keys_present_in_both = a_keys & b_keys
+#   changed = {}
+#   for key_in_both in keys_present_in_both:
+#       if a[key_in_both] != b[key_in_both]:
+#           changed[key_in_both] = b[key_in_both]
+#   added = {}
+#   for added_key in added_keys:
+#       added[added_key] = b[added_key]
+#   final_tuple = [added, removed, changed]
+class SerializedBpmnDefinitionModel(SpiffworkflowBaseDBModel):
+
+    __tablename__ = "serialized_bpmn_definition"
+    id: int = db.Column(db.Integer, primary_key=True)
+    hash: str = db.Column(db.String(255), nullable=False)
+    static_json: str | None = deferred(db.Column(db.JSON))  # type: ignore

spiffworkflow-backend/src/spiffworkflow_backend/models/spiff_step_details.py

@@ -31,6 +31,7 @@ class SpiffStepDetailsModel(SpiffworkflowBaseDBModel):
     task_id: str = db.Column(db.String(50), nullable=False)
     task_state: str = db.Column(db.String(50), nullable=False)
     bpmn_task_identifier: str = db.Column(db.String(255), nullable=False)
+    delta_json: list = deferred(db.Column(db.JSON, nullable=False))  # type: ignore
 
     start_in_seconds: float = db.Column(db.DECIMAL(17, 6), nullable=False)
 

spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py

@@ -204,6 +204,7 @@ class NonTaskDataBasedScriptEngineEnvironment(BasePythonScriptEngineEnvironment)
         external_methods: Optional[Dict[str, Any]] = None,
     ) -> None:
         # TODO: once integrated look at the tests that fail without Box
+        # context is task.data
         Box.convert_to_box(context)
         self.state.update(self.globals)
         self.state.update(external_methods or {})

spiffworkflow-backend/tests/spiffworkflow_backend/integration/test_process_api.py

@@ -1,8 +1,10 @@
 """Test Process Api Blueprint."""
 import io
+from typing import Set
 import json
 import os
 import time
+import json_delta
 from typing import Any
 from typing import Dict