From 6a3f8a212ff9e20b9e944a3721fdc17267540d6d Mon Sep 17 00:00:00 2001 From: Kevin Burnett <18027+burnettk@users.noreply.github.com> Date: Thu, 1 Feb 2024 06:51:19 -0800 Subject: [PATCH] update spiff and remove dot notation (#933) * update spiff and remove dot notation * update script_engine dependency based on spiff lib update --------- Co-authored-by: burnettk --- spiffworkflow-backend/poetry.lock | 8 ++-- .../spiffworkflow_backend/data_stores/json.py | 2 +- .../spiffworkflow_backend/data_stores/kkv.py | 2 +- .../data_stores/typeahead.py | 2 +- .../models/message_instance.py | 2 +- .../routes/extensions_controller.py | 3 +- .../services/message_service.py | 2 +- .../services/process_instance_processor.py | 8 ++-- .../services/process_instance_service.py | 48 ++----------------- .../process_model_test_runner_service.py | 3 +- .../services/service_task_service.py | 2 +- .../unit/test_dot_notation.py | 18 ++++--- .../unit/test_process_instance_service.py | 2 +- 13 files changed, 35 insertions(+), 67 deletions(-) diff --git a/spiffworkflow-backend/poetry.lock b/spiffworkflow-backend/poetry.lock index 6d5d4ad5f..0513ab14b 100644 --- a/spiffworkflow-backend/poetry.lock +++ b/spiffworkflow-backend/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. [[package]] name = "alembic" @@ -2036,6 +2036,7 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, @@ -2366,7 +2367,8 @@ files = [ {file = "ruamel.yaml.clib-0.2.7-cp310-cp310-win32.whl", hash = "sha256:763d65baa3b952479c4e972669f679fe490eee058d5aa85da483ebae2009d231"}, {file = "ruamel.yaml.clib-0.2.7-cp310-cp310-win_amd64.whl", hash = "sha256:d000f258cf42fec2b1bbf2863c61d7b8918d31ffee905da62dede869254d3b8a"}, {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:045e0626baf1c52e5527bd5db361bc83180faaba2ff586e763d3d5982a876a9e"}, - {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-macosx_12_6_arm64.whl", hash = "sha256:721bc4ba4525f53f6a611ec0967bdcee61b31df5a56801281027a3a6d1c2daf5"}, + {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:1a6391a7cabb7641c32517539ca42cf84b87b667bad38b78d4d42dd23e957c81"}, + {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:9c7617df90c1365638916b98cdd9be833d31d337dbcd722485597b43c4a215bf"}, {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:41d0f1fa4c6830176eef5b276af04c89320ea616655d01327d5ce65e50575c94"}, {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-win32.whl", hash = "sha256:f6d3d39611ac2e4f62c3128a9eed45f19a6608670c5a2f4f07f24e8de3441d38"}, {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-win_amd64.whl", hash = "sha256:da538167284de58a52109a9b89b8f6a53ff8437dd6dc26d33b57bf6699153122"}, @@ -2663,7 +2665,7 @@ lxml = "*" type = "git" url = "https://github.com/sartography/SpiffWorkflow" reference = "main" -resolved_reference = "5b768ad681b8fd1a7df93daef8be83f157dd49b6" +resolved_reference = "633de80a722cf28f4a79df9de7be911130f1f5ad" [[package]] name = "spiffworkflow-connector-command" diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/data_stores/json.py b/spiffworkflow-backend/src/spiffworkflow_backend/data_stores/json.py index 46f56ed92..5b069e088 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/data_stores/json.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/data_stores/json.py @@ -2,7 +2,7 @@ from typing import Any import jsonschema # type: ignore from flask import current_app -from SpiffWorkflow.bpmn.serializer.helpers.registry import BpmnConverter # type: ignore +from SpiffWorkflow.bpmn.serializer.helpers import BpmnConverter # type: ignore from SpiffWorkflow.bpmn.specs.data_spec import BpmnDataStoreSpecification # type: ignore from SpiffWorkflow.task import Task as SpiffTask # type: ignore diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/data_stores/kkv.py b/spiffworkflow-backend/src/spiffworkflow_backend/data_stores/kkv.py index e2d220e3b..2c384e181 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/data_stores/kkv.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/data_stores/kkv.py @@ -1,6 +1,6 @@ from typing import Any -from SpiffWorkflow.bpmn.serializer.helpers.registry import BpmnConverter # type: ignore +from SpiffWorkflow.bpmn.serializer.helpers import BpmnConverter # type: ignore from SpiffWorkflow.bpmn.specs.data_spec import BpmnDataStoreSpecification # type: ignore from SpiffWorkflow.task import Task as SpiffTask # type: ignore diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/data_stores/typeahead.py b/spiffworkflow-backend/src/spiffworkflow_backend/data_stores/typeahead.py index 5f8b26c75..156eaffc8 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/data_stores/typeahead.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/data_stores/typeahead.py @@ -1,7 +1,7 @@ from time import time from typing import Any -from SpiffWorkflow.bpmn.serializer.helpers.registry import BpmnConverter # type: ignore +from SpiffWorkflow.bpmn.serializer.helpers import BpmnConverter # type: ignore from SpiffWorkflow.bpmn.specs.data_spec import BpmnDataStoreSpecification # type: ignore from SpiffWorkflow.task import Task as SpiffTask # type: ignore diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/message_instance.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/message_instance.py index 07938d8a4..5c8b4f7eb 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/message_instance.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/message_instance.py @@ -4,7 +4,7 @@ from typing import TYPE_CHECKING from typing import Any from flask import current_app -from SpiffWorkflow.bpmn.PythonScriptEngine import PythonScriptEngine # type: ignore +from SpiffWorkflow.bpmn.script_engine import PythonScriptEngine # type: ignore from sqlalchemy import ForeignKey from sqlalchemy.event import listens_for from sqlalchemy.orm import Session diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/routes/extensions_controller.py b/spiffworkflow-backend/src/spiffworkflow_backend/routes/extensions_controller.py index e9a42aaef..97f1ebe43 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/routes/extensions_controller.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/routes/extensions_controller.py @@ -7,6 +7,7 @@ from flask import g from flask import jsonify from flask import make_response from flask.wrappers import Response +from SpiffWorkflow.util.deep_merge import DeepMerge # type: ignore from spiffworkflow_backend.exceptions.api_error import ApiError from spiffworkflow_backend.models.process_instance import ProcessInstanceModel @@ -132,7 +133,7 @@ def _run_extension( if body and "extension_input" in body: processor.do_engine_steps(save=save_to_db, execution_strategy_name="run_current_ready_tasks") next_task = processor.next_task() - next_task.update_data(body["extension_input"]) + DeepMerge.merge(next_task.data, body["extension_input"]) processor.do_engine_steps(save=save_to_db, execution_strategy_name="greedy") except ( ApiError, diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/services/message_service.py b/spiffworkflow-backend/src/spiffworkflow_backend/services/message_service.py index 4cbd3b0f0..ad6f2c51c 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/services/message_service.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/services/message_service.py @@ -1,4 +1,4 @@ -from SpiffWorkflow.bpmn.event import BpmnEvent # type: ignore +from SpiffWorkflow.bpmn import BpmnEvent # type: ignore from SpiffWorkflow.bpmn.specs.event_definitions.message import CorrelationProperty # type: ignore from SpiffWorkflow.spiff.specs.event_definitions import MessageEventDefinition # type: ignore diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py b/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py index faf3b42b8..73176cd30 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py @@ -25,12 +25,12 @@ from flask import current_app from lxml import etree # type: ignore from lxml.etree import XMLSyntaxError # type: ignore from RestrictedPython import safe_globals # type: ignore -from SpiffWorkflow.bpmn.event import BpmnEvent # type: ignore +from SpiffWorkflow.bpmn import BpmnEvent # type: ignore from SpiffWorkflow.bpmn.exceptions import WorkflowTaskException # type: ignore from SpiffWorkflow.bpmn.parser.ValidationException import ValidationException # type: ignore -from SpiffWorkflow.bpmn.PythonScriptEngine import PythonScriptEngine # type: ignore -from SpiffWorkflow.bpmn.PythonScriptEngineEnvironment import BasePythonScriptEngineEnvironment # type: ignore -from SpiffWorkflow.bpmn.PythonScriptEngineEnvironment import TaskDataEnvironment +from SpiffWorkflow.bpmn.script_engine import BasePythonScriptEngineEnvironment # type: ignore +from SpiffWorkflow.bpmn.script_engine import PythonScriptEngine +from SpiffWorkflow.bpmn.script_engine import TaskDataEnvironment from SpiffWorkflow.bpmn.serializer.default.task_spec import EventConverter # type: ignore from SpiffWorkflow.bpmn.serializer.helpers.registry import DefaultRegistry # type: ignore from SpiffWorkflow.bpmn.serializer.workflow import BpmnWorkflowSerializer # type: ignore diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_service.py b/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_service.py index 43c4ab824..24a013e19 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_service.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_service.py @@ -11,11 +11,12 @@ from urllib.parse import unquote import sentry_sdk from flask import current_app from flask import g -from SpiffWorkflow.bpmn.event import PendingBpmnEvent # type: ignore from SpiffWorkflow.bpmn.specs.control import BoundaryEventSplit # type: ignore from SpiffWorkflow.bpmn.specs.defaults import BoundaryEvent # type: ignore from SpiffWorkflow.bpmn.specs.event_definitions.timer import TimerEventDefinition # type: ignore +from SpiffWorkflow.bpmn.util import PendingBpmnEvent # type: ignore from SpiffWorkflow.task import Task as SpiffTask # type: ignore +from SpiffWorkflow.util.deep_merge import DeepMerge # type: ignore from SpiffWorkflow.util.task import TaskState # type: ignore from spiffworkflow_backend.background_processing.celery_tasks.process_instance_task_producer import ( @@ -456,8 +457,7 @@ class ProcessInstanceService: data, process_instance.id, ) - dot_dct = cls.create_dot_dict(data) - spiff_task.update_data(dot_dct) + DeepMerge.merge(spiff_task.data, data) @staticmethod def complete_form_task( @@ -483,48 +483,6 @@ class ProcessInstanceService: # maybe move this out once we have the interstitial page since this is here just so we can get the next human task processor.do_engine_steps(save=True) - @staticmethod - def create_dot_dict(data: dict) -> dict[str, Any]: - dot_dict: dict[str, Any] = {} - for key, value in data.items(): - ProcessInstanceService.set_dot_value(key, value, dot_dict) - return dot_dict - - @staticmethod - def get_dot_value(path: str, source: dict) -> Any: - # Given a path in dot notation, uas as 'fruit.type' tries to find that value in - # the source, but looking deep in the dictionary. - paths = path.split(".") # [a,b,c] - s = source - index = 0 - for p in paths: - index += 1 - if isinstance(s, dict) and p in s: - if index == len(paths): - return s[p] - else: - s = s[p] - if path in source: - return source[path] - return None - - @staticmethod - def set_dot_value(path: str, value: Any, target: dict) -> dict: - # Given a path in dot notation, such as "fruit.type", and a value "apple", will - # set the value in the target dictionary, as target["fruit"]["type"]="apple" - destination = target - paths = path.split(".") # [a,b,c] - index = 0 - for p in paths: - index += 1 - if p not in destination: - if index == len(paths): - destination[p] = value - else: - destination[p] = {} - destination = destination[p] - return target - @staticmethod def spiff_task_to_api_task( processor: ProcessInstanceProcessor, diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/services/process_model_test_runner_service.py b/spiffworkflow-backend/src/spiffworkflow_backend/services/process_model_test_runner_service.py index e46864718..25673a8ad 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/services/process_model_test_runner_service.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/services/process_model_test_runner_service.py @@ -10,6 +10,7 @@ from lxml import etree # type: ignore from SpiffWorkflow.bpmn.exceptions import WorkflowTaskException # type: ignore from SpiffWorkflow.bpmn.workflow import BpmnWorkflow # type: ignore from SpiffWorkflow.task import Task as SpiffTask # type: ignore +from SpiffWorkflow.util.deep_merge import DeepMerge # type: ignore from SpiffWorkflow.util.task import TaskState # type: ignore from spiffworkflow_backend.services.custom_parser import MyCustomParser @@ -121,7 +122,7 @@ class ProcessModelTestRunnerMostlyPureSpiffDelegate(ProcessModelTestRunnerDelega def execute_task(self, spiff_task: SpiffTask, task_data_for_submit: dict | None = None) -> None: if task_data_for_submit is not None or spiff_task.task_spec.manual: if task_data_for_submit is not None: - spiff_task.update_data(task_data_for_submit) + DeepMerge.merge(spiff_task.data, task_data_for_submit) spiff_task.complete() else: spiff_task.run() diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/services/service_task_service.py b/spiffworkflow-backend/src/spiffworkflow_backend/services/service_task_service.py index ba26a2989..7773d49eb 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/services/service_task_service.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/services/service_task_service.py @@ -7,7 +7,7 @@ import requests import sentry_sdk from flask import current_app from flask import g -from SpiffWorkflow.bpmn.event import BpmnEvent # type: ignore +from SpiffWorkflow.bpmn import BpmnEvent # type: ignore from SpiffWorkflow.bpmn.exceptions import WorkflowTaskException # type: ignore from SpiffWorkflow.spiff.specs.defaults import ServiceTask # type: ignore from SpiffWorkflow.spiff.specs.event_definitions import ErrorEventDefinition # type: ignore diff --git a/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_dot_notation.py b/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_dot_notation.py index adf93a82e..0a3962efb 100644 --- a/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_dot_notation.py +++ b/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_dot_notation.py @@ -8,6 +8,10 @@ from tests.spiffworkflow_backend.helpers.test_data import load_test_spec class TestDotNotation(BaseTest): + # this used to prove the point that dot notation got converted into deeply-nested dictionaries. + # it doesn't do that any more, and just behaves more like you would expect (flat dictionary with dots in the keys), + # but it didn't seem obvious that the test was worthless, and this will at least prove it doesn't go back to the old behavior, + # which would be awkward. def test_dot_notation_in_message_path( self, app: Flask, @@ -39,12 +43,14 @@ class TestDotNotation(BaseTest): ProcessInstanceService.complete_form_task(processor, user_task, form_data, process_instance.process_initiator, human_task) expected = { - "contibutorName": "Elizabeth", - "contributorId": 100, - "invoiceId": 10001, - "invoiceAmount": "1000.00", - "dueDate": "09/30/2022", + "invoice.contibutorName": "Elizabeth", + "invoice.contributorId": 100, + "invoice.invoiceId": 10001, + "invoice.invoiceAmount": "1000.00", + "invoice.dueDate": "09/30/2022", } processor.do_engine_steps(save=True) - assert processor.get_data()["invoice"] == expected + actual_data = processor.get_data() + del actual_data["validate_only"] + assert actual_data == expected diff --git a/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_process_instance_service.py b/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_process_instance_service.py index 56095f190..a314b68c6 100644 --- a/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_process_instance_service.py +++ b/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_process_instance_service.py @@ -6,7 +6,7 @@ from typing import Any import pytest from flask.app import Flask -from SpiffWorkflow.bpmn.event import PendingBpmnEvent # type: ignore +from SpiffWorkflow.bpmn.util import PendingBpmnEvent # type: ignore from spiffworkflow_backend.services.process_instance_service import ProcessInstanceService from tests.spiffworkflow_backend.helpers.base_test import BaseTest