Support non task data environment, step 1 (#1745)

spiffworkflow-backend/dev.docker-compose.yml

@@ -13,5 +13,8 @@ services:
       SPIFFWORKFLOW_BACKEND_ENV: "${SPIFFWORKFLOW_BACKEND_ENV:-local_development}"
       SPIFFWORKFLOW_BACKEND_LOAD_FIXTURE_DATA: ""
       XDG_CACHE_HOME: "/app/.cache"
+    env_file:
+      - path: .env
+        required: false
     volumes:
       - ./spiffworkflow-backend:/app

spiffworkflow-backend/src/spiffworkflow_backend/config/default.py

@@ -234,7 +234,7 @@ config_from_env("SPIFFWORKFLOW_BACKEND_DEBUG_TASK_CONSISTENCY", default=False)
 # we load the CustomBpmnScriptEngine at import time, where we do not have access to current_app,
 # so instead of using config, we use os.environ directly over there.
 # config_from_env("SPIFFWORKFLOW_BACKEND_USE_RESTRICTED_SCRIPT_ENGINE", default=True)
-
+# config_from_env("SPIFFWORKFLOW_BACKEND_USE_NON_TASK_DATA_BASED_SCRIPT_ENGINE_ENVIRONMENT", default=False)
 
 # adds the ProxyFix to Flask on http by processing the 'X-Forwarded-Proto' header
 # to make SpiffWorkflow aware that it should return https for the server urls etc rather than http.

spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py

@@ -162,7 +162,33 @@ class MissingProcessInfoError(Exception):
     pass
 
 
-class TaskDataBasedScriptEngineEnvironment(TaskDataEnvironment):  # type: ignore
+class BaseCustomScriptEngineEnvironment(BasePythonScriptEngineEnvironment):  # type: ignore
+    def user_defined_state(self, external_context: dict[str, Any] | None = None) -> dict[str, Any]:
+        return {}
+
+    def last_result(self) -> dict[str, Any]:
+        return dict(self._last_result.items())
+
+    def clear_state(self) -> None:
+        pass
+
+    def pop_state(self, data: dict[str, Any]) -> dict[str, Any]:
+        return {}
+
+    def preserve_state(self, bpmn_process_instance: BpmnWorkflow) -> None:
+        pass
+
+    def restore_state(self, bpmn_process_instance: BpmnWorkflow) -> None:
+        pass
+
+    def finalize_result(self, bpmn_process_instance: BpmnWorkflow) -> None:
+        pass
+
+    def revise_state_with_task_data(self, task: SpiffTask) -> None:
+        pass
+
+
+class TaskDataBasedScriptEngineEnvironment(BaseCustomScriptEngineEnvironment, TaskDataEnvironment):  # type: ignore
     def __init__(self, environment_globals: dict[str, Any]):
         self._last_result: dict[str, Any] = {}
         self._non_user_defined_keys = {"__annotations__"}
@@ -181,29 +207,8 @@ class TaskDataBasedScriptEngineEnvironment(TaskDataEnvironment):  # type: ignore
         self._last_result = context
         return True
 
-    def user_defined_state(self, external_context: dict[str, Any] | None = None) -> dict[str, Any]:
-        return {}
 
-    def last_result(self) -> dict[str, Any]:
+class NonTaskDataBasedScriptEngineEnvironment(BaseCustomScriptEngineEnvironment):
-        return dict(self._last_result.items())
-
-    def clear_state(self) -> None:
-        pass
-
-    def preserve_state(self, bpmn_process_instance: BpmnWorkflow) -> None:
-        pass
-
-    def restore_state(self, bpmn_process_instance: BpmnWorkflow) -> None:
-        pass
-
-    def finalize_result(self, bpmn_process_instance: BpmnWorkflow) -> None:
-        pass
-
-    def revise_state_with_task_data(self, task: SpiffTask) -> None:
-        pass
-
-
-class NonTaskDataBasedScriptEngineEnvironment(BasePythonScriptEngineEnvironment):  # type: ignore
     PYTHON_ENVIRONMENT_STATE_KEY = "spiff__python_env_state"
 
     def __init__(self, environment_globals: dict[str, Any]):
@@ -263,6 +268,10 @@ class NonTaskDataBasedScriptEngineEnvironment(BasePythonScriptEngineEnvironment)
     def clear_state(self) -> None:
         self.state = {}
 
+    def pop_state(self, data: dict[str, Any]) -> dict[str, Any]:
+        key = self.PYTHON_ENVIRONMENT_STATE_KEY
+        return data.pop(key, {})  # type: ignore
+
     def preserve_state(self, bpmn_process_instance: BpmnWorkflow) -> None:
         key = self.PYTHON_ENVIRONMENT_STATE_KEY
         state = self.user_defined_state()
@@ -290,8 +299,13 @@ class NonTaskDataBasedScriptEngineEnvironment(BasePythonScriptEngineEnvironment)
                 self.state[result_variable] = task.data.pop(result_variable)
 
 
-class CustomScriptEngineEnvironment(TaskDataBasedScriptEngineEnvironment):
+class CustomScriptEngineEnvironment:
-    pass
+    @staticmethod
+    def create(environment_globals: dict[str, Any]) -> BaseCustomScriptEngineEnvironment:
+        if os.environ.get("SPIFFWORKFLOW_BACKEND_USE_NON_TASK_DATA_BASED_SCRIPT_ENGINE_ENVIRONMENT") == "true":
+            return NonTaskDataBasedScriptEngineEnvironment(environment_globals)
+
+        return TaskDataBasedScriptEngineEnvironment(environment_globals)
 
 
 class CustomBpmnScriptEngine(PythonScriptEngine):  # type: ignore
@@ -334,7 +348,7 @@ class CustomBpmnScriptEngine(PythonScriptEngine):  # type: ignore
             default_globals.update(safe_globals)
             default_globals["__builtins__"]["__import__"] = _import
 
-        environment = CustomScriptEngineEnvironment(default_globals)
+        environment = CustomScriptEngineEnvironment.create(default_globals)
         super().__init__(environment=environment)
 
     def __get_augment_methods(self, task: SpiffTask | None) -> dict[str, Callable]:
@@ -564,9 +578,6 @@ class ProcessInstanceProcessor:
         script_engine_to_use.environment.restore_state(bpmn_process_instance)
         bpmn_process_instance.script_engine = script_engine_to_use
 
-    def preserve_script_engine_state(self) -> None:
-        self._script_engine.environment.preserve_state(self.bpmn_process_instance)
-
     @classmethod
     def _update_bpmn_definition_mappings(
         cls,
@@ -1519,10 +1530,18 @@ class ProcessInstanceProcessor:
                 )
             )
 
-    def serialize(self) -> dict:
+    def serialize(self, serialize_script_engine_state: bool = True) -> dict:
         self.check_task_data_size()
-        self.preserve_script_engine_state()
+
-        return self._serializer.to_dict(self.bpmn_process_instance)  # type: ignore
+        if serialize_script_engine_state:
+            self._script_engine.environment.preserve_state(self.bpmn_process_instance)
+
+        result = self._serializer.to_dict(self.bpmn_process_instance)
+
+        if not serialize_script_engine_state and "data" in result:
+            self._script_engine.environment.pop_state(result["data"])
+
+        return result  # type: ignore
 
     def next_user_tasks(self) -> list[SpiffTask]:
         return self.bpmn_process_instance.get_tasks(state=TaskState.READY, manual=True)  # type: ignore

spiffworkflow-backend/src/spiffworkflow_backend/services/process_model_test_runner_service.py

@@ -149,7 +149,7 @@ class ProcessModelTestRunnerScriptEngine(PythonScriptEngine):  # type: ignore
         default_globals.update(safe_globals)
         default_globals["__builtins__"]["__import__"] = _import
 
-        environment = CustomScriptEngineEnvironment(default_globals)
+        environment = CustomScriptEngineEnvironment.create(default_globals)
         self.method_overrides = method_overrides
         super().__init__(environment=environment)
 

spiffworkflow-backend/src/spiffworkflow_backend/services/task_service.py

@@ -104,8 +104,6 @@ class TaskModelError(Exception):
 
 
 class TaskService:
-    PYTHON_ENVIRONMENT_STATE_KEY = "spiff__python_env_state"
-
     def __init__(
         self,
         process_instance: ProcessInstanceModel,

spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_process_instance_migrator.py

@@ -111,7 +111,7 @@ class TestProcessInstanceMigrator(BaseTest):
         processor = ProcessInstanceProcessor(
             process_instance, include_task_data_for_completed_tasks=True, include_completed_subprocesses=True
         )
-        bpmn_process_dict_version_4 = processor.serialize()
+        bpmn_process_dict_version_4 = processor.serialize(serialize_script_engine_state=False)
         self.round_last_state_change(bpmn_process_dict_version_4)
         self.round_last_state_change(bpmn_process_dict_version_4_from_spiff)
         assert bpmn_process_dict_version_4 == bpmn_process_dict_version_4_from_spiff
@@ -208,7 +208,7 @@ class TestProcessInstanceMigrator(BaseTest):
         processor = ProcessInstanceProcessor(
             process_instance, include_task_data_for_completed_tasks=True, include_completed_subprocesses=True
         )
-        bpmn_process_dict_version_3_after_import = processor.serialize()
+        bpmn_process_dict_version_3_after_import = processor.serialize(serialize_script_engine_state=False)
         self.round_last_state_change(bpmn_process_dict_before_import)
         self.round_last_state_change(bpmn_process_dict_version_3_after_import)
         assert bpmn_process_dict_version_3_after_import == bpmn_process_dict_before_import