merged in save_to_task_data branch and pyl with some cleanup w/ burnettk

.github/workflows/docker_image_for_main_builds.yml

@@ -31,7 +31,7 @@ on:
   push:
     branches:
       - main
-      - feature/move_task_data_into_tables
+      - feature/use_tasks_as_logs
 
 jobs:
   create_frontend_docker_image:

spiffworkflow-backend/migrations/env.py

@@ -1,5 +1,3 @@
-from __future__ import with_statement
-
 import logging
 from logging.config import fileConfig
 

spiffworkflow-backend/poetry.lock

@@ -1894,8 +1894,8 @@ lxml = "*"
 [package.source]
 type = "git"
 url = "https://github.com/sartography/SpiffWorkflow"
-reference = "6cad2981712bb61eca23af1adfafce02d3277cb9"
+reference = "main"
-resolved_reference = "6cad2981712bb61eca23af1adfafce02d3277cb9"
+resolved_reference = "f162aac43af3af18d1a55186aeccea154fb8b05d"
 
 [[package]]
 name = "SQLAlchemy"
@@ -2274,7 +2274,7 @@ testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools"
 [metadata]
 lock-version = "1.1"
 python-versions = ">=3.9,<3.12"
-content-hash = "253dc24203f175f363158329b0303c11044bc1bb400b17189658251cb37029f7"
+content-hash = "b9ea32912509637f1378d060771de7548d93953aa3db12d6a48098f7dc15205f"
 
 [metadata.files]
 alabaster = [

spiffworkflow-backend/pyproject.toml

@@ -27,8 +27,8 @@ flask-marshmallow = "*"
 flask-migrate = "*"
 flask-restful = "*"
 werkzeug = "*"
-# SpiffWorkflow = {git = "https://github.com/sartography/SpiffWorkflow", rev = "f162aac43af3af18d1a55186aeccea154fb8b05d"}
+SpiffWorkflow = {git = "https://github.com/sartography/SpiffWorkflow", rev = "main"}
-SpiffWorkflow = {git = "https://github.com/sartography/SpiffWorkflow", rev = "6cad2981712bb61eca23af1adfafce02d3277cb9"}
+# SpiffWorkflow = {git = "https://github.com/sartography/SpiffWorkflow", rev = "6cad2981712bb61eca23af1adfafce02d3277cb9"}
 # SpiffWorkflow = {develop = true, path = "../SpiffWorkflow" }
 sentry-sdk = "^1.10"
 sphinx-autoapi = "^2.0"

spiffworkflow-backend/src/spiffworkflow_backend/models/human_task.py

@@ -11,7 +11,8 @@ from spiffworkflow_backend.models.db import db
 from spiffworkflow_backend.models.db import SpiffworkflowBaseDBModel
 from spiffworkflow_backend.models.group import GroupModel
 from spiffworkflow_backend.models.process_instance import ProcessInstanceModel
-from spiffworkflow_backend.models.task import Task, TaskModel
+from spiffworkflow_backend.models.task import Task
+from spiffworkflow_backend.models.task import TaskModel
 from spiffworkflow_backend.models.user import UserModel
 
 

spiffworkflow-backend/src/spiffworkflow_backend/routes/process_instances_controller.py

@@ -42,7 +42,8 @@ from spiffworkflow_backend.models.spec_reference import SpecReferenceCache
 from spiffworkflow_backend.models.spec_reference import SpecReferenceNotFoundError
 from spiffworkflow_backend.models.spiff_logging import SpiffLoggingModel
 from spiffworkflow_backend.models.spiff_step_details import SpiffStepDetailsModel
-from spiffworkflow_backend.models.task import Task, TaskModel
+from spiffworkflow_backend.models.task import Task
+from spiffworkflow_backend.models.task import TaskModel
 from spiffworkflow_backend.models.user import UserModel
 from spiffworkflow_backend.routes.process_api_blueprint import (
     _find_process_instance_by_id_or_raise,
@@ -236,9 +237,7 @@ def process_instance_log_list(
     logs = (
         log_query.order_by(TaskModel.end_in_seconds.desc())  # type: ignore
         .outerjoin(HumanTaskModel, HumanTaskModel.task_model_id == TaskModel.id)
-        .outerjoin(
+        .outerjoin(UserModel, UserModel.id == HumanTaskModel.completed_by_user_id)
-            UserModel, UserModel.id == HumanTaskModel.completed_by_user_id
-        )
         .add_columns(
             UserModel.username,
         )

spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py

@@ -225,7 +225,6 @@ class NonTaskDataBasedScriptEngineEnvironment(BasePythonScriptEngineEnvironment)
     ) -> None:
         # TODO: once integrated look at the tests that fail without Box
         # context is task.data
-        # import pdb; pdb.set_trace()
         Box.convert_to_box(context)
         self.state.update(self.globals)
         self.state.update(external_methods or {})
@@ -235,20 +234,15 @@ class NonTaskDataBasedScriptEngineEnvironment(BasePythonScriptEngineEnvironment)
         finally:
             # since the task data is not directly mutated when the script executes, need to determine which keys
             # have been deleted from the environment and remove them from task data if present.
-            # import pdb; pdb.set_trace()
             context_keys_to_drop = context.keys() - self.state.keys()
 
-            # import pdb; pdb.set_trace()
             for key_to_drop in context_keys_to_drop:
                 context.pop(key_to_drop)
-            # import pdb; pdb.set_trace()
 
             self.state = self.user_defined_state(external_methods)
 
-            # import pdb; pdb.set_trace()
             # the task data needs to be updated with the current state so data references can be resolved properly.
             # the state will be removed later once the task is completed.
-            import pdb; pdb.set_trace()
             context.update(self.state)
 
     def user_defined_state(self, external_methods: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
@@ -291,7 +285,12 @@ class NonTaskDataBasedScriptEngineEnvironment(BasePythonScriptEngineEnvironment)
                 self.state[result_variable] = task.data.pop(result_variable)
 
 
-class CustomScriptEngineEnvironment(NonTaskDataBasedScriptEngineEnvironment):
+# SpiffWorkflow at revision f162aac43af3af18d1a55186aeccea154fb8b05d runs script tasks on ready
+# which means that our will_complete_task hook does not have the correct env state when it runs
+# so save everything to task data for now until we can figure out a better way to hook into that.
+# Revision 6cad2981712bb61eca23af1adfafce02d3277cb9 is the last revision that can run with this.
+# class CustomScriptEngineEnvironment(NonTaskDataBasedScriptEngineEnvironment):
+class CustomScriptEngineEnvironment(BoxedTaskDataBasedScriptEngineEnvironment):
     pass
 
 
@@ -1559,7 +1558,7 @@ class ProcessInstanceProcessor:
             serializer=self._serializer,
             process_instance=self.process_instance_model,
             bpmn_definition_to_task_definitions_mappings=self.bpmn_definition_to_task_definitions_mappings,
-            script_engine=self._script_engine,
+            # script_engine=self._script_engine,
         )
 
         if execution_strategy_name is None:

spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_report_service.py

@@ -404,10 +404,10 @@ class ProcessInstanceReportService:
             )
 
         if report_filter.process_initiator_username is not None:
-            user = UserModel.query.filter_by(username=report_filter.process_initiator_username).first()
+            initiator = UserModel.query.filter_by(username=report_filter.process_initiator_username).first()
             process_initiator_id = -1
-            if user:
+            if initiator:
-                process_initiator_id = user.id
+                process_initiator_id = initiator.id
             process_instance_query = process_instance_query.filter_by(process_initiator_id=process_initiator_id)
 
         if (

spiffworkflow-backend/src/spiffworkflow_backend/services/task_service.py

@@ -171,12 +171,12 @@ class TaskService:
         tasks = bpmn_process_dict.pop("tasks")
         bpmn_process_data_dict = bpmn_process_dict.pop("data")
 
-        if 'subprocesses' in bpmn_process_dict:
+        if "subprocesses" in bpmn_process_dict:
-            bpmn_process_dict.pop('subprocesses')
+            bpmn_process_dict.pop("subprocesses")
-        if 'spec' in bpmn_process_dict:
+        if "spec" in bpmn_process_dict:
-            bpmn_process_dict.pop('spec')
+            bpmn_process_dict.pop("spec")
-        if 'subprocess_specs' in bpmn_process_dict:
+        if "subprocess_specs" in bpmn_process_dict:
-            bpmn_process_dict.pop('subprocess_specs')
+            bpmn_process_dict.pop("subprocess_specs")
 
         new_task_models = {}
         new_json_data_dicts: dict[str, JsonDataDict] = {}

spiffworkflow-backend/src/spiffworkflow_backend/services/workflow_execution_service.py

@@ -58,13 +58,13 @@ class TaskModelSavingDelegate(EngineStepDelegate):
         serializer: BpmnWorkflowSerializer,
         process_instance: ProcessInstanceModel,
         bpmn_definition_to_task_definitions_mappings: dict,
-        script_engine,
+        # script_engine,
         secondary_engine_step_delegate: Optional[EngineStepDelegate] = None,
     ) -> None:
         self.secondary_engine_step_delegate = secondary_engine_step_delegate
         self.process_instance = process_instance
         self.bpmn_definition_to_task_definitions_mappings = bpmn_definition_to_task_definitions_mappings
-        self.script_engine = script_engine
+        # self.script_engine = script_engine
 
         self.current_task_model: Optional[TaskModel] = None
         self.current_task_start_in_seconds: Optional[float] = None
@@ -106,8 +106,8 @@ class TaskModelSavingDelegate(EngineStepDelegate):
     def did_complete_task(self, spiff_task: SpiffTask) -> None:
         # if self.current_task_model and self.should_update_task_model():
         if self.should_update_task_model():
-            if spiff_task.task_spec.name == 'top_level_script':
+            # if spiff_task.task_spec.name == 'top_level_script':
-                import pdb; pdb.set_trace()
+            # import pdb; pdb.set_trace()
             # spiff_task.workflow.script_engine.environment.revise_state_with_task_data(spiff_task)
             _bpmn_process, task_model, new_task_models, new_json_data_dicts = (
                 TaskService.find_or_create_task_model_from_spiff_task(

spiffworkflow-backend/tests/spiffworkflow_backend/helpers/example_data.py

@@ -41,7 +41,7 @@ class ExampleDataLoader:
 
         bpmn_file_name_with_extension = bpmn_file_name
         if not bpmn_file_name_with_extension:
-            bpmn_file_name_with_extension = process_model_id
+            bpmn_file_name_with_extension = os.path.basename(process_model_id)
 
         if not bpmn_file_name_with_extension.endswith(".bpmn"):
             bpmn_file_name_with_extension += ".bpmn"
@@ -65,7 +65,7 @@ class ExampleDataLoader:
             file_name_matcher,
         )
 
-        files = glob.glob(file_glob)
+        files = sorted(glob.glob(file_glob))
         for file_path in files:
             if os.path.isdir(file_path):
                 continue  # Don't try to process sub directories

spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_process_instance_processor.py

@@ -344,7 +344,9 @@ class TestProcessInstanceProcessor(BaseTest):
                 task_definition = task.task_definition
                 assert task_definition.bpmn_identifier == spiff_task_name
                 assert task_definition.bpmn_process_definition.bpmn_identifier == bpmn_process_identifier
-                assert task.python_env_data() == expected_python_env_data, f"{base_failure_message} Expected: {expected_python_env_data}. Received: {task.python_env_data()}"
+                message = f"{base_failure_message} Expected: {expected_python_env_data}. Received: {task.json_data()}"
+                # assert task.python_env_data() == expected_python_env_data, message
+                assert task.json_data() == expected_python_env_data, message
                 spiff_tasks_checked_once.append(spiff_task.task_spec.name)
 
         all_spiff_tasks = processor_final.bpmn_process_instance.get_tasks()

spiffworkflow-frontend/src/routes/ProcessInstanceLogList.tsx

@@ -29,6 +29,11 @@ export default function ProcessInstanceLogList({ variant }: OwnProps) {
   }
 
   useEffect(() => {
+    // Clear out any previous results to avoid a "flicker" effect where columns
+    // are updated above the incorrect data.
+    setProcessInstanceLogs([]);
+    setPagination(null);
+
     const setProcessInstanceLogListFromResult = (result: any) => {
       setProcessInstanceLogs(result.results);
       setPagination(result.pagination);