From c00338e95124addc959cdc89698b18905a347c29 Mon Sep 17 00:00:00 2001 From: jbirddog <100367399+jbirddog@users.noreply.github.com> Date: Mon, 6 Feb 2023 15:25:49 -0500 Subject: [PATCH] Backend do_engine_steps performance improvements (#129) Co-authored-by: Dan --- .../realm_exports/spiffworkflow-realm.json | 2 +- .../services/logging_service.py | 2 +- .../services/process_instance_processor.py | 61 ++++++++++++++----- 3 files changed, 47 insertions(+), 18 deletions(-) diff --git a/spiffworkflow-backend/keycloak/realm_exports/spiffworkflow-realm.json b/spiffworkflow-backend/keycloak/realm_exports/spiffworkflow-realm.json index b4e2e08a..4e386398 100644 --- a/spiffworkflow-backend/keycloak/realm_exports/spiffworkflow-realm.json +++ b/spiffworkflow-backend/keycloak/realm_exports/spiffworkflow-realm.json @@ -3634,4 +3634,4 @@ "clientPolicies" : { "policies" : [ ] } -} \ No newline at end of file +} diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/services/logging_service.py b/spiffworkflow-backend/src/spiffworkflow_backend/services/logging_service.py index aba695b8..4b9a7318 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/services/logging_service.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/services/logging_service.py @@ -243,5 +243,5 @@ class DBHandler(logging.Handler): # so at some point we are going to insert logs. # we don't want to insert on every log, so we will insert every 100 logs, which is just about as fast as inserting # on every 1,000 logs. if we get deadlocks in the database, this can be changed to 1 in order to insert on every log. - if len(self.logs) % 1 == 0: + if len(self.logs) >= 100: self.bulk_insert_logs() diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py b/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py index 51601115..63c5a25e 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py @@ -149,7 +149,7 @@ class BoxedTaskDataBasedScriptEngineEnvironment(BoxedTaskDataEnvironment): # ty self._last_result = context def last_result(self) -> Dict[str, Any]: - return self._last_result + return {k: v for k, v in self._last_result.items()} def clear_state(self) -> None: pass @@ -226,7 +226,7 @@ class NonTaskDataBasedScriptEngineEnvironment(BasePythonScriptEngineEnvironment) } def last_result(self) -> Dict[str, Any]: - return self.state + return {k: v for k, v in self.state.items()} def clear_state(self) -> None: self.state = {} @@ -254,8 +254,13 @@ class NonTaskDataBasedScriptEngineEnvironment(BasePythonScriptEngineEnvironment) } task.data = {k: v for k, v in task.data.items() if k in task_data_keys_to_keep} + if hasattr(task.task_spec, "_result_variable"): + result_variable = task.task_spec._result_variable(task) + if result_variable in task.data: + self.state[result_variable] = task.data.pop(result_variable) -class CustomScriptEngineEnvironment(BoxedTaskDataBasedScriptEngineEnvironment): + +class CustomScriptEngineEnvironment(NonTaskDataBasedScriptEngineEnvironment): pass @@ -685,9 +690,13 @@ class ProcessInstanceProcessor: def spiff_step_details_mapping(self) -> dict: """SaveSpiffStepDetails.""" - bpmn_json = self.serialize() - wf_json = json.loads(bpmn_json) - task_json = {"tasks": wf_json["tasks"], "subprocesses": wf_json["subprocesses"]} + # bpmn_json = self.serialize() + # wf_json = json.loads(bpmn_json) + task_json: Dict[str, Any] = { + # "tasks": wf_json["tasks"], + # "subprocesses": wf_json["subprocesses"], + # "python_env": self._script_engine.environment.last_result(), + } return { "process_instance_id": self.process_instance_model.id, @@ -700,13 +709,7 @@ class ProcessInstanceProcessor: def spiff_step_details(self) -> SpiffStepDetailsModel: """SaveSpiffStepDetails.""" details_mapping = self.spiff_step_details_mapping() - details_model = SpiffStepDetailsModel( - process_instance_id=details_mapping["process_instance_id"], - spiff_step=details_mapping["spiff_step"], - task_json=details_mapping["task_json"], - timestamp=details_mapping["timestamp"], - # completed_by_user_id=details_mapping["completed_by_user_id"], - ) + details_model = SpiffStepDetailsModel(**details_mapping) return details_model def extract_metadata(self, process_model_info: ProcessModelInfo) -> None: @@ -1490,16 +1493,42 @@ class ProcessInstanceProcessor: """Do_engine_steps.""" step_details = [] + tasks_to_log = { + "BPMN Task", + "Script Task", + "Service Task" + # "End Event", + # "Default Start Event", + # "Exclusive Gateway", + # "End Join", + # "End Event", + # "Default Throwing Event", + # "Subprocess" + } + + def should_log(task: SpiffTask) -> bool: + if ( + task.task_spec.spec_type in tasks_to_log + and not task.task_spec.name.endswith(".EndJoin") + ): + return True + return False + + def will_complete_task(task: SpiffTask) -> None: + if should_log(task): + self.increment_spiff_step() + def did_complete_task(task: SpiffTask) -> None: - self._script_engine.environment.revise_state_with_task_data(task) - step_details.append(self.spiff_step_details_mapping()) + if should_log(task): + self._script_engine.environment.revise_state_with_task_data(task) + step_details.append(self.spiff_step_details_mapping()) try: self.bpmn_process_instance.refresh_waiting_tasks() self.bpmn_process_instance.do_engine_steps( exit_at=exit_at, - will_complete_task=lambda t: self.increment_spiff_step(), + will_complete_task=will_complete_task, did_complete_task=did_complete_task, )