Feature/onboarding ephemeral (#442)

* The onboarding controller should not save the process model to the database.  It creates a pile of pointless noise.  So just cleaning that up.

* run_pyl

* assure we can handle user tasks if they happen during on-boarding, while keeping the list of processes clean.
Lots of weird stuff getting run_pyl going.

* pyl

---------

Co-authored-by: burnettk <burnettk@users.noreply.github.com>

spiffworkflow-backend/src/spiffworkflow_backend/routes/onboarding_controller.py

@@ -1,39 +1,47 @@
 """APIs for dealing with process groups, process models, and process instances."""
-from contextlib import suppress
 
 from flask import make_response
 from flask.wrappers import Response
+from SpiffWorkflow.exceptions import WorkflowException  # type: ignore
 
 from spiffworkflow_backend import db
-from spiffworkflow_backend.models.task import TaskModel
+from spiffworkflow_backend.exceptions.api_error import ApiError
 from spiffworkflow_backend.routes.process_instances_controller import _process_instance_start
 from spiffworkflow_backend.services.jinja_service import JinjaService
 
 
 def get_onboarding() -> Response:
-    result = {}
+    result: dict = {}
 
-    with suppress(Exception):
+    try:
         process_instance, processor = _process_instance_start("site-administration/onboarding")
-
+    except ApiError:
+        # The process doesn't exist, so bail out without an error
+        return make_response(result, 200)
+    try:
+        processor.do_engine_steps(save=True, execution_strategy_name="greedy")  # type: ignore
         if processor is not None:
-            if process_instance.status == "complete":
+            bpmn_process = processor.bpmn_process_instance
-                workflow_data = processor.bpmn_process_instance.data
+            if bpmn_process.is_completed():
+                workflow_data = bpmn_process.data
                 result = workflow_data.get("onboarding", {})
-            elif process_instance.status == "user_input_required" and len(process_instance.active_human_tasks) > 0:
+                # Delete the process instance, we don't need to keep this around if no users tasks were created.
+                db.session.delete(process_instance)
+                db.session.flush()  # Clear it out BEFORE returning.
+            elif len(bpmn_process.get_ready_user_tasks()) > 0:
+                process_instance.persistence_level = "full"
+                processor.save()
                 result = {
                     "type": "user_input_required",
                     "process_instance_id": process_instance.id,
                 }
-
             task = processor.next_task()
-            db.session.flush()
             if task:
-                task_model: TaskModel | None = TaskModel.query.filter_by(
+                result["task_id"] = task.id
-                    guid=str(task.id), process_instance_id=process_instance.id
+                result["instructions"] = JinjaService.render_instructions_for_end_user(task)
-                ).first()
+    except WorkflowException as e:
-                if task_model is not None:
+        raise ApiError.from_workflow_exception("onboard_failed", "Error building onboarding message", e) from e
-                    result["task_id"] = task_model.guid
+    except Exception as e:
-                    result["instructions"] = JinjaService.render_instructions_for_end_user(task_model)
+        raise ApiError("onboard_failed", "Error building onboarding message") from e
 
     return make_response(result, 200)

spiffworkflow-backend/src/spiffworkflow_backend/services/jinja_service.py

@@ -3,6 +3,8 @@ from sys import exc_info
 
 import jinja2
 from jinja2 import TemplateSyntaxError
+from SpiffWorkflow.bpmn.exceptions import WorkflowTaskException  # type: ignore
+from SpiffWorkflow.task import Task as SpiffTask  # type: ignore
 from spiffworkflow_backend.exceptions.api_error import ApiError
 from spiffworkflow_backend.models.task import TaskModel  # noqa: F401
 from spiffworkflow_backend.services.task_service import TaskModelError
@@ -35,14 +37,17 @@ class JinjaHelpers:
 
 class JinjaService:
     @classmethod
-    def render_instructions_for_end_user(cls, task_model: TaskModel, extensions: dict | None = None) -> str:
+    def render_instructions_for_end_user(cls, task: TaskModel | SpiffTask, extensions: dict | None = None) -> str:
         """Assure any instructions for end user are processed for jinja syntax."""
         if extensions is None:
-            extensions = TaskService.get_extensions_from_task_model(task_model)
+            if isinstance(task, TaskModel):
+                extensions = TaskService.get_extensions_from_task_model(task)
+            else:
+                extensions = task.task_spec.extensions
         if extensions and "instructionsForEndUser" in extensions:
             if extensions["instructionsForEndUser"]:
                 try:
-                    instructions = cls.render_jinja_template(extensions["instructionsForEndUser"], task_model)
+                    instructions = cls.render_jinja_template(extensions["instructionsForEndUser"], task)
                     extensions["instructionsForEndUser"] = instructions
                     return instructions
                 except TaskModelError as wfe:
@@ -51,14 +56,21 @@ class JinjaService:
         return ""
 
     @classmethod
-    def render_jinja_template(cls, unprocessed_template: str, task_model: TaskModel) -> str:
+    def render_jinja_template(cls, unprocessed_template: str, task: TaskModel | SpiffTask) -> str:
         jinja_environment = jinja2.Environment(autoescape=True, lstrip_blocks=True, trim_blocks=True)
         jinja_environment.filters.update(JinjaHelpers.get_helper_mapping())
         try:
             template = jinja_environment.from_string(unprocessed_template)
-            return template.render(**(task_model.get_data()), **JinjaHelpers.get_helper_mapping())
+            if isinstance(task, TaskModel):
+                data = task.get_data()
+            else:
+                data = task.data
+            return template.render(**data, **JinjaHelpers.get_helper_mapping())
         except jinja2.exceptions.TemplateError as template_error:
-            wfe = TaskModelError(str(template_error), task_model=task_model, exception=template_error)
+            if isinstance(task, TaskModel):
+                wfe = TaskModelError(str(template_error), task_model=task, exception=template_error)
+            else:
+                wfe = WorkflowTaskException(str(template_error), task=task, exception=template_error)
             if isinstance(template_error, TemplateSyntaxError):
                 wfe.line_number = template_error.lineno
                 wfe.error_line = template_error.source.split("\n")[template_error.lineno - 1]
@@ -66,7 +78,10 @@ class JinjaService:
             raise wfe from template_error
         except Exception as error:
             _type, _value, tb = exc_info()
-            wfe = TaskModelError(str(error), task_model=task_model, exception=error)
+            if isinstance(task, TaskModel):
+                wfe = TaskModelError(str(error), task_model=task, exception=error)
+            else:
+                wfe = WorkflowTaskException(str(error), task=task, exception=error)
             while tb:
                 if tb.tb_frame.f_code.co_filename == "<template>":
                     wfe.line_number = tb.tb_lineno

spiffworkflow-backend/tests/data/onboarding_with_user_task/onboarding_with_user_task.bpmn

@@ -0,0 +1,95 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<bpmn:definitions xmlns:bpmn="http://www.omg.org/spec/BPMN/20100524/MODEL" xmlns:bpmndi="http://www.omg.org/spec/BPMN/20100524/DI" xmlns:dc="http://www.omg.org/spec/DD/20100524/DC" xmlns:spiffworkflow="http://spiffworkflow.org/bpmn/schema/1.0/core" xmlns:di="http://www.omg.org/spec/DD/20100524/DI" id="Definitions_96f6665" targetNamespace="http://bpmn.io/schema/bpmn" exporter="Camunda Modeler" exporterVersion="3.0.0-dev">
+  <bpmn:process id="Process_WithForm" name="Process With Form" isExecutable="true">
+    <bpmn:startEvent id="StartEvent_1">
+      <bpmn:outgoing>Flow_0smvjir</bpmn:outgoing>
+    </bpmn:startEvent>
+    <bpmn:sequenceFlow id="Flow_0smvjir" sourceRef="StartEvent_1" targetRef="Activity_SimpleForm" />
+    <bpmn:endEvent id="Event_00xci7j">
+      <bpmn:incoming>Flow_1scft9v</bpmn:incoming>
+    </bpmn:endEvent>
+    <bpmn:manualTask id="Activity_1cscoeg" name="DisplayInfo">
+      <bpmn:extensionElements>
+        <spiffworkflow:instructionsForEndUser>Hello {{ name }}
+Department: {{ department }}
+</spiffworkflow:instructionsForEndUser>
+        <spiffworkflow:postScript>user_completing_task = get_last_user_completing_task("Process_WithForm", "Activity_SimpleForm")</spiffworkflow:postScript>
+      </bpmn:extensionElements>
+      <bpmn:incoming>Flow_028o7v5</bpmn:incoming>
+      <bpmn:outgoing>Flow_18ytjgo</bpmn:outgoing>
+    </bpmn:manualTask>
+    <bpmn:userTask id="Activity_SimpleForm" name="Simple Form">
+      <bpmn:extensionElements>
+        <spiffworkflow:properties>
+          <spiffworkflow:property name="formJsonSchemaFilename" value="simple_form.json" />
+          <spiffworkflow:property name="formUiSchemaFilename" value="simple_form_ui.json" />
+        </spiffworkflow:properties>
+        <spiffworkflow:postScript>process_initiator_user = get_process_initiator_user()</spiffworkflow:postScript>
+      </bpmn:extensionElements>
+      <bpmn:incoming>Flow_0smvjir</bpmn:incoming>
+      <bpmn:outgoing>Flow_163ufsx</bpmn:outgoing>
+    </bpmn:userTask>
+    <bpmn:intermediateThrowEvent id="completed_form" name="Completed Form">
+      <bpmn:incoming>Flow_163ufsx</bpmn:incoming>
+      <bpmn:outgoing>Flow_028o7v5</bpmn:outgoing>
+    </bpmn:intermediateThrowEvent>
+    <bpmn:sequenceFlow id="Flow_163ufsx" sourceRef="Activity_SimpleForm" targetRef="completed_form" />
+    <bpmn:intermediateThrowEvent id="completed_manual_task" name="Completed Manual Task">
+      <bpmn:incoming>Flow_18ytjgo</bpmn:incoming>
+      <bpmn:outgoing>Flow_1scft9v</bpmn:outgoing>
+    </bpmn:intermediateThrowEvent>
+    <bpmn:sequenceFlow id="Flow_18ytjgo" sourceRef="Activity_1cscoeg" targetRef="completed_manual_task" />
+    <bpmn:sequenceFlow id="Flow_028o7v5" sourceRef="completed_form" targetRef="Activity_1cscoeg" />
+    <bpmn:sequenceFlow id="Flow_1scft9v" sourceRef="completed_manual_task" targetRef="Event_00xci7j" />
+  </bpmn:process>
+  <bpmndi:BPMNDiagram id="BPMNDiagram_1">
+    <bpmndi:BPMNPlane id="BPMNPlane_1" bpmnElement="Process_WithForm">
+      <bpmndi:BPMNShape id="_BPMNShape_StartEvent_2" bpmnElement="StartEvent_1">
+        <dc:Bounds x="179" y="159" width="36" height="36" />
+      </bpmndi:BPMNShape>
+      <bpmndi:BPMNShape id="Activity_0x5k4l1_di" bpmnElement="Activity_SimpleForm">
+        <dc:Bounds x="270" y="137" width="100" height="80" />
+        <bpmndi:BPMNLabel />
+      </bpmndi:BPMNShape>
+      <bpmndi:BPMNShape id="Activity_00g930h_di" bpmnElement="Activity_1cscoeg">
+        <dc:Bounds x="510" y="137" width="100" height="80" />
+        <bpmndi:BPMNLabel />
+      </bpmndi:BPMNShape>
+      <bpmndi:BPMNShape id="Event_00xci7j_di" bpmnElement="Event_00xci7j">
+        <dc:Bounds x="722" y="159" width="36" height="36" />
+      </bpmndi:BPMNShape>
+      <bpmndi:BPMNShape id="Event_0pi2wuv_di" bpmnElement="completed_form">
+        <dc:Bounds x="432" y="159" width="36" height="36" />
+        <bpmndi:BPMNLabel>
+          <dc:Bounds x="409" y="202" width="82" height="14" />
+        </bpmndi:BPMNLabel>
+      </bpmndi:BPMNShape>
+      <bpmndi:BPMNShape id="Event_071q0rf_di" bpmnElement="completed_manual_task">
+        <dc:Bounds x="662" y="159" width="36" height="36" />
+        <bpmndi:BPMNLabel>
+          <dc:Bounds x="649" y="202" width="63" height="27" />
+        </bpmndi:BPMNLabel>
+      </bpmndi:BPMNShape>
+      <bpmndi:BPMNEdge id="Flow_0smvjir_di" bpmnElement="Flow_0smvjir">
+        <di:waypoint x="215" y="177" />
+        <di:waypoint x="270" y="177" />
+      </bpmndi:BPMNEdge>
+      <bpmndi:BPMNEdge id="Flow_163ufsx_di" bpmnElement="Flow_163ufsx">
+        <di:waypoint x="370" y="177" />
+        <di:waypoint x="432" y="177" />
+      </bpmndi:BPMNEdge>
+      <bpmndi:BPMNEdge id="Flow_18ytjgo_di" bpmnElement="Flow_18ytjgo">
+        <di:waypoint x="610" y="177" />
+        <di:waypoint x="662" y="177" />
+      </bpmndi:BPMNEdge>
+      <bpmndi:BPMNEdge id="Flow_028o7v5_di" bpmnElement="Flow_028o7v5">
+        <di:waypoint x="468" y="177" />
+        <di:waypoint x="510" y="177" />
+      </bpmndi:BPMNEdge>
+      <bpmndi:BPMNEdge id="Flow_1scft9v_di" bpmnElement="Flow_1scft9v">
+        <di:waypoint x="698" y="177" />
+        <di:waypoint x="722" y="177" />
+      </bpmndi:BPMNEdge>
+    </bpmndi:BPMNPlane>
+  </bpmndi:BPMNDiagram>
+</bpmn:definitions>

spiffworkflow-backend/tests/data/onboarding_with_user_task/simple_form.json

@@ -0,0 +1,18 @@
+{
+  "title": "Simple form",
+  "description": "A simple form example.",
+  "type": "object",
+  "required": ["name"],
+  "properties": {
+    "name": {
+      "type": "string",
+      "title": "Name",
+      "default": "World"
+    },
+    "department": {
+      "type": "string",
+      "title": "Department",
+      "enum": ["Finance", "HR", "IT"]
+    }
+  }
+}

spiffworkflow-backend/tests/data/onboarding_with_user_task/simple_form_ui.json

@@ -0,0 +1,11 @@
+{
+  "name": {
+    "ui:title": "Name",
+    "ui:description": "(Your name)"
+  },
+  "department": {
+    "ui:title": "Department",
+    "ui:description": "(Your department)"
+  },
+  "ui:order": ["name", "department"]
+}

spiffworkflow-backend/tests/spiffworkflow_backend/integration/test_onboarding.py

@@ -1,5 +1,6 @@
 from flask import Flask
 from flask.testing import FlaskClient
+from spiffworkflow_backend.models.process_instance import ProcessInstanceModel
 from spiffworkflow_backend.models.user import UserModel
 
 from tests.spiffworkflow_backend.helpers.base_test import BaseTest
@@ -21,16 +22,10 @@ class TestOnboarding(BaseTest):
         assert results.status_code == 200
         assert results.json == {}
 
-    def test_returns_onboarding_if_onboarding_model(
+    def set_up_onboarding(self, client: FlaskClient, with_super_admin_user: UserModel, file_location: str) -> None:
-        self,
-        app: Flask,
-        client: FlaskClient,
-        with_db_and_bpmn_file_cleanup: None,
-        with_super_admin_user: UserModel,
-    ) -> None:
         process_group_id = "site-administration"
         process_model_id = "onboarding"
-        bpmn_file_location = "onboarding"
+        bpmn_file_location = file_location
         self.create_group_and_model_with_bpmn(
             client,
             with_super_admin_user,
@@ -39,6 +34,15 @@ class TestOnboarding(BaseTest):
             bpmn_file_location=bpmn_file_location,
         )
 
+    def test_returns_onboarding_if_onboarding_model(
+        self,
+        app: Flask,
+        client: FlaskClient,
+        with_db_and_bpmn_file_cleanup: None,
+        with_super_admin_user: UserModel,
+    ) -> None:
+        self.set_up_onboarding(client, with_super_admin_user, "onboarding")
+
         results = client.get(
             "/v1.0/onboarding",
             headers=self.logged_in_headers(with_super_admin_user),
@@ -50,3 +54,27 @@ class TestOnboarding(BaseTest):
         assert results.json["value"] == "my_tasks"
         assert results.json["instructions"] == ""
         assert results.json["task_id"] is not None
+
+        # Assure no residual process model is left behind if it executes and completes without additinal user tasks
+        assert len(ProcessInstanceModel.query.all()) == 0
+
+    def test_persists_if_user_task_encountered(
+        self,
+        app: Flask,
+        client: FlaskClient,
+        with_db_and_bpmn_file_cleanup: None,
+        with_super_admin_user: UserModel,
+    ) -> None:
+        self.set_up_onboarding(client, with_super_admin_user, "onboarding_with_user_task")
+        results = client.get(
+            "/v1.0/onboarding",
+            headers=self.logged_in_headers(with_super_admin_user),
+        )
+        assert results.status_code == 200
+        assert len(results.json.keys()) == 4
+        assert results.json["type"] == "user_input_required"
+        assert results.json["process_instance_id"] is not None
+        instance = ProcessInstanceModel.query.filter(
+            ProcessInstanceModel.id == results.json["process_instance_id"]
+        ).first()
+        assert instance is not None

spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_jinja_service.py

@@ -42,3 +42,21 @@ class TestJinjaService(BaseTest):
             "from_script_task": "Sanitized \\| from \\| script \\| task",
         }
         assert task_model.get_data() == expected_task_data
+
+    def test_can_render_directly_from_spiff_task(self, app: Flask, with_db_and_bpmn_file_cleanup: None) -> None:
+        process_model = load_test_spec(
+            process_model_id="test_group/manual-task-with-sanitized-markdown",
+            process_model_source_directory="manual-task-with-sanitized-markdown",
+        )
+        process_instance = self.create_process_instance_from_process_model(process_model=process_model)
+        processor = ProcessInstanceProcessor(process_instance)
+        processor.do_engine_steps(save=True)
+
+        JinjaService.render_instructions_for_end_user(processor.get_ready_user_tasks()[0])
+        "\n".join(
+            [
+                r"* From Filter: Sanitized \| from \| filter",
+                r"* From Method Call: Sanitized \| from \| method \| call",
+                r"* From ScriptTask: Sanitized \| from \| script \| task",
+            ]
+        )

spiffworkflow-frontend/src/routes/OnboardingView.tsx

@@ -1,6 +1,6 @@
 import React, { useEffect, useState } from 'react';
 import MDEditor from '@uiw/react-md-editor';
-import { useLocation } from 'react-router-dom';
+import { useLocation, useNavigate } from 'react-router-dom';
 import HttpService from '../services/HttpService';
 import { Onboarding } from '../interfaces';
 import { objectIsEmpty } from '../helpers';
@@ -8,8 +8,7 @@ import { objectIsEmpty } from '../helpers';
 export default function OnboardingView() {
   const [onboarding, setOnboarding] = useState<Onboarding | null>(null);
   const location = useLocation();
-
+  const navigate = useNavigate();
-  // const navigate = useNavigate();
 
   useEffect(() => {
     HttpService.makeCallToBackend({
@@ -24,6 +23,15 @@ export default function OnboardingView() {
     }
 
     if (
+      onboarding &&
+      onboarding.type === 'user_input_required' &&
+      onboarding.process_instance_id &&
+      onboarding.task_id
+    ) {
+      navigate(
+        `/tasks/${onboarding.process_instance_id}/${onboarding.task_id}`
+      );
+    } else if (
       onboarding &&
       !objectIsEmpty(onboarding) &&
       onboarding.instructions.length > 0
@@ -37,25 +45,6 @@ export default function OnboardingView() {
           />
         </div>
       );
-      /*
-      if (onboarding.type === 'default_view') {
-        if (onboarding.value === 'my_tasks') {
-          return <MyTasks />;
-        }
-      } else if (
-          onboarding.type === 'user_input_required'
-      ) {
-        console.log("onboarding");
-      } else if (
-        onboarding.type === 'user_input_required' &&
-        onboarding.process_instance_id &&
-        onboarding.task_id
-      ) {
-        navigate(
-          `/tasks/${onboarding.process_instance_id}/${onboarding.task_id}`
-        );
-      }
-      */
     }
     return null;
   };