added bpmn_name columns to definition tables and added test for simple logs as well

This commit is contained in:
jasquat 2023-03-17 10:00:59 -04:00
parent ed1b45c453
commit 8dc7c5fb2f
No known key found for this signature in database
10 changed files with 162 additions and 37 deletions

View File

@ -1,8 +1,8 @@
"""empty message
Revision ID: 8ee0f1c23cc7
Revision ID: 8dce75b80bfd
Revises:
Create Date: 2023-03-16 16:24:47.364768
Create Date: 2023-03-17 09:08:24.146736
"""
from alembic import op
@ -10,7 +10,7 @@ import sqlalchemy as sa
from sqlalchemy.dialects import mysql
# revision identifiers, used by Alembic.
revision = '8ee0f1c23cc7'
revision = '8dce75b80bfd'
down_revision = None
branch_labels = None
depends_on = None
@ -22,6 +22,7 @@ def upgrade():
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('hash', sa.String(length=255), nullable=False),
sa.Column('bpmn_identifier', sa.String(length=255), nullable=False),
sa.Column('bpmn_name', sa.String(length=255), nullable=True),
sa.Column('properties_json', sa.JSON(), nullable=False),
sa.Column('type', sa.String(length=32), nullable=True),
sa.Column('bpmn_version_control_type', sa.String(length=50), nullable=True),
@ -31,6 +32,7 @@ def upgrade():
sa.PrimaryKeyConstraint('id')
)
op.create_index(op.f('ix_bpmn_process_definition_bpmn_identifier'), 'bpmn_process_definition', ['bpmn_identifier'], unique=False)
op.create_index(op.f('ix_bpmn_process_definition_bpmn_name'), 'bpmn_process_definition', ['bpmn_name'], unique=False)
op.create_index(op.f('ix_bpmn_process_definition_hash'), 'bpmn_process_definition', ['hash'], unique=True)
op.create_table('correlation_property_cache',
sa.Column('id', sa.Integer(), nullable=False),
@ -187,6 +189,7 @@ def upgrade():
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('bpmn_process_definition_id', sa.Integer(), nullable=False),
sa.Column('bpmn_identifier', sa.String(length=255), nullable=False),
sa.Column('bpmn_name', sa.String(length=255), nullable=True),
sa.Column('properties_json', sa.JSON(), nullable=False),
sa.Column('typename', sa.String(length=255), nullable=False),
sa.Column('updated_at_in_seconds', sa.Integer(), nullable=True),
@ -196,6 +199,7 @@ def upgrade():
sa.UniqueConstraint('bpmn_process_definition_id', 'bpmn_identifier', name='task_definition_unique')
)
op.create_index(op.f('ix_task_definition_bpmn_identifier'), 'task_definition', ['bpmn_identifier'], unique=False)
op.create_index(op.f('ix_task_definition_bpmn_name'), 'task_definition', ['bpmn_name'], unique=False)
op.create_table('user_group_assignment',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('user_id', sa.Integer(), nullable=False),
@ -427,6 +431,7 @@ def downgrade():
op.drop_table('permission_assignment')
op.drop_table('user_group_assignment_waiting')
op.drop_table('user_group_assignment')
op.drop_index(op.f('ix_task_definition_bpmn_name'), table_name='task_definition')
op.drop_index(op.f('ix_task_definition_bpmn_identifier'), table_name='task_definition')
op.drop_table('task_definition')
op.drop_table('secret')
@ -453,6 +458,7 @@ def downgrade():
op.drop_table('group')
op.drop_table('correlation_property_cache')
op.drop_index(op.f('ix_bpmn_process_definition_hash'), table_name='bpmn_process_definition')
op.drop_index(op.f('ix_bpmn_process_definition_bpmn_name'), table_name='bpmn_process_definition')
op.drop_index(op.f('ix_bpmn_process_definition_bpmn_identifier'), table_name='bpmn_process_definition')
op.drop_table('bpmn_process_definition')
# ### end Alembic commands ###

View File

@ -21,6 +21,7 @@ class BpmnProcessDefinitionModel(SpiffworkflowBaseDBModel):
hash: str = db.Column(db.String(255), nullable=False, index=True, unique=True)
bpmn_identifier: str = db.Column(db.String(255), nullable=False, index=True)
bpmn_name: str = db.Column(db.String(255), nullable=True, index=True)
properties_json: dict = db.Column(db.JSON, nullable=False)

View File

@ -28,6 +28,8 @@ class TaskDefinitionModel(SpiffworkflowBaseDBModel):
bpmn_process_definition = relationship(BpmnProcessDefinitionModel)
bpmn_identifier: str = db.Column(db.String(255), nullable=False, index=True)
bpmn_name: str = db.Column(db.String(255), nullable=True, index=True)
properties_json: dict = db.Column(db.JSON, nullable=False)
typename: str = db.Column(db.String(255), nullable=False)

View File

@ -235,20 +235,41 @@ def process_instance_log_list(
# )
# .paginate(page=page, per_page=per_page, error_out=False)
# )
log_query = TaskModel.query.filter_by(process_instance_id=process_instance.id)
logs = (
log_query.order_by(TaskModel.end_in_seconds.desc()) # type: ignore
log_query = (
TaskModel.query.filter_by(process_instance_id=process_instance.id)
.join(TaskDefinitionModel, TaskDefinitionModel.id == TaskModel.task_definition_id)
.join(
BpmnProcessDefinitionModel, BpmnProcessDefinitionModel.id == TaskDefinitionModel.bpmn_process_definition_id
)
)
if not detailed:
log_query = log_query.filter(
# 1. this was the previous implementation, where we only show completed tasks and skipped tasks.
# maybe we want to iterate on this in the future (in a third tab under process instance logs?)
# or_(
# SpiffLoggingModel.message.in_(["State change to COMPLETED"]), # type: ignore
# SpiffLoggingModel.message.like("Skipped task %"), # type: ignore
# )
# 2. We included ["End Event", "Default Start Event"] along with Default Throwing Event, but feb 2023
# we decided to remove them, since they get really chatty when there are lots of subprocesses and call activities.
and_(
TaskModel.state.in_(["COMPLETED"]), # type: ignore
TaskDefinitionModel.typename.in_(["IntermediateThrowEvent"]), # type: ignore
)
)
logs = (
log_query.order_by(TaskModel.end_in_seconds.desc()) # type: ignore
.outerjoin(HumanTaskModel, HumanTaskModel.task_model_id == TaskModel.id)
.outerjoin(UserModel, UserModel.id == HumanTaskModel.completed_by_user_id)
.add_columns(
TaskModel.guid.label("spiff_task_guid"), # type: ignore
UserModel.username,
BpmnProcessDefinitionModel.bpmn_identifier.label("bpmn_process_definition_identifier"), # type: ignore
BpmnProcessDefinitionModel.bpmn_name.label("bpmn_process_definition_name"), # type: ignore
TaskDefinitionModel.bpmn_identifier.label("task_definition_identifier"), # type: ignore
TaskDefinitionModel.bpmn_name.label("task_definition_name"), # type: ignore
TaskDefinitionModel.typename.label("bpmn_type"), # type: ignore
)
.paginate(page=page, per_page=per_page, error_out=False)
)

View File

@ -996,6 +996,7 @@ class ProcessInstanceProcessor:
store_bpmn_definition_mappings: bool = False,
) -> BpmnProcessDefinitionModel:
process_bpmn_identifier = process_bpmn_properties["name"]
process_bpmn_name = process_bpmn_properties["description"]
new_hash_digest = sha256(json.dumps(process_bpmn_properties, sort_keys=True).encode("utf8")).hexdigest()
bpmn_process_definition: Optional[BpmnProcessDefinitionModel] = BpmnProcessDefinitionModel.query.filter_by(
hash=new_hash_digest
@ -1006,6 +1007,7 @@ class ProcessInstanceProcessor:
bpmn_process_definition = BpmnProcessDefinitionModel(
hash=new_hash_digest,
bpmn_identifier=process_bpmn_identifier,
bpmn_name=process_bpmn_name,
properties_json=process_bpmn_properties,
)
db.session.add(bpmn_process_definition)
@ -1016,9 +1018,11 @@ class ProcessInstanceProcessor:
)
for task_bpmn_identifier, task_bpmn_properties in task_specs.items():
task_bpmn_name = task_bpmn_properties["description"]
task_definition = TaskDefinitionModel(
bpmn_process_definition=bpmn_process_definition,
bpmn_identifier=task_bpmn_identifier,
bpmn_name=task_bpmn_name,
properties_json=task_bpmn_properties,
typename=task_bpmn_properties["typename"],
)

View File

@ -16,7 +16,7 @@
<bpmn:outgoing>Flow_09gjylo</bpmn:outgoing>
</bpmn:manualTask>
<bpmn:sequenceFlow id="Flow_0stlaxe" sourceRef="StartEvent_1" targetRef="top_level_script" />
<bpmn:scriptTask id="top_level_script">
<bpmn:scriptTask id="top_level_script" name="Top Level Script">
<bpmn:incoming>Flow_0stlaxe</bpmn:incoming>
<bpmn:outgoing>Flow_1fktmf7</bpmn:outgoing>
<bpmn:script>set_in_top_level_script = 1</bpmn:script>
@ -35,7 +35,7 @@
<bpmn:incoming>Flow_1b4o55k</bpmn:incoming>
</bpmn:endEvent>
<bpmn:sequenceFlow id="Flow_1b4o55k" sourceRef="top_level_subprocess_script" targetRef="Event_0zi0szr" />
<bpmn:scriptTask id="top_level_subprocess_script">
<bpmn:scriptTask id="top_level_subprocess_script" name="Top Level Subprocess Script">
<bpmn:incoming>Flow_00k1tii</bpmn:incoming>
<bpmn:outgoing>Flow_1b4o55k</bpmn:outgoing>
<bpmn:script>set_in_top_level_subprocess = 1

View File

@ -1,6 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<bpmn:definitions xmlns:bpmn="http://www.omg.org/spec/BPMN/20100524/MODEL" xmlns:bpmndi="http://www.omg.org/spec/BPMN/20100524/DI" xmlns:dc="http://www.omg.org/spec/DD/20100524/DC" xmlns:di="http://www.omg.org/spec/DD/20100524/DI" id="Definitions_96f6665" targetNamespace="http://bpmn.io/schema/bpmn" exporter="Camunda Modeler" exporterVersion="3.0.0-dev">
<bpmn:process id="test_process_to_call" isExecutable="true">
<bpmn:process id="test_process_to_call" name="Test Process To Call" isExecutable="true">
<bpmn:startEvent id="StartEvent_1">
<bpmn:outgoing>Flow_06g687y</bpmn:outgoing>
</bpmn:startEvent>
@ -9,7 +9,7 @@
<bpmn:incoming>Flow_01e21r0</bpmn:incoming>
</bpmn:endEvent>
<bpmn:sequenceFlow id="Flow_01e21r0" sourceRef="test_process_to_call_script" targetRef="Event_1nn875f" />
<bpmn:scriptTask id="test_process_to_call_script">
<bpmn:scriptTask id="test_process_to_call_script" name="Test Process To Call Script">
<bpmn:incoming>Flow_06g687y</bpmn:incoming>
<bpmn:outgoing>Flow_01e21r0</bpmn:outgoing>
<bpmn:script>set_in_test_process_to_call_script = 1</bpmn:script>

View File

@ -6,9 +6,8 @@
</bpmn:startEvent>
<bpmn:sequenceFlow id="Flow_0smvjir" sourceRef="StartEvent_1" targetRef="Activity_SimpleForm" />
<bpmn:endEvent id="Event_00xci7j">
<bpmn:incoming>Flow_1boyhcj</bpmn:incoming>
<bpmn:incoming>Flow_1scft9v</bpmn:incoming>
</bpmn:endEvent>
<bpmn:sequenceFlow id="Flow_1boyhcj" sourceRef="Activity_1cscoeg" targetRef="Event_00xci7j" />
<bpmn:manualTask id="Activity_1cscoeg" name="DisplayInfo">
<bpmn:extensionElements>
<spiffworkflow:instructionsForEndUser>Hello {{ name }}
@ -16,10 +15,9 @@ Department: {{ department }}
</spiffworkflow:instructionsForEndUser>
<spiffworkflow:postScript>user_completing_task = get_last_user_completing_task("Process_WithForm", "Activity_SimpleForm")</spiffworkflow:postScript>
</bpmn:extensionElements>
<bpmn:incoming>Flow_1ly1khd</bpmn:incoming>
<bpmn:outgoing>Flow_1boyhcj</bpmn:outgoing>
<bpmn:incoming>Flow_028o7v5</bpmn:incoming>
<bpmn:outgoing>Flow_18ytjgo</bpmn:outgoing>
</bpmn:manualTask>
<bpmn:sequenceFlow id="Flow_1ly1khd" sourceRef="Activity_SimpleForm" targetRef="Activity_1cscoeg" />
<bpmn:userTask id="Activity_SimpleForm" name="Simple Form">
<bpmn:extensionElements>
<spiffworkflow:properties>
@ -29,36 +27,68 @@ Department: {{ department }}
<spiffworkflow:postScript>process_initiator_user = get_process_initiator_user()</spiffworkflow:postScript>
</bpmn:extensionElements>
<bpmn:incoming>Flow_0smvjir</bpmn:incoming>
<bpmn:outgoing>Flow_1ly1khd</bpmn:outgoing>
<bpmn:outgoing>Flow_163ufsx</bpmn:outgoing>
</bpmn:userTask>
<bpmn:intermediateThrowEvent id="completed_form" name="Completed Form">
<bpmn:incoming>Flow_163ufsx</bpmn:incoming>
<bpmn:outgoing>Flow_028o7v5</bpmn:outgoing>
</bpmn:intermediateThrowEvent>
<bpmn:sequenceFlow id="Flow_163ufsx" sourceRef="Activity_SimpleForm" targetRef="completed_form" />
<bpmn:intermediateThrowEvent id="completed_manual_task" name="Completed Manual Task">
<bpmn:incoming>Flow_18ytjgo</bpmn:incoming>
<bpmn:outgoing>Flow_1scft9v</bpmn:outgoing>
</bpmn:intermediateThrowEvent>
<bpmn:sequenceFlow id="Flow_18ytjgo" sourceRef="Activity_1cscoeg" targetRef="completed_manual_task" />
<bpmn:sequenceFlow id="Flow_028o7v5" sourceRef="completed_form" targetRef="Activity_1cscoeg" />
<bpmn:sequenceFlow id="Flow_1scft9v" sourceRef="completed_manual_task" targetRef="Event_00xci7j" />
</bpmn:process>
<bpmndi:BPMNDiagram id="BPMNDiagram_1">
<bpmndi:BPMNPlane id="BPMNPlane_1" bpmnElement="Process_WithForm">
<bpmndi:BPMNShape id="_BPMNShape_StartEvent_2" bpmnElement="StartEvent_1">
<dc:Bounds x="179" y="159" width="36" height="36" />
</bpmndi:BPMNShape>
<bpmndi:BPMNShape id="Event_00xci7j_di" bpmnElement="Event_00xci7j">
<dc:Bounds x="592" y="159" width="36" height="36" />
</bpmndi:BPMNShape>
<bpmndi:BPMNShape id="Activity_00g930h_di" bpmnElement="Activity_1cscoeg">
<dc:Bounds x="430" y="137" width="100" height="80" />
<bpmndi:BPMNLabel />
</bpmndi:BPMNShape>
<bpmndi:BPMNShape id="Activity_0x5k4l1_di" bpmnElement="Activity_SimpleForm">
<dc:Bounds x="270" y="137" width="100" height="80" />
<bpmndi:BPMNLabel />
</bpmndi:BPMNShape>
<bpmndi:BPMNShape id="Activity_00g930h_di" bpmnElement="Activity_1cscoeg">
<dc:Bounds x="510" y="137" width="100" height="80" />
<bpmndi:BPMNLabel />
</bpmndi:BPMNShape>
<bpmndi:BPMNShape id="Event_00xci7j_di" bpmnElement="Event_00xci7j">
<dc:Bounds x="722" y="159" width="36" height="36" />
</bpmndi:BPMNShape>
<bpmndi:BPMNShape id="Event_0pi2wuv_di" bpmnElement="completed_form">
<dc:Bounds x="432" y="159" width="36" height="36" />
<bpmndi:BPMNLabel>
<dc:Bounds x="409" y="202" width="82" height="14" />
</bpmndi:BPMNLabel>
</bpmndi:BPMNShape>
<bpmndi:BPMNShape id="Event_071q0rf_di" bpmnElement="completed_manual_task">
<dc:Bounds x="662" y="159" width="36" height="36" />
<bpmndi:BPMNLabel>
<dc:Bounds x="649" y="202" width="63" height="27" />
</bpmndi:BPMNLabel>
</bpmndi:BPMNShape>
<bpmndi:BPMNEdge id="Flow_0smvjir_di" bpmnElement="Flow_0smvjir">
<di:waypoint x="215" y="177" />
<di:waypoint x="270" y="177" />
</bpmndi:BPMNEdge>
<bpmndi:BPMNEdge id="Flow_1boyhcj_di" bpmnElement="Flow_1boyhcj">
<di:waypoint x="530" y="177" />
<di:waypoint x="592" y="177" />
</bpmndi:BPMNEdge>
<bpmndi:BPMNEdge id="Flow_1ly1khd_di" bpmnElement="Flow_1ly1khd">
<bpmndi:BPMNEdge id="Flow_163ufsx_di" bpmnElement="Flow_163ufsx">
<di:waypoint x="370" y="177" />
<di:waypoint x="430" y="177" />
<di:waypoint x="432" y="177" />
</bpmndi:BPMNEdge>
<bpmndi:BPMNEdge id="Flow_18ytjgo_di" bpmnElement="Flow_18ytjgo">
<di:waypoint x="610" y="177" />
<di:waypoint x="662" y="177" />
</bpmndi:BPMNEdge>
<bpmndi:BPMNEdge id="Flow_028o7v5_di" bpmnElement="Flow_028o7v5">
<di:waypoint x="468" y="177" />
<di:waypoint x="510" y="177" />
</bpmndi:BPMNEdge>
<bpmndi:BPMNEdge id="Flow_1scft9v_di" bpmnElement="Flow_1scft9v">
<di:waypoint x="698" y="177" />
<di:waypoint x="722" y="177" />
</bpmndi:BPMNEdge>
</bpmndi:BPMNPlane>
</bpmndi:BPMNDiagram>

View File

@ -1,9 +1,12 @@
"""Test_logging_service."""
from uuid import UUID
from flask.app import Flask
from flask.testing import FlaskClient
from tests.spiffworkflow_backend.helpers.base_test import BaseTest
from tests.spiffworkflow_backend.helpers.test_data import load_test_spec
from spiffworkflow_backend.models.process_instance import ProcessInstanceModel
from spiffworkflow_backend.models.user import UserModel
from spiffworkflow_backend.services.authorization_service import AuthorizationService
from spiffworkflow_backend.services.process_instance_processor import (
@ -15,9 +18,7 @@ from spiffworkflow_backend.services.process_instance_service import (
class TestLoggingService(BaseTest):
"""Test logging service."""
def test_logging_service_spiff_logger(
def test_logging_service_detailed_logs(
self,
app: Flask,
client: FlaskClient,
@ -58,7 +59,7 @@ class TestLoggingService(BaseTest):
assert log_response.status_code == 200
assert log_response.json
logs: list = log_response.json["results"]
assert len(logs) == 7
assert len(logs) == 9
for log in logs:
assert log["process_instance_id"] == process_instance.id
@ -67,9 +68,65 @@ class TestLoggingService(BaseTest):
"end_in_seconds",
"spiff_task_guid",
"bpmn_process_definition_identifier",
"bpmn_process_definition_name",
"task_definition_identifier",
"task_definition_name",
"bpmn_type",
]:
assert key in log.keys()
if log["task_definition_identifier"] == "Activity_SimpleForm":
assert log["username"] == initiator_user.username
def test_logging_service_simple_logs(
self,
app: Flask,
client: FlaskClient,
with_db_and_bpmn_file_cleanup: None,
with_super_admin_user: UserModel,
) -> None:
self.create_process_group(client, with_super_admin_user, "test_group", "test_group")
initiator_user = self.find_or_create_user("initiator_user")
assert initiator_user.principal is not None
AuthorizationService.import_permissions_from_yaml_file()
process_model = load_test_spec(
process_model_id="misc/category_number_one/simple_form",
# bpmn_file_name="simp.bpmn",
process_model_source_directory="simple_form",
)
process_instance = self.create_process_instance_from_process_model(
process_model=process_model, user=initiator_user
)
processor = ProcessInstanceProcessor(process_instance)
processor.do_engine_steps(save=True)
assert len(process_instance.active_human_tasks) == 1
human_task = process_instance.active_human_tasks[0]
assert len(human_task.potential_owners) == 1
assert human_task.potential_owners[0] == initiator_user
spiff_task = processor.__class__.get_task_by_bpmn_identifier(
human_task.task_name, processor.bpmn_process_instance
)
ProcessInstanceService.complete_form_task(processor, spiff_task, {"name": "HEY"}, initiator_user, human_task)
process_instance = ProcessInstanceModel.query.filter_by(id=process_instance.id).first()
processor = ProcessInstanceProcessor(process_instance)
human_task_one = process_instance.active_human_tasks[0]
spiff_manual_task = processor.bpmn_process_instance.get_task(UUID(human_task_one.task_id))
ProcessInstanceService.complete_form_task(processor, spiff_manual_task, {}, initiator_user, human_task_one)
headers = self.logged_in_headers(with_super_admin_user)
log_response = client.get(
f"/v1.0/logs/{self.modify_process_identifier_for_path_param(process_model.id)}/{process_instance.id}?detailed=false",
headers=headers,
)
assert log_response.status_code == 200
assert log_response.json
logs: list = log_response.json["results"]
assert len(logs) == 2
for log in logs:
assert log["process_instance_id"] == process_instance.id
assert log["bpmn_type"] == "IntermediateThrowEvent"

View File

@ -336,9 +336,9 @@ class TestProcessInstanceProcessor(BaseTest):
spiff_tasks_checked_once: list = []
# TODO: also check task data here from the spiff_task directly to ensure we hydrated spiff correctly
def assert_spiff_task_is_in_process(spiff_task_name: str, bpmn_process_identifier: str) -> None:
if spiff_task.task_spec.name == spiff_task_name:
base_failure_message = f"Failed on {bpmn_process_identifier} - {spiff_task_name}."
def assert_spiff_task_is_in_process(spiff_task_identifier: str, bpmn_process_identifier: str) -> None:
if spiff_task.task_spec.name == spiff_task_identifier:
base_failure_message = f"Failed on {bpmn_process_identifier} - {spiff_task_identifier}."
expected_python_env_data = expected_task_data[spiff_task.task_spec.name]
if spiff_task.task_spec.name in spiff_tasks_checked_once:
expected_python_env_data = expected_task_data[f"{spiff_task.task_spec.name}_second"]
@ -347,9 +347,12 @@ class TestProcessInstanceProcessor(BaseTest):
assert task_model.start_in_seconds is not None
assert task_model.end_in_seconds is not None
assert task_model.task_definition_id is not None
task_definition = task_model.task_definition
assert task_definition.bpmn_identifier == spiff_task_name
assert task_definition.bpmn_identifier == spiff_task_identifier
assert task_definition.bpmn_name == spiff_task_identifier.replace("_", " ").title()
assert task_definition.bpmn_process_definition.bpmn_identifier == bpmn_process_identifier
message = (
f"{base_failure_message} Expected: {expected_python_env_data}. Received: {task_model.json_data()}"
)
@ -373,6 +376,7 @@ class TestProcessInstanceProcessor(BaseTest):
bpmn_process_definition = bpmn_process.bpmn_process_definition
assert bpmn_process_definition is not None
assert bpmn_process_definition.bpmn_identifier == "test_process_to_call"
assert bpmn_process_definition.bpmn_name == "Test Process To Call"
assert processor.get_data() == fifth_data_set