added api to run process model unit tests w/ burnettk

This commit is contained in:
jasquat 2023-05-22 17:36:07 -04:00
parent 3f6bc76a7e
commit 4ed43e5058
13 changed files with 365 additions and 38 deletions

View File

@ -451,6 +451,48 @@ paths:
schema:
$ref: "#/components/schemas/ProcessModel"
/process-model-tests/{modified_process_model_identifier}:
parameters:
- name: modified_process_model_identifier
in: path
required: true
description: The process_model_id, modified to replace slashes (/)
schema:
type: string
- name: test_case_file
in: query
required: false
description: The name of the test case file to run
schema:
type: string
- name: test_case_identifier
in: query
required: false
description: The name of the test case file to run
schema:
type: string
post:
operationId: spiffworkflow_backend.routes.process_models_controller.process_model_test_run
summary: Run a test for a process model
tags:
- Process Model Tests
requestBody:
content:
multipart/form-data:
schema:
type: object
properties:
file:
type: string
format: binary
responses:
"201":
description: Metadata about the uploaded file, but not the file content.
content:
application/json:
schema:
$ref: "#/components/schemas/File"
/process-models/{modified_process_model_identifier}/files:
parameters:
- name: modified_process_model_identifier

View File

@ -1,5 +1,7 @@
"""APIs for dealing with process groups, process models, and process instances."""
import json
from spiffworkflow_backend.services.file_system_service import FileSystemService
from spiffworkflow_backend.services.process_model_test_runner_service import ProcessModelTestRunner
import os
import re
from hashlib import sha256
@ -314,6 +316,29 @@ def process_model_file_show(modified_process_model_identifier: str, file_name: s
return make_response(jsonify(file), 200)
def process_model_test_run(
modified_process_model_identifier: str,
test_case_file: Optional[str] = None,
test_case_identifier: Optional[str] = None,
) -> flask.wrappers.Response:
process_model_identifier = modified_process_model_identifier.replace(":", "/")
process_model = _get_process_model(process_model_identifier)
process_model_test_runner = ProcessModelTestRunner(
process_model_directory_path=FileSystemService.root_path(),
process_model_directory_for_test_discovery=FileSystemService.full_path_from_id(process_model.id),
test_case_file=test_case_file,
test_case_identifier=test_case_identifier,
)
process_model_test_runner.run()
response_json = {
"all_passed": process_model_test_runner.all_test_cases_passed(),
"passing": process_model_test_runner.passing_tests(),
"failing": process_model_test_runner.failing_tests(),
}
return make_response(jsonify(response_json), 200)
# {
# "natural_language_text": "Create a bug tracker process model \
# with a bug-details form that collects summary, description, and priority"

View File

@ -47,7 +47,7 @@ class MissingInputTaskData(Exception):
class TestCaseResult:
passed: bool
bpmn_file: str
test_case_name: str
test_case_identifier: str
error_messages: Optional[list[str]] = None
@ -90,6 +90,8 @@ class ProcessModelTestRunner:
instantiate_executer_callback: Optional[Callable[[str], Any]] = None,
execute_task_callback: Optional[Callable[[Any, str, Optional[dict]], Any]] = None,
get_next_task_callback: Optional[Callable[[Any], Any]] = None,
test_case_file: Optional[str] = None,
test_case_identifier: Optional[str] = None,
) -> None:
self.process_model_directory_path = process_model_directory_path
self.process_model_directory_for_test_discovery = (
@ -98,6 +100,8 @@ class ProcessModelTestRunner:
self.instantiate_executer_callback = instantiate_executer_callback
self.execute_task_callback = execute_task_callback
self.get_next_task_callback = get_next_task_callback
self.test_case_file = test_case_file
self.test_case_identifier = test_case_identifier
# keep track of the current task data index
self.task_data_index: dict[str, int] = {}
@ -116,13 +120,16 @@ class ProcessModelTestRunner:
def failing_tests(self) -> list[TestCaseResult]:
return [t for t in self.test_case_results if t.passed is False]
def passing_tests(self) -> list[TestCaseResult]:
return [t for t in self.test_case_results if t.passed is True]
def failing_tests_formatted(self) -> str:
formatted_tests = ["FAILING TESTS:"]
for failing_test in self.failing_tests():
msg = ""
if failing_test.error_messages:
msg = "\n\t\t".join(failing_test.error_messages)
formatted_tests.append(f"\t{failing_test.bpmn_file}: {failing_test.test_case_name}: {msg}")
formatted_tests.append(f"\t{failing_test.bpmn_file}: {failing_test.test_case_identifier}: {msg}")
return "\n".join(formatted_tests)
def run(self) -> None:
@ -134,15 +141,16 @@ class ProcessModelTestRunner:
with open(json_test_case_file) as f:
json_file_contents = json.loads(f.read())
for test_case_name, test_case_contents in json_file_contents.items():
self.task_data_index = {}
try:
self.run_test_case(bpmn_file, test_case_name, test_case_contents)
except Exception as ex:
ex_as_array = str(ex).split("\n")
self._add_test_result(False, bpmn_file, test_case_name, ex_as_array)
for test_case_identifier, test_case_contents in json_file_contents.items():
if self.test_case_identifier is None or test_case_identifier == self.test_case_identifier:
self.task_data_index = {}
try:
self.run_test_case(bpmn_file, test_case_identifier, test_case_contents)
except Exception as ex:
ex_as_array = str(ex).split("\n")
self._add_test_result(False, bpmn_file, test_case_identifier, ex_as_array)
def run_test_case(self, bpmn_file: str, test_case_name: str, test_case_contents: dict) -> None:
def run_test_case(self, bpmn_file: str, test_case_identifier: str, test_case_contents: dict) -> None:
bpmn_process_instance = self._instantiate_executer(bpmn_file)
next_task = self._get_next_task(bpmn_process_instance)
while next_task is not None:
@ -158,7 +166,7 @@ class ProcessModelTestRunner:
task_type = next_task.task_spec.__class__.__name__
if task_type in ["ServiceTask", "UserTask", "CallActivity"] and test_case_task_properties is None:
raise UnrunnableTestCaseError(
f"Cannot run test case '{test_case_name}'. It requires task data for"
f"Cannot run test case '{test_case_identifier}'. It requires task data for"
f" {next_task.task_spec.bpmn_id} because it is of type '{task_type}'"
)
self._execute_task(next_task, test_case_task_key, test_case_task_properties)
@ -183,7 +191,7 @@ class ProcessModelTestRunner:
f"expected: {test_case_contents['expected_output_json']}",
f"actual: {bpmn_process_instance.data}",
]
self._add_test_result(error_message is None, bpmn_file, test_case_name, error_message)
self._add_test_result(error_message is None, bpmn_file, test_case_identifier, error_message)
def _discover_process_model_test_cases(
self,
@ -196,14 +204,15 @@ class ProcessModelTestRunner:
file_norm = os.path.normpath(file)
file_dir = os.path.dirname(file_norm)
json_file_name = os.path.basename(file_norm)
bpmn_file_name = re.sub(r"^test_(.*)\.json", r"\1.bpmn", json_file_name)
bpmn_file_path = os.path.join(file_dir, bpmn_file_name)
if os.path.isfile(bpmn_file_path):
test_mappings[file_norm] = bpmn_file_path
else:
raise MissingBpmnFileForTestCaseError(
f"Cannot find a matching bpmn file for test case json file: '{file_norm}'"
)
if self.test_case_file is None or json_file_name == self.test_case_file:
bpmn_file_name = re.sub(r"^test_(.*)\.json", r"\1.bpmn", json_file_name)
bpmn_file_path = os.path.join(file_dir, bpmn_file_name)
if os.path.isfile(bpmn_file_path):
test_mappings[file_norm] = bpmn_file_path
else:
raise MissingBpmnFileForTestCaseError(
f"Cannot find a matching bpmn file for test case json file: '{file_norm}'"
)
return test_mappings
def _discover_process_model_processes(
@ -218,14 +227,23 @@ class ProcessModelTestRunner:
with open(file_norm, "rb") as f:
file_contents = f.read()
etree_xml_parser = etree.XMLParser(resolve_entities=False)
root = etree.fromstring(file_contents, parser=etree_xml_parser)
# if we cannot load process model then ignore it since it can cause errors unrelated
# to the test and if it is related, it will most likely be caught further along the test
try:
root = etree.fromstring(file_contents, parser=etree_xml_parser)
except etree.XMLSyntaxError:
continue
call_activities = root.findall(".//bpmn:callActivity", namespaces=DEFAULT_NSMAP)
for call_activity in call_activities:
called_element = call_activity.attrib["calledElement"]
self.bpmn_files_to_called_element_mappings[file_norm].append(called_element)
if "calledElement" in call_activity.attrib:
called_element = call_activity.attrib["calledElement"]
self.bpmn_files_to_called_element_mappings[file_norm].append(called_element)
bpmn_process_element = root.find('.//bpmn:process[@isExecutable="true"]', namespaces=DEFAULT_NSMAP)
bpmn_process_identifier = bpmn_process_element.attrib["id"]
self.bpmn_processes_to_file_mappings[bpmn_process_identifier] = file_norm
if bpmn_process_element is not None:
bpmn_process_identifier = bpmn_process_element.attrib["id"]
self.bpmn_processes_to_file_mappings[bpmn_process_identifier] = file_norm
def _execute_task(
self, spiff_task: SpiffTask, test_case_task_key: str, test_case_task_properties: Optional[dict]
@ -312,13 +330,13 @@ class ProcessModelTestRunner:
return os.path.relpath(bpmn_file, start=self.process_model_directory_path)
def _add_test_result(
self, passed: bool, bpmn_file: str, test_case_name: str, error_messages: Optional[list[str]] = None
self, passed: bool, bpmn_file: str, test_case_identifier: str, error_messages: Optional[list[str]] = None
) -> None:
bpmn_file_relative = self._get_relative_path_of_bpmn_file(bpmn_file)
test_result = TestCaseResult(
passed=passed,
bpmn_file=bpmn_file_relative,
test_case_name=test_case_name,
test_case_identifier=test_case_identifier,
error_messages=error_messages,
)
self.test_case_results.append(test_result)
@ -329,9 +347,16 @@ class BpmnFileMissingExecutableProcessError(Exception):
class ProcessModelTestRunnerService:
def __init__(self, process_model_directory_path: str) -> None:
def __init__(
self,
process_model_directory_path: str,
test_case_file: Optional[str] = None,
test_case_identifier: Optional[str] = None,
) -> None:
self.process_model_test_runner = ProcessModelTestRunner(
process_model_directory_path,
test_case_file=test_case_file,
test_case_identifier=test_case_identifier,
# instantiate_executer_callback=self._instantiate_executer_callback,
# execute_task_callback=self._execute_task_callback,
# get_next_task_callback=self._get_next_task_callback,

View File

@ -0,0 +1,39 @@
<?xml version="1.0" encoding="UTF-8"?>
<bpmn:definitions xmlns:bpmn="http://www.omg.org/spec/BPMN/20100524/MODEL" xmlns:bpmndi="http://www.omg.org/spec/BPMN/20100524/DI" xmlns:dc="http://www.omg.org/spec/DD/20100524/DC" xmlns:di="http://www.omg.org/spec/DD/20100524/DI" id="Definitions_96f6665" targetNamespace="http://bpmn.io/schema/bpmn" exporter="Camunda Modeler" exporterVersion="3.0.0-dev">
<bpmn:process id="ProcessA" name="ProcessA" isExecutable="true">
<bpmn:startEvent id="StartEvent_1">
<bpmn:outgoing>Flow_0jk46kf</bpmn:outgoing>
</bpmn:startEvent>
<bpmn:sequenceFlow id="Flow_0jk46kf" sourceRef="StartEvent_1" targetRef="Activity_0e9rl60" />
<bpmn:endEvent id="Event_1srknca">
<bpmn:incoming>Flow_0pw6euz</bpmn:incoming>
</bpmn:endEvent>
<bpmn:sequenceFlow id="Flow_0pw6euz" sourceRef="Activity_0e9rl60" targetRef="Event_1srknca" />
<bpmn:scriptTask id="Activity_0e9rl60">
<bpmn:incoming>Flow_0jk46kf</bpmn:incoming>
<bpmn:outgoing>Flow_0pw6euz</bpmn:outgoing>
<bpmn:script>a = 1</bpmn:script>
</bpmn:scriptTask>
</bpmn:process>
<bpmndi:BPMNDiagram id="BPMNDiagram_1">
<bpmndi:BPMNPlane id="BPMNPlane_1" bpmnElement="ProcessA">
<bpmndi:BPMNShape id="_BPMNShape_StartEvent_2" bpmnElement="StartEvent_1">
<dc:Bounds x="179" y="159" width="36" height="36" />
</bpmndi:BPMNShape>
<bpmndi:BPMNShape id="Event_1srknca_di" bpmnElement="Event_1srknca">
<dc:Bounds x="432" y="159" width="36" height="36" />
</bpmndi:BPMNShape>
<bpmndi:BPMNShape id="Activity_0yxs81w_di" bpmnElement="Activity_0e9rl60">
<dc:Bounds x="270" y="137" width="100" height="80" />
</bpmndi:BPMNShape>
<bpmndi:BPMNEdge id="Flow_0jk46kf_di" bpmnElement="Flow_0jk46kf">
<di:waypoint x="215" y="177" />
<di:waypoint x="270" y="177" />
</bpmndi:BPMNEdge>
<bpmndi:BPMNEdge id="Flow_0pw6euz_di" bpmnElement="Flow_0pw6euz">
<di:waypoint x="370" y="177" />
<di:waypoint x="432" y="177" />
</bpmndi:BPMNEdge>
</bpmndi:BPMNPlane>
</bpmndi:BPMNDiagram>
</bpmn:definitions>

View File

@ -0,0 +1,42 @@
<?xml version="1.0" encoding="UTF-8"?>
<bpmn:definitions xmlns:bpmn="http://www.omg.org/spec/BPMN/20100524/MODEL" xmlns:bpmndi="http://www.omg.org/spec/BPMN/20100524/DI" xmlns:dc="http://www.omg.org/spec/DD/20100524/DC" xmlns:di="http://www.omg.org/spec/DD/20100524/DI" id="Definitions_96f6665" targetNamespace="http://bpmn.io/schema/bpmn" exporter="Camunda Modeler" exporterVersion="3.0.0-dev">
<bpmn:process id="Process_39edgqg" isExecutable="true">
<bpmn:startEvent id="StartEvent_1">
<bpmn:outgoing>Flow_1qgv480</bpmn:outgoing>
</bpmn:startEvent>
<bpmn:sequenceFlow id="Flow_1qgv480" sourceRef="StartEvent_1" targetRef="Activity_1kral0x" />
<bpmn:endEvent id="ProcessB" name="ProcessB">
<bpmn:incoming>Flow_1sbj39z</bpmn:incoming>
</bpmn:endEvent>
<bpmn:sequenceFlow id="Flow_1sbj39z" sourceRef="Activity_1kral0x" targetRef="ProcessB" />
<bpmn:scriptTask id="Activity_1kral0x">
<bpmn:incoming>Flow_1qgv480</bpmn:incoming>
<bpmn:outgoing>Flow_1sbj39z</bpmn:outgoing>
<bpmn:script>b = 1</bpmn:script>
</bpmn:scriptTask>
</bpmn:process>
<bpmndi:BPMNDiagram id="BPMNDiagram_1">
<bpmndi:BPMNPlane id="BPMNPlane_1" bpmnElement="Process_39edgqg">
<bpmndi:BPMNShape id="_BPMNShape_StartEvent_2" bpmnElement="StartEvent_1">
<dc:Bounds x="179" y="159" width="36" height="36" />
</bpmndi:BPMNShape>
<bpmndi:BPMNShape id="Event_12lq7sg_di" bpmnElement="ProcessB">
<dc:Bounds x="432" y="159" width="36" height="36" />
<bpmndi:BPMNLabel>
<dc:Bounds x="427" y="202" width="48" height="14" />
</bpmndi:BPMNLabel>
</bpmndi:BPMNShape>
<bpmndi:BPMNShape id="Activity_0pkm1sr_di" bpmnElement="Activity_1kral0x">
<dc:Bounds x="270" y="137" width="100" height="80" />
</bpmndi:BPMNShape>
<bpmndi:BPMNEdge id="Flow_1qgv480_di" bpmnElement="Flow_1qgv480">
<di:waypoint x="215" y="177" />
<di:waypoint x="270" y="177" />
</bpmndi:BPMNEdge>
<bpmndi:BPMNEdge id="Flow_1sbj39z_di" bpmnElement="Flow_1sbj39z">
<di:waypoint x="370" y="177" />
<di:waypoint x="432" y="177" />
</bpmndi:BPMNEdge>
</bpmndi:BPMNPlane>
</bpmndi:BPMNDiagram>
</bpmn:definitions>

View File

@ -0,0 +1,11 @@
{
"description": "",
"display_name": "Multiple Test Files",
"display_order": 0,
"exception_notification_addresses": [],
"fault_or_suspend_on_exception": "fault",
"files": [],
"metadata_extraction_paths": null,
"primary_file_name": "a.bpmn",
"primary_process_id": "ProcessA"
}

View File

@ -0,0 +1,5 @@
{
"test_case_1": {
"expected_output_json": { "a": 1 }
}
}

View File

@ -0,0 +1,8 @@
{
"test_case_1": {
"expected_output_json": { "b": 1 }
},
"test_case_2": {
"expected_output_json": { "b": 1 }
}
}

View File

@ -53,6 +53,24 @@ class TestProcessModelTestRunner(BaseTest):
process_model_test_runner = self._run_model_tests(parent_directory="expected-to-fail")
assert len(process_model_test_runner.test_case_results) == 1
def test_can_test_process_model_with_multiple_files(
self,
app: Flask,
with_db_and_bpmn_file_cleanup: None,
with_mocked_root_path: Any,
) -> None:
process_model_test_runner = self._run_model_tests(bpmn_process_directory_name="multiple-test-files")
assert len(process_model_test_runner.test_case_results) == 3
process_model_test_runner = self._run_model_tests(bpmn_process_directory_name="multiple-test-files", test_case_file='test_a.json')
assert len(process_model_test_runner.test_case_results) == 1
process_model_test_runner = self._run_model_tests(bpmn_process_directory_name="multiple-test-files", test_case_file='test_b.json')
assert len(process_model_test_runner.test_case_results) == 2
process_model_test_runner = self._run_model_tests(bpmn_process_directory_name="multiple-test-files", test_case_file='test_b.json', test_case_identifier='test_case_2')
assert len(process_model_test_runner.test_case_results) == 1
def test_can_test_process_model_call_activity(
self,
app: Flask,
@ -81,7 +99,8 @@ class TestProcessModelTestRunner(BaseTest):
assert len(process_model_test_runner.test_case_results) == 1
def _run_model_tests(
self, bpmn_process_directory_name: Optional[str] = None, parent_directory: str = "expected-to-pass"
self, bpmn_process_directory_name: Optional[str] = None, parent_directory: str = "expected-to-pass",
test_case_file: Optional[str] = None, test_case_identifier: Optional[str] = None,
) -> ProcessModelTestRunner:
base_process_model_dir_path_segments = [FileSystemService.root_path(), parent_directory]
path_segments = base_process_model_dir_path_segments
@ -90,6 +109,8 @@ class TestProcessModelTestRunner(BaseTest):
process_model_test_runner = ProcessModelTestRunner(
process_model_directory_path=os.path.join(*base_process_model_dir_path_segments),
process_model_directory_for_test_discovery=os.path.join(*path_segments),
test_case_file=test_case_file,
test_case_identifier=test_case_identifier,
)
process_model_test_runner.run()

View File

@ -0,0 +1,93 @@
import { PlayOutline, Checkmark, Close } from '@carbon/icons-react';
import { Button, Modal } from '@carbon/react';
import { useState } from 'react';
import { useUriListForPermissions } from '../hooks/UriListForPermissions';
import HttpService from '../services/HttpService';
import { ProcessFile } from '../interfaces';
type OwnProps = {
processModelFile: ProcessFile;
};
export default function ProcessModelTestRun({ processModelFile }: OwnProps) {
const [testCaseResults, setTestCaseResults] = useState<any>(null);
const [showTestCaseResultsModal, setShowTestCaseResultsModal] =
useState<boolean>(false);
const { targetUris } = useUriListForPermissions();
const onProcessModelTestRunSuccess = (result: any) => {
setTestCaseResults(result);
};
const processModelTestRunResultTag = () => {
if (testCaseResults) {
if (testCaseResults.all_passed) {
return (
<Button
kind="ghost"
className="green-icon"
renderIcon={Checkmark}
iconDescription="PASS"
hasIconOnly
size="lg"
onClick={() => setShowTestCaseResultsModal(true)}
/>
);
}
return (
<Button
kind="ghost"
className="red-icon"
renderIcon={Close}
iconDescription="FAILS"
hasIconOnly
size="lg"
onClick={() => setShowTestCaseResultsModal(true)}
/>
);
}
return null;
};
const onProcessModelTestRun = (fileName: string) => {
const httpMethod = 'POST';
setTestCaseResults(null);
HttpService.makeCallToBackend({
path: `${targetUris.processModelTestsPath}?test_case_file=${fileName}`,
successCallback: onProcessModelTestRunSuccess,
httpMethod,
});
};
const testCaseResultsModal = () => {
return (
<Modal
open={showTestCaseResultsModal}
data-qa="test-case-results-modal"
modalHeading="RESULT FOR"
modalLabel="LABLE"
primaryButtonText="OK"
onRequestSubmit={() => setShowTestCaseResultsModal(false)}
onRequestClose={() => setShowTestCaseResultsModal(false)}
>
{JSON.stringify(testCaseResults)}
</Modal>
);
};
return (
<>
{testCaseResultsModal()}
<Button
kind="ghost"
renderIcon={PlayOutline}
iconDescription="Run Test"
hasIconOnly
size="lg"
onClick={() => onProcessModelTestRun(processModelFile.name)}
/>
{processModelTestRunResultTag()}
</>
);
}

View File

@ -29,6 +29,7 @@ export const useUriListForPermissions = () => {
processModelFileShowPath: `/v1.0/process-models/${params.process_model_id}/files/${params.file_name}`,
processModelPublishPath: `/v1.0/process-model-publish/${params.process_model_id}`,
processModelShowPath: `/v1.0/process-models/${params.process_model_id}`,
processModelTestsPath: `/v1.0/process-model-tests/${params.process_model_id}`,
secretListPath: `/v1.0/secrets`,
userSearch: `/v1.0/users/search`,
userExists: `/v1.0/users/exists/by-username`,

View File

@ -418,6 +418,12 @@ td.actions-cell {
padding-bottom: 10px;
}
.cds--btn--ghost:not([disabled]).red-icon svg {
fill: red;
}
.cds--btn--ghost:not([disabled]).green-icon svg {
fill: #198038;
}
.cds--btn--ghost:not([disabled]) svg.red-icon {
fill: red;
}

View File

@ -2,11 +2,11 @@ import { useEffect, useState } from 'react';
import { Link, useNavigate, useParams } from 'react-router-dom';
import {
Add,
Upload,
Download,
TrashCan,
Favorite,
Edit,
Favorite,
TrashCan,
Upload,
View,
// @ts-ignore
} from '@carbon/icons-react';
@ -14,18 +14,18 @@ import {
Accordion,
AccordionItem,
Button,
Grid,
Column,
Stack,
ButtonSet,
Modal,
Column,
FileUploader,
Grid,
Modal,
Stack,
Table,
TableBody,
TableCell,
TableHead,
TableHeader,
TableRow,
TableCell,
TableBody,
// @ts-ignore
} from '@carbon/react';
import { Can } from '@casl/react';
@ -49,6 +49,7 @@ import { usePermissionFetcher } from '../hooks/PermissionService';
import { useUriListForPermissions } from '../hooks/UriListForPermissions';
import ProcessInstanceRun from '../components/ProcessInstanceRun';
import { Notification } from '../components/Notification';
import ProcessModelTestRun from '../components/ProcessModelTestRun';
export default function ProcessModelShow() {
const params = useParams();
@ -68,6 +69,7 @@ export default function ProcessModelShow() {
const { targetUris } = useUriListForPermissions();
const permissionRequestData: PermissionsToCheck = {
[targetUris.processModelShowPath]: ['PUT', 'DELETE'],
[targetUris.processModelTestsPath]: ['POST'],
[targetUris.processModelPublishPath]: ['POST'],
[targetUris.processInstanceListPath]: ['GET'],
[targetUris.processInstanceCreatePath]: ['POST'],
@ -308,6 +310,13 @@ export default function ProcessModelShow() {
</Can>
);
}
if (processModelFile.name.match(/^test_.*\.json$/)) {
elements.push(
<Can I="POST" a={targetUris.processModelTestsPath} ability={ability}>
<ProcessModelTestRun processModelFile={processModelFile} />
</Can>
);
}
return elements;
};