From 4ed43e505877cbc838a3e9a52ee13d82f62bf450 Mon Sep 17 00:00:00 2001 From: jasquat Date: Mon, 22 May 2023 17:36:07 -0400 Subject: [PATCH] added api to run process model unit tests w/ burnettk --- .../src/spiffworkflow_backend/api.yml | 42 +++++++++ .../routes/process_models_controller.py | 25 +++++ .../process_model_test_runner_service.py | 81 ++++++++++------ .../multiple-test-files/a.bpmn | 39 ++++++++ .../multiple-test-files/b.bpmn | 42 +++++++++ .../multiple-test-files/process_model.json | 11 +++ .../multiple-test-files/test_a.json | 5 + .../multiple-test-files/test_b.json | 8 ++ .../unit/test_process_model_test_runner.py | 23 ++++- .../src/components/ProcessModelTestRun.tsx | 93 +++++++++++++++++++ .../src/hooks/UriListForPermissions.tsx | 1 + spiffworkflow-frontend/src/index.css | 6 ++ .../src/routes/ProcessModelShow.tsx | 27 ++++-- 13 files changed, 365 insertions(+), 38 deletions(-) create mode 100644 spiffworkflow-backend/tests/data/bpmn_unit_test_process_models/expected-to-pass/multiple-test-files/a.bpmn create mode 100644 spiffworkflow-backend/tests/data/bpmn_unit_test_process_models/expected-to-pass/multiple-test-files/b.bpmn create mode 100644 spiffworkflow-backend/tests/data/bpmn_unit_test_process_models/expected-to-pass/multiple-test-files/process_model.json create mode 100644 spiffworkflow-backend/tests/data/bpmn_unit_test_process_models/expected-to-pass/multiple-test-files/test_a.json create mode 100644 spiffworkflow-backend/tests/data/bpmn_unit_test_process_models/expected-to-pass/multiple-test-files/test_b.json create mode 100644 spiffworkflow-frontend/src/components/ProcessModelTestRun.tsx diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/api.yml b/spiffworkflow-backend/src/spiffworkflow_backend/api.yml index fd308f44e..fed169ec6 100755 --- a/spiffworkflow-backend/src/spiffworkflow_backend/api.yml +++ b/spiffworkflow-backend/src/spiffworkflow_backend/api.yml @@ -451,6 +451,48 @@ paths: schema: $ref: "#/components/schemas/ProcessModel" + /process-model-tests/{modified_process_model_identifier}: + parameters: + - name: modified_process_model_identifier + in: path + required: true + description: The process_model_id, modified to replace slashes (/) + schema: + type: string + - name: test_case_file + in: query + required: false + description: The name of the test case file to run + schema: + type: string + - name: test_case_identifier + in: query + required: false + description: The name of the test case file to run + schema: + type: string + post: + operationId: spiffworkflow_backend.routes.process_models_controller.process_model_test_run + summary: Run a test for a process model + tags: + - Process Model Tests + requestBody: + content: + multipart/form-data: + schema: + type: object + properties: + file: + type: string + format: binary + responses: + "201": + description: Metadata about the uploaded file, but not the file content. + content: + application/json: + schema: + $ref: "#/components/schemas/File" + /process-models/{modified_process_model_identifier}/files: parameters: - name: modified_process_model_identifier diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/routes/process_models_controller.py b/spiffworkflow-backend/src/spiffworkflow_backend/routes/process_models_controller.py index 00d82639b..81e64dce2 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/routes/process_models_controller.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/routes/process_models_controller.py @@ -1,5 +1,7 @@ """APIs for dealing with process groups, process models, and process instances.""" import json +from spiffworkflow_backend.services.file_system_service import FileSystemService +from spiffworkflow_backend.services.process_model_test_runner_service import ProcessModelTestRunner import os import re from hashlib import sha256 @@ -314,6 +316,29 @@ def process_model_file_show(modified_process_model_identifier: str, file_name: s return make_response(jsonify(file), 200) +def process_model_test_run( + modified_process_model_identifier: str, + test_case_file: Optional[str] = None, + test_case_identifier: Optional[str] = None, +) -> flask.wrappers.Response: + process_model_identifier = modified_process_model_identifier.replace(":", "/") + process_model = _get_process_model(process_model_identifier) + process_model_test_runner = ProcessModelTestRunner( + process_model_directory_path=FileSystemService.root_path(), + process_model_directory_for_test_discovery=FileSystemService.full_path_from_id(process_model.id), + test_case_file=test_case_file, + test_case_identifier=test_case_identifier, + ) + process_model_test_runner.run() + + response_json = { + "all_passed": process_model_test_runner.all_test_cases_passed(), + "passing": process_model_test_runner.passing_tests(), + "failing": process_model_test_runner.failing_tests(), + } + return make_response(jsonify(response_json), 200) + + # { # "natural_language_text": "Create a bug tracker process model \ # with a bug-details form that collects summary, description, and priority" diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/services/process_model_test_runner_service.py b/spiffworkflow-backend/src/spiffworkflow_backend/services/process_model_test_runner_service.py index 47875e10a..01057d449 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/services/process_model_test_runner_service.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/services/process_model_test_runner_service.py @@ -47,7 +47,7 @@ class MissingInputTaskData(Exception): class TestCaseResult: passed: bool bpmn_file: str - test_case_name: str + test_case_identifier: str error_messages: Optional[list[str]] = None @@ -90,6 +90,8 @@ class ProcessModelTestRunner: instantiate_executer_callback: Optional[Callable[[str], Any]] = None, execute_task_callback: Optional[Callable[[Any, str, Optional[dict]], Any]] = None, get_next_task_callback: Optional[Callable[[Any], Any]] = None, + test_case_file: Optional[str] = None, + test_case_identifier: Optional[str] = None, ) -> None: self.process_model_directory_path = process_model_directory_path self.process_model_directory_for_test_discovery = ( @@ -98,6 +100,8 @@ class ProcessModelTestRunner: self.instantiate_executer_callback = instantiate_executer_callback self.execute_task_callback = execute_task_callback self.get_next_task_callback = get_next_task_callback + self.test_case_file = test_case_file + self.test_case_identifier = test_case_identifier # keep track of the current task data index self.task_data_index: dict[str, int] = {} @@ -116,13 +120,16 @@ class ProcessModelTestRunner: def failing_tests(self) -> list[TestCaseResult]: return [t for t in self.test_case_results if t.passed is False] + def passing_tests(self) -> list[TestCaseResult]: + return [t for t in self.test_case_results if t.passed is True] + def failing_tests_formatted(self) -> str: formatted_tests = ["FAILING TESTS:"] for failing_test in self.failing_tests(): msg = "" if failing_test.error_messages: msg = "\n\t\t".join(failing_test.error_messages) - formatted_tests.append(f"\t{failing_test.bpmn_file}: {failing_test.test_case_name}: {msg}") + formatted_tests.append(f"\t{failing_test.bpmn_file}: {failing_test.test_case_identifier}: {msg}") return "\n".join(formatted_tests) def run(self) -> None: @@ -134,15 +141,16 @@ class ProcessModelTestRunner: with open(json_test_case_file) as f: json_file_contents = json.loads(f.read()) - for test_case_name, test_case_contents in json_file_contents.items(): - self.task_data_index = {} - try: - self.run_test_case(bpmn_file, test_case_name, test_case_contents) - except Exception as ex: - ex_as_array = str(ex).split("\n") - self._add_test_result(False, bpmn_file, test_case_name, ex_as_array) + for test_case_identifier, test_case_contents in json_file_contents.items(): + if self.test_case_identifier is None or test_case_identifier == self.test_case_identifier: + self.task_data_index = {} + try: + self.run_test_case(bpmn_file, test_case_identifier, test_case_contents) + except Exception as ex: + ex_as_array = str(ex).split("\n") + self._add_test_result(False, bpmn_file, test_case_identifier, ex_as_array) - def run_test_case(self, bpmn_file: str, test_case_name: str, test_case_contents: dict) -> None: + def run_test_case(self, bpmn_file: str, test_case_identifier: str, test_case_contents: dict) -> None: bpmn_process_instance = self._instantiate_executer(bpmn_file) next_task = self._get_next_task(bpmn_process_instance) while next_task is not None: @@ -158,7 +166,7 @@ class ProcessModelTestRunner: task_type = next_task.task_spec.__class__.__name__ if task_type in ["ServiceTask", "UserTask", "CallActivity"] and test_case_task_properties is None: raise UnrunnableTestCaseError( - f"Cannot run test case '{test_case_name}'. It requires task data for" + f"Cannot run test case '{test_case_identifier}'. It requires task data for" f" {next_task.task_spec.bpmn_id} because it is of type '{task_type}'" ) self._execute_task(next_task, test_case_task_key, test_case_task_properties) @@ -183,7 +191,7 @@ class ProcessModelTestRunner: f"expected: {test_case_contents['expected_output_json']}", f"actual: {bpmn_process_instance.data}", ] - self._add_test_result(error_message is None, bpmn_file, test_case_name, error_message) + self._add_test_result(error_message is None, bpmn_file, test_case_identifier, error_message) def _discover_process_model_test_cases( self, @@ -196,14 +204,15 @@ class ProcessModelTestRunner: file_norm = os.path.normpath(file) file_dir = os.path.dirname(file_norm) json_file_name = os.path.basename(file_norm) - bpmn_file_name = re.sub(r"^test_(.*)\.json", r"\1.bpmn", json_file_name) - bpmn_file_path = os.path.join(file_dir, bpmn_file_name) - if os.path.isfile(bpmn_file_path): - test_mappings[file_norm] = bpmn_file_path - else: - raise MissingBpmnFileForTestCaseError( - f"Cannot find a matching bpmn file for test case json file: '{file_norm}'" - ) + if self.test_case_file is None or json_file_name == self.test_case_file: + bpmn_file_name = re.sub(r"^test_(.*)\.json", r"\1.bpmn", json_file_name) + bpmn_file_path = os.path.join(file_dir, bpmn_file_name) + if os.path.isfile(bpmn_file_path): + test_mappings[file_norm] = bpmn_file_path + else: + raise MissingBpmnFileForTestCaseError( + f"Cannot find a matching bpmn file for test case json file: '{file_norm}'" + ) return test_mappings def _discover_process_model_processes( @@ -218,14 +227,23 @@ class ProcessModelTestRunner: with open(file_norm, "rb") as f: file_contents = f.read() etree_xml_parser = etree.XMLParser(resolve_entities=False) - root = etree.fromstring(file_contents, parser=etree_xml_parser) + + # if we cannot load process model then ignore it since it can cause errors unrelated + # to the test and if it is related, it will most likely be caught further along the test + try: + root = etree.fromstring(file_contents, parser=etree_xml_parser) + except etree.XMLSyntaxError: + continue + call_activities = root.findall(".//bpmn:callActivity", namespaces=DEFAULT_NSMAP) for call_activity in call_activities: - called_element = call_activity.attrib["calledElement"] - self.bpmn_files_to_called_element_mappings[file_norm].append(called_element) + if "calledElement" in call_activity.attrib: + called_element = call_activity.attrib["calledElement"] + self.bpmn_files_to_called_element_mappings[file_norm].append(called_element) bpmn_process_element = root.find('.//bpmn:process[@isExecutable="true"]', namespaces=DEFAULT_NSMAP) - bpmn_process_identifier = bpmn_process_element.attrib["id"] - self.bpmn_processes_to_file_mappings[bpmn_process_identifier] = file_norm + if bpmn_process_element is not None: + bpmn_process_identifier = bpmn_process_element.attrib["id"] + self.bpmn_processes_to_file_mappings[bpmn_process_identifier] = file_norm def _execute_task( self, spiff_task: SpiffTask, test_case_task_key: str, test_case_task_properties: Optional[dict] @@ -312,13 +330,13 @@ class ProcessModelTestRunner: return os.path.relpath(bpmn_file, start=self.process_model_directory_path) def _add_test_result( - self, passed: bool, bpmn_file: str, test_case_name: str, error_messages: Optional[list[str]] = None + self, passed: bool, bpmn_file: str, test_case_identifier: str, error_messages: Optional[list[str]] = None ) -> None: bpmn_file_relative = self._get_relative_path_of_bpmn_file(bpmn_file) test_result = TestCaseResult( passed=passed, bpmn_file=bpmn_file_relative, - test_case_name=test_case_name, + test_case_identifier=test_case_identifier, error_messages=error_messages, ) self.test_case_results.append(test_result) @@ -329,9 +347,16 @@ class BpmnFileMissingExecutableProcessError(Exception): class ProcessModelTestRunnerService: - def __init__(self, process_model_directory_path: str) -> None: + def __init__( + self, + process_model_directory_path: str, + test_case_file: Optional[str] = None, + test_case_identifier: Optional[str] = None, + ) -> None: self.process_model_test_runner = ProcessModelTestRunner( process_model_directory_path, + test_case_file=test_case_file, + test_case_identifier=test_case_identifier, # instantiate_executer_callback=self._instantiate_executer_callback, # execute_task_callback=self._execute_task_callback, # get_next_task_callback=self._get_next_task_callback, diff --git a/spiffworkflow-backend/tests/data/bpmn_unit_test_process_models/expected-to-pass/multiple-test-files/a.bpmn b/spiffworkflow-backend/tests/data/bpmn_unit_test_process_models/expected-to-pass/multiple-test-files/a.bpmn new file mode 100644 index 000000000..6eb2e3313 --- /dev/null +++ b/spiffworkflow-backend/tests/data/bpmn_unit_test_process_models/expected-to-pass/multiple-test-files/a.bpmn @@ -0,0 +1,39 @@ + + + + + Flow_0jk46kf + + + + Flow_0pw6euz + + + + Flow_0jk46kf + Flow_0pw6euz + a = 1 + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/spiffworkflow-backend/tests/data/bpmn_unit_test_process_models/expected-to-pass/multiple-test-files/b.bpmn b/spiffworkflow-backend/tests/data/bpmn_unit_test_process_models/expected-to-pass/multiple-test-files/b.bpmn new file mode 100644 index 000000000..33eaa6084 --- /dev/null +++ b/spiffworkflow-backend/tests/data/bpmn_unit_test_process_models/expected-to-pass/multiple-test-files/b.bpmn @@ -0,0 +1,42 @@ + + + + + Flow_1qgv480 + + + + Flow_1sbj39z + + + + Flow_1qgv480 + Flow_1sbj39z + b = 1 + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/spiffworkflow-backend/tests/data/bpmn_unit_test_process_models/expected-to-pass/multiple-test-files/process_model.json b/spiffworkflow-backend/tests/data/bpmn_unit_test_process_models/expected-to-pass/multiple-test-files/process_model.json new file mode 100644 index 000000000..f8d1350e1 --- /dev/null +++ b/spiffworkflow-backend/tests/data/bpmn_unit_test_process_models/expected-to-pass/multiple-test-files/process_model.json @@ -0,0 +1,11 @@ +{ + "description": "", + "display_name": "Multiple Test Files", + "display_order": 0, + "exception_notification_addresses": [], + "fault_or_suspend_on_exception": "fault", + "files": [], + "metadata_extraction_paths": null, + "primary_file_name": "a.bpmn", + "primary_process_id": "ProcessA" +} \ No newline at end of file diff --git a/spiffworkflow-backend/tests/data/bpmn_unit_test_process_models/expected-to-pass/multiple-test-files/test_a.json b/spiffworkflow-backend/tests/data/bpmn_unit_test_process_models/expected-to-pass/multiple-test-files/test_a.json new file mode 100644 index 000000000..756b774ed --- /dev/null +++ b/spiffworkflow-backend/tests/data/bpmn_unit_test_process_models/expected-to-pass/multiple-test-files/test_a.json @@ -0,0 +1,5 @@ +{ + "test_case_1": { + "expected_output_json": { "a": 1 } + } +} \ No newline at end of file diff --git a/spiffworkflow-backend/tests/data/bpmn_unit_test_process_models/expected-to-pass/multiple-test-files/test_b.json b/spiffworkflow-backend/tests/data/bpmn_unit_test_process_models/expected-to-pass/multiple-test-files/test_b.json new file mode 100644 index 000000000..0cc7cf5f9 --- /dev/null +++ b/spiffworkflow-backend/tests/data/bpmn_unit_test_process_models/expected-to-pass/multiple-test-files/test_b.json @@ -0,0 +1,8 @@ +{ + "test_case_1": { + "expected_output_json": { "b": 1 } + }, + "test_case_2": { + "expected_output_json": { "b": 1 } + } +} \ No newline at end of file diff --git a/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_process_model_test_runner.py b/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_process_model_test_runner.py index 53abc61c7..c0fc92006 100644 --- a/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_process_model_test_runner.py +++ b/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_process_model_test_runner.py @@ -53,6 +53,24 @@ class TestProcessModelTestRunner(BaseTest): process_model_test_runner = self._run_model_tests(parent_directory="expected-to-fail") assert len(process_model_test_runner.test_case_results) == 1 + def test_can_test_process_model_with_multiple_files( + self, + app: Flask, + with_db_and_bpmn_file_cleanup: None, + with_mocked_root_path: Any, + ) -> None: + process_model_test_runner = self._run_model_tests(bpmn_process_directory_name="multiple-test-files") + assert len(process_model_test_runner.test_case_results) == 3 + + process_model_test_runner = self._run_model_tests(bpmn_process_directory_name="multiple-test-files", test_case_file='test_a.json') + assert len(process_model_test_runner.test_case_results) == 1 + + process_model_test_runner = self._run_model_tests(bpmn_process_directory_name="multiple-test-files", test_case_file='test_b.json') + assert len(process_model_test_runner.test_case_results) == 2 + + process_model_test_runner = self._run_model_tests(bpmn_process_directory_name="multiple-test-files", test_case_file='test_b.json', test_case_identifier='test_case_2') + assert len(process_model_test_runner.test_case_results) == 1 + def test_can_test_process_model_call_activity( self, app: Flask, @@ -81,7 +99,8 @@ class TestProcessModelTestRunner(BaseTest): assert len(process_model_test_runner.test_case_results) == 1 def _run_model_tests( - self, bpmn_process_directory_name: Optional[str] = None, parent_directory: str = "expected-to-pass" + self, bpmn_process_directory_name: Optional[str] = None, parent_directory: str = "expected-to-pass", + test_case_file: Optional[str] = None, test_case_identifier: Optional[str] = None, ) -> ProcessModelTestRunner: base_process_model_dir_path_segments = [FileSystemService.root_path(), parent_directory] path_segments = base_process_model_dir_path_segments @@ -90,6 +109,8 @@ class TestProcessModelTestRunner(BaseTest): process_model_test_runner = ProcessModelTestRunner( process_model_directory_path=os.path.join(*base_process_model_dir_path_segments), process_model_directory_for_test_discovery=os.path.join(*path_segments), + test_case_file=test_case_file, + test_case_identifier=test_case_identifier, ) process_model_test_runner.run() diff --git a/spiffworkflow-frontend/src/components/ProcessModelTestRun.tsx b/spiffworkflow-frontend/src/components/ProcessModelTestRun.tsx new file mode 100644 index 000000000..c67478579 --- /dev/null +++ b/spiffworkflow-frontend/src/components/ProcessModelTestRun.tsx @@ -0,0 +1,93 @@ +import { PlayOutline, Checkmark, Close } from '@carbon/icons-react'; +import { Button, Modal } from '@carbon/react'; +import { useState } from 'react'; +import { useUriListForPermissions } from '../hooks/UriListForPermissions'; +import HttpService from '../services/HttpService'; +import { ProcessFile } from '../interfaces'; + +type OwnProps = { + processModelFile: ProcessFile; +}; + +export default function ProcessModelTestRun({ processModelFile }: OwnProps) { + const [testCaseResults, setTestCaseResults] = useState(null); + const [showTestCaseResultsModal, setShowTestCaseResultsModal] = + useState(false); + const { targetUris } = useUriListForPermissions(); + + const onProcessModelTestRunSuccess = (result: any) => { + setTestCaseResults(result); + }; + + const processModelTestRunResultTag = () => { + if (testCaseResults) { + if (testCaseResults.all_passed) { + return ( +