From 18600189c89c6b4323581c0ad1b020e3f8ede928 Mon Sep 17 00:00:00 2001 From: jasquat <2487833+jasquat@users.noreply.github.com> Date: Tue, 5 Dec 2023 11:41:59 -0500 Subject: [PATCH] Feature/background proc with celery (#788) * WIP: some initial test code to test out celery w/ burnettk * some cleanup for celery and added base model to put tasks waiting on timers * removed dup bpmn file * some more cleanup and added strategy to queue instructions * some minor code changes w/ burnettk * remove the unused next_task key from api calls since nobody uses it w/ burnettk essweine * added migration for future tasks and added test to make sure we are inserting into it w/ burnettk essweine * ensure future task run at time can be updated w/ burnettk * added table to queue instructions for end user in w/ burnettk * added test to ensure we are storing instructions for end users w/ burnettk * added progress page to display new instructions to user * ignore dup instructions on db insert w/ burnettk * some more updates for celery w/ burnettk * some pyl and test fixes w/ burnettk * fixed tests w/ burnettk * WIP: added in page to show instructions on pi show page w/ burnettk * pi show page is fully using not interstitial now w/ burnettk * fixed broken test w/ burnettk * moved background processing items to own module w/ burnettk * fixed apscheduler start script * updated celery task queue to handle future tasks and upgraded black and set its line-length to match ruff w/ burnettk * added support to run future tasks using countdown w/ burnettk * build image for celery branch w/ burnettk * poet does not exist in the image w/ burnettk * start blocking scheduler should always start the scheduler w/ burnettk * add init and stuff for this branch * make this work not just on my mac * send other args to only * added running status for process instance and use that on fe to go to show page and added additional identifier to locking system to isolate celery workers better w/ burnettk * fixed typing error that typeguard found, not sure why mypy did not w/ burnettk * do not check for no instructions on interstitial page for cypress tests on frontend w/ burnettk * do not queue process instances twice w/ burnettk * removed bad file w/ burnettk * queue tasks using strings to avoid circular imports when attmepting to queue w/ burnettk * only queue imminent new timer events and mock celery * some keyboard shortcut support on frontend and added ability to force run a process instance over the api w/ burnettk * some styles added for the shortcut menu w/ burnettk * pyl w/ burnettk * fixed test w/ burnettk * removed temporary celery script and added support for celery worker in run server locally w/ burnettk * cleaned up migrations w/ burnettk * created new migration to clean up old migrations --------- Co-authored-by: jasquat Co-authored-by: burnettk --- .../docker_image_for_main_builds.yml | 2 +- .pre-commit-config.yaml | 2 +- poetry.lock | 47 +-- .../bin/data_migrations/run_all.py | 4 +- .../bin/import_tickets_for_command_line.py | 4 +- .../bin/local_development_environment_setup | 4 + .../bin/run_local_python_script | 4 +- .../bin/run_process_model_with_api | 5 + spiffworkflow-backend/bin/run_pydeps | 24 +- spiffworkflow-backend/bin/run_server_locally | 20 +- spiffworkflow-backend/bin/save_all_bpmn.py | 5 +- .../bin/start_blocking_appscheduler.py | 9 +- spiffworkflow-backend/bin/start_celery_worker | 13 + .../migrations/versions/441dca328887_.py | 62 ++++ spiffworkflow-backend/poetry.lock | 346 +++++++++++++++--- spiffworkflow-backend/pyproject.toml | 2 + .../src/spiffworkflow_backend/__init__.py | 177 +-------- .../src/spiffworkflow_backend/api.yml | 58 ++- .../background_processing/__init__.py | 3 + .../background_processing/apscheduler.py | 103 ++++++ .../background_processing_service.py | 40 +- .../background_processing/celery.py | 34 ++ .../celery_tasks/__init__.py | 0 .../celery_tasks/process_instance_task.py | 53 +++ .../process_instance_task_producer.py | 38 ++ .../background_processing/celery_worker.py | 7 + .../spiffworkflow_backend/config/__init__.py | 11 +- .../spiffworkflow_backend/config/default.py | 17 +- .../config/normalized_environment.py | 12 +- .../src/spiffworkflow_backend/config/qa2.py | 4 +- .../process_instance_migrator.py | 4 +- .../data_migrations/version_1_3.py | 16 +- .../data_migrations/version_2.py | 4 +- .../data_migrations/version_3.py | 4 +- .../spiffworkflow_backend/data_stores/json.py | 4 +- .../spiffworkflow_backend/data_stores/kkv.py | 6 +- .../data_stores/typeahead.py | 4 +- .../exceptions/api_error.py | 8 +- .../load_database_models.py | 3 + .../src/spiffworkflow_backend/models/db.py | 8 +- .../models/future_task.py | 48 +++ .../models/human_task.py | 4 +- .../spiffworkflow_backend/models/json_data.py | 4 +- .../models/message_instance.py | 8 +- .../models/process_group.py | 8 +- .../models/process_instance.py | 27 +- .../models/process_instance_file_data.py | 4 +- .../models/process_instance_metadata.py | 4 +- .../models/process_instance_queue.py | 4 +- .../models/process_instance_report.py | 4 +- .../models/reference_cache.py | 4 +- .../src/spiffworkflow_backend/models/task.py | 3 + .../models/task_draft_data.py | 4 +- .../models/task_instructions_for_end_user.py | 65 ++++ .../routes/active_users_controller.py | 8 +- .../routes/authentication_controller.py | 28 +- .../routes/data_store_controller.py | 4 +- .../routes/debug_controller.py | 2 +- .../routes/extensions_controller.py | 4 +- .../openid_blueprint/openid_blueprint.py | 4 +- .../routes/process_api_blueprint.py | 56 +++ .../routes/process_groups_controller.py | 4 +- .../process_instance_events_controller.py | 4 +- .../routes/process_instances_controller.py | 208 +++++------ .../routes/process_models_controller.py | 31 +- .../routes/script_unit_tests_controller.py | 12 +- .../routes/service_tasks_controller.py | 4 +- .../routes/tasks_controller.py | 165 +++++---- .../scripts/get_group_members.py | 4 +- .../get_url_for_task_with_bpmn_identifier.py | 3 +- .../scripts/markdown_file_download_link.py | 9 +- .../spiffworkflow_backend/scripts/script.py | 7 +- .../scripts/set_user_properties.py | 4 +- .../services/authentication_service.py | 53 +-- .../services/authorization_service.py | 50 +-- .../services/element_units_service.py | 8 +- .../services/file_system_service.py | 8 +- .../services/git_service.py | 18 +- .../services/jinja_service.py | 2 +- .../services/logging_service.py | 12 +- .../services/monitoring_service.py | 108 ++++++ .../services/process_caller_service.py | 8 +- .../services/process_instance_lock_service.py | 46 ++- .../services/process_instance_processor.py | 113 ++---- .../process_instance_queue_service.py | 30 +- .../process_instance_report_service.py | 58 +-- .../services/process_instance_service.py | 42 ++- .../services/process_model_service.py | 27 +- .../process_model_test_runner_service.py | 12 +- .../services/reference_cache_service.py | 4 +- .../services/service_task_service.py | 11 +- .../services/spec_file_service.py | 20 +- .../services/task_service.py | 68 +--- .../services/user_service.py | 18 +- .../services/workflow_execution_service.py | 152 ++++++-- .../script_task_with_instruction.bpmn | 59 +++ .../user_task_with_timer.bpmn | 69 ++++ .../helpers/base_test.py | 8 +- .../integration/test_authentication.py | 4 +- .../integration/test_for_good_errors.py | 4 +- .../integration/test_logging_service.py | 16 +- .../integration/test_onboarding.py | 4 +- .../integration/test_process_api.py | 104 +++--- .../integration/test_tasks_controller.py | 4 +- .../scripts/test_get_all_permissions.py | 4 +- .../scripts/test_get_current_task_info.py | 8 +- .../scripts/test_get_group_members.py | 4 +- .../test_get_last_user_completing_task.py | 12 +- .../scripts/test_get_localtime.py | 12 +- .../test_get_process_initiator_user.py | 8 +- ...t_get_url_for_task_with_bpmn_identifier.py | 8 +- .../scripts/test_refresh_permissions.py | 8 +- .../unit/test_authorization_service.py | 32 +- .../unit/test_dot_notation.py | 4 +- .../unit/test_future_task.py | 47 +++ .../unit/test_permissions.py | 8 +- .../unit/test_process_caller_service.py | 4 +- .../unit/test_process_instance_processor.py | 162 ++++---- .../test_process_instance_queue_service.py | 4 +- .../test_process_instance_report_service.py | 4 +- .../unit/test_script_unit_test_runner.py | 16 +- .../unit/test_service_task_delegate.py | 12 +- .../unit/test_task_service.py | 4 +- .../cypress/support/commands.js | 8 +- .../src/components/ErrorDisplay.tsx | 29 +- .../src/components/InstructionsForEndUser.tsx | 56 ++- .../ProcessInstanceCurrentTaskInfo.tsx | 113 ++++++ .../components/ProcessInstanceListTable.tsx | 13 +- .../components/ProcessInstanceProgress.tsx | 174 +++++++++ .../src/components/ProcessInstanceRun.tsx | 19 +- spiffworkflow-frontend/src/config.tsx | 7 +- .../src/hooks/useKeyboardShortcut.tsx | 150 ++++++++ spiffworkflow-frontend/src/index.css | 24 ++ spiffworkflow-frontend/src/interfaces.ts | 31 +- .../routes/ProcessInstanceProgressPage.tsx | 39 ++ .../src/routes/ProcessInstanceRoutes.tsx | 9 + .../src/routes/ProcessInstanceShow.tsx | 47 ++- .../src/routes/TaskShow.tsx | 16 +- 138 files changed, 2676 insertions(+), 1483 deletions(-) create mode 100755 spiffworkflow-backend/bin/start_celery_worker create mode 100644 spiffworkflow-backend/migrations/versions/441dca328887_.py create mode 100644 spiffworkflow-backend/src/spiffworkflow_backend/background_processing/__init__.py create mode 100644 spiffworkflow-backend/src/spiffworkflow_backend/background_processing/apscheduler.py rename spiffworkflow-backend/src/spiffworkflow_backend/{services => background_processing}/background_processing_service.py (53%) create mode 100644 spiffworkflow-backend/src/spiffworkflow_backend/background_processing/celery.py create mode 100644 spiffworkflow-backend/src/spiffworkflow_backend/background_processing/celery_tasks/__init__.py create mode 100644 spiffworkflow-backend/src/spiffworkflow_backend/background_processing/celery_tasks/process_instance_task.py create mode 100644 spiffworkflow-backend/src/spiffworkflow_backend/background_processing/celery_tasks/process_instance_task_producer.py create mode 100644 spiffworkflow-backend/src/spiffworkflow_backend/background_processing/celery_worker.py create mode 100644 spiffworkflow-backend/src/spiffworkflow_backend/models/future_task.py create mode 100644 spiffworkflow-backend/src/spiffworkflow_backend/models/task_instructions_for_end_user.py create mode 100644 spiffworkflow-backend/src/spiffworkflow_backend/services/monitoring_service.py create mode 100644 spiffworkflow-backend/tests/data/script-task-with-instruction/script_task_with_instruction.bpmn create mode 100644 spiffworkflow-backend/tests/data/user-task-with-timer/user_task_with_timer.bpmn create mode 100644 spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_future_task.py create mode 100644 spiffworkflow-frontend/src/components/ProcessInstanceCurrentTaskInfo.tsx create mode 100644 spiffworkflow-frontend/src/components/ProcessInstanceProgress.tsx create mode 100644 spiffworkflow-frontend/src/hooks/useKeyboardShortcut.tsx create mode 100644 spiffworkflow-frontend/src/routes/ProcessInstanceProgressPage.tsx diff --git a/.github/workflows/docker_image_for_main_builds.yml b/.github/workflows/docker_image_for_main_builds.yml index 7a722932..7f1473e1 100644 --- a/.github/workflows/docker_image_for_main_builds.yml +++ b/.github/workflows/docker_image_for_main_builds.yml @@ -32,7 +32,7 @@ on: branches: - main - spiffdemo - - feature/no-data-for-finished-spiff-tasks + - feature/background-proc-with-celery jobs: create_frontend_docker_image: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 411190af..c4178fe8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,7 +18,7 @@ repos: # --line-length because then we can avoid the fancy line wrapping in more instances and jason, kb, and elizabeth # kind of prefer long lines rather than cutely-formatted sets of lines. # TODO: enable when its safe to update the files - args: [--preview, --line-length, "119"] + args: [--preview, --line-length, "130"] - id: check-added-large-files files: ^spiffworkflow-backend/ diff --git a/poetry.lock b/poetry.lock index 617cce99..92159dd3 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,10 +1,9 @@ -# This file is automatically @generated by Poetry 1.4.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. [[package]] name = "attrs" version = "22.2.0" description = "Classes Without Boilerplate" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -23,7 +22,6 @@ tests-no-zope = ["cloudpickle", "cloudpickle", "hypothesis", "hypothesis", "mypy name = "bandit" version = "1.7.2" description = "Security oriented static analyser for python code." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -46,7 +44,6 @@ yaml = ["PyYAML"] name = "black" version = "23.1.0" description = "The uncompromising code formatter." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -94,7 +91,6 @@ uvloop = ["uvloop (>=0.15.2)"] name = "cfgv" version = "3.3.1" description = "Validate configuration and produce human readable error messages." -category = "dev" optional = false python-versions = ">=3.6.1" files = [ @@ -106,7 +102,6 @@ files = [ name = "classify-imports" version = "4.2.0" description = "Utilities for refactoring imports in python-like syntax." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -118,7 +113,6 @@ files = [ name = "click" version = "8.1.3" description = "Composable command line interface toolkit" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -133,7 +127,6 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""} name = "colorama" version = "0.4.6" description = "Cross-platform colored terminal text." -category = "dev" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" files = [ @@ -145,7 +138,6 @@ files = [ name = "distlib" version = "0.3.6" description = "Distribution utilities" -category = "dev" optional = false python-versions = "*" files = [ @@ -157,7 +149,6 @@ files = [ name = "docutils" version = "0.19" description = "Docutils -- Python Documentation Utilities" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -169,7 +160,6 @@ files = [ name = "filelock" version = "3.10.7" description = "A platform independent file lock." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -185,7 +175,6 @@ testing = ["covdefaults (>=2.3)", "coverage (>=7.2.2)", "diff-cover (>=7.5)", "p name = "flake8" version = "4.0.1" description = "the modular source code checker: pep8 pyflakes and co" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -202,7 +191,6 @@ pyflakes = ">=2.4.0,<2.5.0" name = "flake8-bandit" version = "2.1.2" description = "Automated security testing with bandit and flake8." -category = "dev" optional = false python-versions = "*" files = [ @@ -219,7 +207,6 @@ pycodestyle = "*" name = "flake8-bugbear" version = "22.12.6" description = "A plugin for flake8 finding likely bugs and design problems in your program. Contains warnings that don't belong in pyflakes and pycodestyle." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -238,7 +225,6 @@ dev = ["coverage", "hypothesis", "hypothesmith (>=0.2)", "pre-commit", "tox"] name = "flake8-docstrings" version = "1.7.0" description = "Extension for flake8 which uses pydocstyle to check docstrings" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -254,7 +240,6 @@ pydocstyle = ">=2.1" name = "flake8-polyfill" version = "1.0.2" description = "Polyfill package for Flake8 plugins" -category = "dev" optional = false python-versions = "*" files = [ @@ -269,7 +254,6 @@ flake8 = "*" name = "flake8-rst-docstrings" version = "0.2.7" description = "Python docstring reStructuredText (RST) validator" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -286,7 +270,6 @@ restructuredtext-lint = "*" name = "gitdb" version = "4.0.10" description = "Git Object Database" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -301,7 +284,6 @@ smmap = ">=3.0.1,<6" name = "gitpython" version = "3.1.36" description = "GitPython is a Python library used to interact with Git repositories" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -319,7 +301,6 @@ test = ["black", "coverage[toml]", "ddt (>=1.1.1,!=1.4.3)", "mypy", "pre-commit" name = "identify" version = "2.5.22" description = "File identification library for Python" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -334,7 +315,6 @@ license = ["ukkonen"] name = "mccabe" version = "0.6.1" description = "McCabe checker, plugin for flake8" -category = "dev" optional = false python-versions = "*" files = [ @@ -346,7 +326,6 @@ files = [ name = "mypy-extensions" version = "1.0.0" description = "Type system extensions for programs checked with the mypy type checker." -category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -358,7 +337,6 @@ files = [ name = "nodeenv" version = "1.7.0" description = "Node.js virtual environment builder" -category = "dev" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*" files = [ @@ -373,7 +351,6 @@ setuptools = "*" name = "packaging" version = "23.0" description = "Core utilities for Python packages" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -385,7 +362,6 @@ files = [ name = "pathspec" version = "0.11.1" description = "Utility library for gitignore style pattern matching of file paths." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -397,7 +373,6 @@ files = [ name = "pbr" version = "5.11.1" description = "Python Build Reasonableness" -category = "dev" optional = false python-versions = ">=2.6" files = [ @@ -409,7 +384,6 @@ files = [ name = "platformdirs" version = "3.2.0" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -425,7 +399,6 @@ test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.2.2)", "pytest- name = "pre-commit" version = "2.21.0" description = "A framework for managing and maintaining multi-language pre-commit hooks." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -444,7 +417,6 @@ virtualenv = ">=20.10.0" name = "pre-commit-hooks" version = "4.4.0" description = "Some out-of-the-box hooks for pre-commit." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -459,7 +431,6 @@ files = [ name = "pycodestyle" version = "2.8.0" description = "Python style guide checker" -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ @@ -471,7 +442,6 @@ files = [ name = "pydocstyle" version = "6.3.0" description = "Python docstring style checker" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -489,7 +459,6 @@ toml = ["tomli (>=1.2.3)"] name = "pyflakes" version = "2.4.0" description = "passive checker of Python programs" -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -501,7 +470,6 @@ files = [ name = "pygments" version = "2.14.0" description = "Pygments is a syntax highlighting package written in Python." -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -516,7 +484,6 @@ plugins = ["importlib-metadata"] name = "pyupgrade" version = "3.3.1" description = "A tool to automatically upgrade syntax for newer versions." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -531,7 +498,6 @@ tokenize-rt = ">=3.2.0" name = "pyyaml" version = "6.0" description = "YAML parser and emitter for Python" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -581,7 +547,6 @@ files = [ name = "reorder-python-imports" version = "3.9.0" description = "Tool for reordering python imports" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -596,7 +561,6 @@ classify-imports = ">=4.1" name = "restructuredtext-lint" version = "1.4.0" description = "reStructuredText linter" -category = "dev" optional = false python-versions = "*" files = [ @@ -610,7 +574,6 @@ docutils = ">=0.11,<1.0" name = "ruamel-yaml" version = "0.17.21" description = "ruamel.yaml is a YAML parser/emitter that supports roundtrip preservation of comments, seq/map flow style, and map key order" -category = "dev" optional = false python-versions = ">=3" files = [ @@ -626,7 +589,6 @@ jinja2 = ["ruamel.yaml.jinja2 (>=0.2)"] name = "ruff" version = "0.0.270" description = "An extremely fast Python linter, written in Rust." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -653,7 +615,6 @@ files = [ name = "setuptools" version = "67.6.1" description = "Easily download, build, install, upgrade, and uninstall Python packages" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -670,7 +631,6 @@ testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs ( name = "smmap" version = "5.0.0" description = "A pure Python implementation of a sliding window memory map manager" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -682,7 +642,6 @@ files = [ name = "snowballstemmer" version = "2.2.0" description = "This package provides 29 stemmers for 28 languages generated from Snowball algorithms." -category = "dev" optional = false python-versions = "*" files = [ @@ -694,7 +653,6 @@ files = [ name = "stevedore" version = "5.0.0" description = "Manage dynamic plugins for Python applications" -category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -709,7 +667,6 @@ pbr = ">=2.0.0,<2.1.0 || >2.1.0" name = "tokenize-rt" version = "5.0.0" description = "A wrapper around the stdlib `tokenize` which roundtrips." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -721,7 +678,6 @@ files = [ name = "tomli" version = "2.0.1" description = "A lil' TOML parser" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -733,7 +689,6 @@ files = [ name = "virtualenv" version = "20.21.0" description = "Virtual Python Environment builder" -category = "dev" optional = false python-versions = ">=3.7" files = [ diff --git a/spiffworkflow-backend/bin/data_migrations/run_all.py b/spiffworkflow-backend/bin/data_migrations/run_all.py index 5495ad2a..93141bca 100644 --- a/spiffworkflow-backend/bin/data_migrations/run_all.py +++ b/spiffworkflow-backend/bin/data_migrations/run_all.py @@ -67,9 +67,7 @@ def main() -> None: put_serializer_version_onto_numeric_track() process_instances = all_potentially_relevant_process_instances() potentially_relevant_instance_count = len(process_instances) - current_app.logger.debug( - f"Found potentially relevant process_instances: {potentially_relevant_instance_count}" - ) + current_app.logger.debug(f"Found potentially relevant process_instances: {potentially_relevant_instance_count}") if potentially_relevant_instance_count > 0: run_version_1() # this will run while using the new per instance on demand data migration framework diff --git a/spiffworkflow-backend/bin/import_tickets_for_command_line.py b/spiffworkflow-backend/bin/import_tickets_for_command_line.py index bea05a61..6dec5db0 100644 --- a/spiffworkflow-backend/bin/import_tickets_for_command_line.py +++ b/spiffworkflow-backend/bin/import_tickets_for_command_line.py @@ -20,9 +20,7 @@ def main(): db.session.commit() """Print process instance count.""" - process_instances = ProcessInstanceModel.query.filter_by( - process_model_identifier=process_model_identifier_ticket - ).all() + process_instances = ProcessInstanceModel.query.filter_by(process_model_identifier=process_model_identifier_ticket).all() process_instance_count = len(process_instances) print(f"process_instance_count: {process_instance_count}") diff --git a/spiffworkflow-backend/bin/local_development_environment_setup b/spiffworkflow-backend/bin/local_development_environment_setup index 95286349..78c60fe4 100755 --- a/spiffworkflow-backend/bin/local_development_environment_setup +++ b/spiffworkflow-backend/bin/local_development_environment_setup @@ -15,6 +15,10 @@ fi port="${SPIFFWORKFLOW_BACKEND_PORT:-7000}" process_model_dir="${1:-}" +if [[ -d "$process_model_dir" ]]; then + shift +fi + if [[ -z "${SPIFFWORKFLOW_BACKEND_BPMN_SPEC_ABSOLUTE_DIR:-}" ]]; then if [[ -n "${process_model_dir}" ]] && [[ -d "${process_model_dir}" ]]; then diff --git a/spiffworkflow-backend/bin/run_local_python_script b/spiffworkflow-backend/bin/run_local_python_script index f2b97d4d..fc21c34f 100755 --- a/spiffworkflow-backend/bin/run_local_python_script +++ b/spiffworkflow-backend/bin/run_local_python_script @@ -7,9 +7,11 @@ function error_handler() { trap 'error_handler ${LINENO} $?' ERR set -o errtrace -o errexit -o nounset -o pipefail +script="$1" +shift script_dir="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" . "${script_dir}/local_development_environment_setup" export SPIFFWORKFLOW_BACKEND_RUN_BACKGROUND_SCHEDULER_IN_CREATE_APP=false -poet run python "$@" +poet run python "$script" diff --git a/spiffworkflow-backend/bin/run_process_model_with_api b/spiffworkflow-backend/bin/run_process_model_with_api index 3886f3ae..85f037ca 100755 --- a/spiffworkflow-backend/bin/run_process_model_with_api +++ b/spiffworkflow-backend/bin/run_process_model_with_api @@ -66,5 +66,10 @@ fi result=$(curl --silent -X POST "${BACKEND_BASE_URL}/v1.0/process-instances/${modified_process_model_identifier}/${process_instance_id}/run" -H "Authorization: Bearer $access_token") check_result_for_error "$result" +if [[ "$(jq -r '.status' <<<"$result")" == "complete" ]]; then + echo "Process instance completed" + exit 0 +fi + next_task=$(jq '.next_task' <<<"$result") process_next_task "$next_task" diff --git a/spiffworkflow-backend/bin/run_pydeps b/spiffworkflow-backend/bin/run_pydeps index bb5db611..2675f273 100755 --- a/spiffworkflow-backend/bin/run_pydeps +++ b/spiffworkflow-backend/bin/run_pydeps @@ -12,9 +12,27 @@ if ! command -v pydeps >/dev/null 2>&1; then pip install pydeps fi -more_args='' +pydeps_args=() + if [[ "${1:-}" == "r" ]]; then - more_args='--rankdir LR' + shift + pydeps_args+=("--rankdir" "LR") fi -pydeps src/spiffworkflow_backend --display "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" --only spiffworkflow_backend --rmprefix spiffworkflow_backend. --exclude-exact spiffworkflow_backend.services.custom_parser spiffworkflow_backend.specs spiffworkflow_backend.services spiffworkflow_backend spiffworkflow_backend.models spiffworkflow_backend.load_database_models spiffworkflow_backend.routes --exclude spiffworkflow_backend.config spiffworkflow_backend.interfaces spiffworkflow_backend.models.db $more_args +# add other args to only +# example usage: +# ./bin/run_pydeps spiffworkflow_backend.services.process_instance_processor spiffworkflow_backend.services.process_instance_service spiffworkflow_backend.background_processing spiffworkflow_backend.routes.tasks_controller spiffworkflow_backend.services.workflow_execution_service +if [[ -n "${1:-}" ]]; then + pydeps_args+=("--only") + for arg in "$@"; do + pydeps_args+=("$arg") + done +fi + +if [[ -f "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" ]]; then + pydeps_args+=("--display" "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome") +fi + + +# the only at the end is specific to this branch +pydeps src/spiffworkflow_backend --only spiffworkflow_backend --rmprefix spiffworkflow_backend. --exclude-exact spiffworkflow_backend.background_processing spiffworkflow_backend.services.custom_parser spiffworkflow_backend.specs spiffworkflow_backend.services spiffworkflow_backend spiffworkflow_backend.models spiffworkflow_backend.load_database_models spiffworkflow_backend.routes --exclude spiffworkflow_backend.config spiffworkflow_backend.interfaces spiffworkflow_backend.models.db "${pydeps_args[@]}" diff --git a/spiffworkflow-backend/bin/run_server_locally b/spiffworkflow-backend/bin/run_server_locally index 43f23f90..970cd6f6 100755 --- a/spiffworkflow-backend/bin/run_server_locally +++ b/spiffworkflow-backend/bin/run_server_locally @@ -10,15 +10,21 @@ set -o errtrace -o errexit -o nounset -o pipefail script_dir="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" . "${script_dir}/local_development_environment_setup" -if [[ -n "${SPIFFWORKFLOW_BACKEND_LOAD_FIXTURE_DATA:-}" ]]; then - ./bin/boot_server_in_docker -else - export FLASK_DEBUG=1 +server_type="${1:-api}" - if [[ "${SPIFFWORKFLOW_BACKEND_RUN_DATA_SETUP:-}" != "false" ]]; then - SPIFFWORKFLOW_BACKEND_RUN_BACKGROUND_SCHEDULER_IN_CREATE_APP=false SPIFFWORKFLOW_BACKEND_FAIL_ON_INVALID_PROCESS_MODELS=false poetry run python bin/save_all_bpmn.py - fi +if [[ "$server_type" == "celery_worker" ]]; then + "${script_dir}/start_celery_worker" +else + if [[ -n "${SPIFFWORKFLOW_BACKEND_LOAD_FIXTURE_DATA:-}" ]]; then + ./bin/boot_server_in_docker + else + export FLASK_DEBUG=1 + + if [[ "${SPIFFWORKFLOW_BACKEND_RUN_DATA_SETUP:-}" != "false" ]]; then + SPIFFWORKFLOW_BACKEND_RUN_BACKGROUND_SCHEDULER_IN_CREATE_APP=false SPIFFWORKFLOW_BACKEND_FAIL_ON_INVALID_PROCESS_MODELS=false poetry run python bin/save_all_bpmn.py + fi # this line blocks poetry run flask run -p "$port" --host=0.0.0.0 + fi fi diff --git a/spiffworkflow-backend/bin/save_all_bpmn.py b/spiffworkflow-backend/bin/save_all_bpmn.py index 95a181e8..dfa2350b 100644 --- a/spiffworkflow-backend/bin/save_all_bpmn.py +++ b/spiffworkflow-backend/bin/save_all_bpmn.py @@ -12,10 +12,7 @@ def main() -> None: failing_process_models = DataSetupService.save_all_process_models() for bpmn_errors in failing_process_models: print(bpmn_errors) - if ( - os.environ.get("SPIFFWORKFLOW_BACKEND_FAIL_ON_INVALID_PROCESS_MODELS") != "false" - and len(failing_process_models) > 0 - ): + if os.environ.get("SPIFFWORKFLOW_BACKEND_FAIL_ON_INVALID_PROCESS_MODELS") != "false" and len(failing_process_models) > 0: exit(1) diff --git a/spiffworkflow-backend/bin/start_blocking_appscheduler.py b/spiffworkflow-backend/bin/start_blocking_appscheduler.py index 89a40b44..bf32997f 100755 --- a/spiffworkflow-backend/bin/start_blocking_appscheduler.py +++ b/spiffworkflow-backend/bin/start_blocking_appscheduler.py @@ -3,7 +3,7 @@ import time from apscheduler.schedulers.background import BlockingScheduler # type: ignore from spiffworkflow_backend import create_app -from spiffworkflow_backend import start_scheduler +from spiffworkflow_backend.background_processing.apscheduler import start_apscheduler from spiffworkflow_backend.data_migrations.version_1_3 import VersionOneThree from spiffworkflow_backend.helpers.db_helper import try_to_connect @@ -23,11 +23,8 @@ def main() -> None: VersionOneThree().run() end_time = time.time() - print( - f"done running data migration from background processor. took {end_time - start_time} seconds. starting" - " scheduler" - ) - start_scheduler(app, BlockingScheduler) + print(f"done running data migration from background processor. took {end_time - start_time} seconds. starting scheduler") + start_apscheduler(app, BlockingScheduler) if __name__ == "__main__": diff --git a/spiffworkflow-backend/bin/start_celery_worker b/spiffworkflow-backend/bin/start_celery_worker new file mode 100755 index 00000000..1ea431c5 --- /dev/null +++ b/spiffworkflow-backend/bin/start_celery_worker @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +function error_handler() { + >&2 echo "Exited with BAD EXIT CODE '${2}' in ${0} script at line: ${1}." + exit "$2" +} +trap 'error_handler ${LINENO} $?' ERR +set -o errtrace -o errexit -o nounset -o pipefail + +export SPIFFWORKFLOW_BACKEND_CELERY_ENABLED=true +export SPIFFWORKFLOW_BACKEND_RUN_BACKGROUND_SCHEDULER_IN_CREATE_APP=false + +poetry run celery -A src.spiffworkflow_backend.background_processing.celery_worker worker --loglevel=info -c 12 diff --git a/spiffworkflow-backend/migrations/versions/441dca328887_.py b/spiffworkflow-backend/migrations/versions/441dca328887_.py new file mode 100644 index 00000000..982c19bf --- /dev/null +++ b/spiffworkflow-backend/migrations/versions/441dca328887_.py @@ -0,0 +1,62 @@ +"""empty message + +Revision ID: 441dca328887 +Revises: 1b5a9f7af28e +Create Date: 2023-12-05 10:36:32.487659 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '441dca328887' +down_revision = '1b5a9f7af28e' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('future_task', + sa.Column('guid', sa.String(length=36), nullable=False), + sa.Column('run_at_in_seconds', sa.Integer(), nullable=False), + sa.Column('completed', sa.Boolean(), nullable=False), + sa.Column('updated_at_in_seconds', sa.Integer(), nullable=False), + sa.PrimaryKeyConstraint('guid') + ) + with op.batch_alter_table('future_task', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_future_task_completed'), ['completed'], unique=False) + batch_op.create_index(batch_op.f('ix_future_task_run_at_in_seconds'), ['run_at_in_seconds'], unique=False) + + op.create_table('task_instructions_for_end_user', + sa.Column('task_guid', sa.String(length=36), nullable=False), + sa.Column('instruction', sa.Text(), nullable=False), + sa.Column('process_instance_id', sa.Integer(), nullable=False), + sa.Column('has_been_retrieved', sa.Boolean(), nullable=False), + sa.Column('timestamp', sa.DECIMAL(precision=17, scale=6), nullable=False), + sa.ForeignKeyConstraint(['process_instance_id'], ['process_instance.id'], ), + sa.PrimaryKeyConstraint('task_guid') + ) + with op.batch_alter_table('task_instructions_for_end_user', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_task_instructions_for_end_user_has_been_retrieved'), ['has_been_retrieved'], unique=False) + batch_op.create_index(batch_op.f('ix_task_instructions_for_end_user_process_instance_id'), ['process_instance_id'], unique=False) + batch_op.create_index(batch_op.f('ix_task_instructions_for_end_user_timestamp'), ['timestamp'], unique=False) + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('task_instructions_for_end_user', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_task_instructions_for_end_user_timestamp')) + batch_op.drop_index(batch_op.f('ix_task_instructions_for_end_user_process_instance_id')) + batch_op.drop_index(batch_op.f('ix_task_instructions_for_end_user_has_been_retrieved')) + + op.drop_table('task_instructions_for_end_user') + with op.batch_alter_table('future_task', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_future_task_run_at_in_seconds')) + batch_op.drop_index(batch_op.f('ix_future_task_completed')) + + op.drop_table('future_task') + # ### end Alembic commands ### diff --git a/spiffworkflow-backend/poetry.lock b/spiffworkflow-backend/poetry.lock index 69f8a731..9e80fc11 100644 --- a/spiffworkflow-backend/poetry.lock +++ b/spiffworkflow-backend/poetry.lock @@ -19,6 +19,20 @@ typing-extensions = ">=4" [package.extras] tz = ["python-dateutil"] +[[package]] +name = "amqp" +version = "5.2.0" +description = "Low-level AMQP client for Python (fork of amqplib)." +optional = false +python-versions = ">=3.6" +files = [ + {file = "amqp-5.2.0-py3-none-any.whl", hash = "sha256:827cb12fb0baa892aad844fd95258143bce4027fdac4fccddbc43330fd281637"}, + {file = "amqp-5.2.0.tar.gz", hash = "sha256:a1ecff425ad063ad42a486c902807d1482311481c8ad95a72694b2975e75f7fd"}, +] + +[package.dependencies] +vine = ">=5.0.0,<6.0.0" + [[package]] name = "aniso8601" version = "9.0.1" @@ -62,6 +76,17 @@ tornado = ["tornado (>=4.3)"] twisted = ["twisted"] zookeeper = ["kazoo"] +[[package]] +name = "async-timeout" +version = "4.0.3" +description = "Timeout context manager for asyncio programs" +optional = false +python-versions = ">=3.7" +files = [ + {file = "async-timeout-4.0.3.tar.gz", hash = "sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f"}, + {file = "async_timeout-4.0.3-py3-none-any.whl", hash = "sha256:7405140ff1230c310e51dc27b3145b9092d659ce68ff733fb0cefe3ee42be028"}, +] + [[package]] name = "attrs" version = "22.2.0" @@ -137,32 +162,51 @@ tests = ["pytest (>=3.2.1,!=3.3.0)"] typecheck = ["mypy"] [[package]] -name = "black" -version = "22.12.0" -description = "The uncompromising code formatter." +name = "billiard" +version = "4.2.0" +description = "Python multiprocessing fork with improvements and bugfixes" optional = false python-versions = ">=3.7" files = [ - {file = "black-22.12.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9eedd20838bd5d75b80c9f5487dbcb06836a43833a37846cf1d8c1cc01cef59d"}, - {file = "black-22.12.0-cp310-cp310-win_amd64.whl", hash = "sha256:159a46a4947f73387b4d83e87ea006dbb2337eab6c879620a3ba52699b1f4351"}, - {file = "black-22.12.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d30b212bffeb1e252b31dd269dfae69dd17e06d92b87ad26e23890f3efea366f"}, - {file = "black-22.12.0-cp311-cp311-win_amd64.whl", hash = "sha256:7412e75863aa5c5411886804678b7d083c7c28421210180d67dfd8cf1221e1f4"}, - {file = "black-22.12.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c116eed0efb9ff870ded8b62fe9f28dd61ef6e9ddd28d83d7d264a38417dcee2"}, - {file = "black-22.12.0-cp37-cp37m-win_amd64.whl", hash = "sha256:1f58cbe16dfe8c12b7434e50ff889fa479072096d79f0a7f25e4ab8e94cd8350"}, - {file = "black-22.12.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:77d86c9f3db9b1bf6761244bc0b3572a546f5fe37917a044e02f3166d5aafa7d"}, - {file = "black-22.12.0-cp38-cp38-win_amd64.whl", hash = "sha256:82d9fe8fee3401e02e79767016b4907820a7dc28d70d137eb397b92ef3cc5bfc"}, - {file = "black-22.12.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:101c69b23df9b44247bd88e1d7e90154336ac4992502d4197bdac35dd7ee3320"}, - {file = "black-22.12.0-cp39-cp39-win_amd64.whl", hash = "sha256:559c7a1ba9a006226f09e4916060982fd27334ae1998e7a38b3f33a37f7a2148"}, - {file = "black-22.12.0-py3-none-any.whl", hash = "sha256:436cc9167dd28040ad90d3b404aec22cedf24a6e4d7de221bec2730ec0c97bcf"}, - {file = "black-22.12.0.tar.gz", hash = "sha256:229351e5a18ca30f447bf724d007f890f97e13af070bb6ad4c0a441cd7596a2f"}, + {file = "billiard-4.2.0-py3-none-any.whl", hash = "sha256:07aa978b308f334ff8282bd4a746e681b3513db5c9a514cbdd810cbbdc19714d"}, + {file = "billiard-4.2.0.tar.gz", hash = "sha256:9a3c3184cb275aa17a732f93f65b20c525d3d9f253722d26a82194803ade5a2c"}, +] + +[[package]] +name = "black" +version = "23.11.0" +description = "The uncompromising code formatter." +optional = false +python-versions = ">=3.8" +files = [ + {file = "black-23.11.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:dbea0bb8575c6b6303cc65017b46351dc5953eea5c0a59d7b7e3a2d2f433a911"}, + {file = "black-23.11.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:412f56bab20ac85927f3a959230331de5614aecda1ede14b373083f62ec24e6f"}, + {file = "black-23.11.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d136ef5b418c81660ad847efe0e55c58c8208b77a57a28a503a5f345ccf01394"}, + {file = "black-23.11.0-cp310-cp310-win_amd64.whl", hash = "sha256:6c1cac07e64433f646a9a838cdc00c9768b3c362805afc3fce341af0e6a9ae9f"}, + {file = "black-23.11.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cf57719e581cfd48c4efe28543fea3d139c6b6f1238b3f0102a9c73992cbb479"}, + {file = "black-23.11.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:698c1e0d5c43354ec5d6f4d914d0d553a9ada56c85415700b81dc90125aac244"}, + {file = "black-23.11.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:760415ccc20f9e8747084169110ef75d545f3b0932ee21368f63ac0fee86b221"}, + {file = "black-23.11.0-cp311-cp311-win_amd64.whl", hash = "sha256:58e5f4d08a205b11800332920e285bd25e1a75c54953e05502052738fe16b3b5"}, + {file = "black-23.11.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:45aa1d4675964946e53ab81aeec7a37613c1cb71647b5394779e6efb79d6d187"}, + {file = "black-23.11.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4c44b7211a3a0570cc097e81135faa5f261264f4dfaa22bd5ee2875a4e773bd6"}, + {file = "black-23.11.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2a9acad1451632021ee0d146c8765782a0c3846e0e0ea46659d7c4f89d9b212b"}, + {file = "black-23.11.0-cp38-cp38-win_amd64.whl", hash = "sha256:fc7f6a44d52747e65a02558e1d807c82df1d66ffa80a601862040a43ec2e3142"}, + {file = "black-23.11.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7f622b6822f02bfaf2a5cd31fdb7cd86fcf33dab6ced5185c35f5db98260b055"}, + {file = "black-23.11.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:250d7e60f323fcfc8ea6c800d5eba12f7967400eb6c2d21ae85ad31c204fb1f4"}, + {file = "black-23.11.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5133f5507007ba08d8b7b263c7aa0f931af5ba88a29beacc4b2dc23fcefe9c06"}, + {file = "black-23.11.0-cp39-cp39-win_amd64.whl", hash = "sha256:421f3e44aa67138ab1b9bfbc22ee3780b22fa5b291e4db8ab7eee95200726b07"}, + {file = "black-23.11.0-py3-none-any.whl", hash = "sha256:54caaa703227c6e0c87b76326d0862184729a69b73d3b7305b6288e1d830067e"}, + {file = "black-23.11.0.tar.gz", hash = "sha256:4c68855825ff432d197229846f971bc4d6666ce90492e5b02013bcaca4d9ab05"}, ] [package.dependencies] click = ">=8.0.0" mypy-extensions = ">=0.4.3" +packaging = ">=22.0" pathspec = ">=0.9.0" platformdirs = ">=2" -tomli = {version = ">=1.1.0", markers = "python_full_version < \"3.11.0a7\""} +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.0.1", markers = "python_version < \"3.11\""} [package.extras] colorama = ["colorama (>=0.4.3)"] @@ -192,6 +236,77 @@ files = [ {file = "cachelib-0.10.2.tar.gz", hash = "sha256:593faeee62a7c037d50fc835617a01b887503f972fb52b188ae7e50e9cb69740"}, ] +[[package]] +name = "celery" +version = "5.3.5" +description = "Distributed Task Queue." +optional = false +python-versions = ">=3.8" +files = [ + {file = "celery-5.3.5-py3-none-any.whl", hash = "sha256:30b75ac60fb081c2d9f8881382c148ed7c9052031a75a1e8743ff4b4b071f184"}, + {file = "celery-5.3.5.tar.gz", hash = "sha256:6b65d8dd5db499dd6190c45aa6398e171b99592f2af62c312f7391587feb5458"}, +] + +[package.dependencies] +billiard = ">=4.2.0,<5.0" +click = ">=8.1.2,<9.0" +click-didyoumean = ">=0.3.0" +click-plugins = ">=1.1.1" +click-repl = ">=0.2.0" +kombu = ">=5.3.3,<6.0" +python-dateutil = ">=2.8.2" +redis = {version = ">=4.5.2,<4.5.5 || >4.5.5,<6.0.0", optional = true, markers = "extra == \"redis\""} +tzdata = ">=2022.7" +vine = ">=5.1.0,<6.0" + +[package.extras] +arangodb = ["pyArango (>=2.0.2)"] +auth = ["cryptography (==41.0.5)"] +azureblockblob = ["azure-storage-blob (>=12.15.0)"] +brotli = ["brotli (>=1.0.0)", "brotlipy (>=0.7.0)"] +cassandra = ["cassandra-driver (>=3.25.0,<4)"] +consul = ["python-consul2 (==0.1.5)"] +cosmosdbsql = ["pydocumentdb (==2.3.5)"] +couchbase = ["couchbase (>=3.0.0)"] +couchdb = ["pycouchdb (==1.14.2)"] +django = ["Django (>=2.2.28)"] +dynamodb = ["boto3 (>=1.26.143)"] +elasticsearch = ["elastic-transport (<=8.10.0)", "elasticsearch (<=8.10.1)"] +eventlet = ["eventlet (>=0.32.0)"] +gevent = ["gevent (>=1.5.0)"] +librabbitmq = ["librabbitmq (>=2.0.0)"] +memcache = ["pylibmc (==1.6.3)"] +mongodb = ["pymongo[srv] (>=4.0.2)"] +msgpack = ["msgpack (==1.0.7)"] +pymemcache = ["python-memcached (==1.59)"] +pyro = ["pyro4 (==4.82)"] +pytest = ["pytest-celery (==0.0.0)"] +redis = ["redis (>=4.5.2,!=4.5.5,<6.0.0)"] +s3 = ["boto3 (>=1.26.143)"] +slmq = ["softlayer-messaging (>=1.0.3)"] +solar = ["ephem (==4.1.5)"] +sqlalchemy = ["sqlalchemy (>=1.4.48,<2.1)"] +sqs = ["boto3 (>=1.26.143)", "kombu[sqs] (>=5.3.0)", "pycurl (>=7.43.0.5)", "urllib3 (>=1.26.16)"] +tblib = ["tblib (>=1.3.0)", "tblib (>=1.5.0)"] +yaml = ["PyYAML (>=3.10)"] +zookeeper = ["kazoo (>=1.3.1)"] +zstd = ["zstandard (==0.22.0)"] + +[[package]] +name = "celery-stubs" +version = "0.1.3" +description = "celery stubs" +optional = false +python-versions = "*" +files = [ + {file = "celery-stubs-0.1.3.tar.gz", hash = "sha256:0fb5345820f8a2bd14e6ffcbef2d10181e12e40f8369f551d7acc99d8d514919"}, + {file = "celery_stubs-0.1.3-py3-none-any.whl", hash = "sha256:dfb9ad27614a8af028b2055bb4a4ae99ca5e9a8d871428a506646d62153218d7"}, +] + +[package.dependencies] +mypy = ">=0.950" +typing-extensions = ">=4.2.0" + [[package]] name = "certifi" version = "2023.7.22" @@ -388,6 +503,55 @@ files = [ [package.dependencies] colorama = {version = "*", markers = "platform_system == \"Windows\""} +[[package]] +name = "click-didyoumean" +version = "0.3.0" +description = "Enables git-like *did-you-mean* feature in click" +optional = false +python-versions = ">=3.6.2,<4.0.0" +files = [ + {file = "click-didyoumean-0.3.0.tar.gz", hash = "sha256:f184f0d851d96b6d29297354ed981b7dd71df7ff500d82fa6d11f0856bee8035"}, + {file = "click_didyoumean-0.3.0-py3-none-any.whl", hash = "sha256:a0713dc7a1de3f06bc0df5a9567ad19ead2d3d5689b434768a6145bff77c0667"}, +] + +[package.dependencies] +click = ">=7" + +[[package]] +name = "click-plugins" +version = "1.1.1" +description = "An extension module for click to enable registering CLI commands via setuptools entry-points." +optional = false +python-versions = "*" +files = [ + {file = "click-plugins-1.1.1.tar.gz", hash = "sha256:46ab999744a9d831159c3411bb0c79346d94a444df9a3a3742e9ed63645f264b"}, + {file = "click_plugins-1.1.1-py2.py3-none-any.whl", hash = "sha256:5d262006d3222f5057fd81e1623d4443e41dcda5dc815c06b442aa3c02889fc8"}, +] + +[package.dependencies] +click = ">=4.0" + +[package.extras] +dev = ["coveralls", "pytest (>=3.6)", "pytest-cov", "wheel"] + +[[package]] +name = "click-repl" +version = "0.3.0" +description = "REPL plugin for Click" +optional = false +python-versions = ">=3.6" +files = [ + {file = "click-repl-0.3.0.tar.gz", hash = "sha256:17849c23dba3d667247dc4defe1757fff98694e90fe37474f3feebb69ced26a9"}, + {file = "click_repl-0.3.0-py3-none-any.whl", hash = "sha256:fb7e06deb8da8de86180a33a9da97ac316751c094c6899382da7feeeeb51b812"}, +] + +[package.dependencies] +click = ">=7.0" +prompt-toolkit = ">=3.0.36" + +[package.extras] +testing = ["pytest (>=7.2.1)", "pytest-cov (>=4.0.0)", "tox (>=4.4.3)"] + [[package]] name = "clickclick" version = "20.10.2" @@ -1094,6 +1258,38 @@ pyrsistent = ">=0.14.0,<0.17.0 || >0.17.0,<0.17.1 || >0.17.1,<0.17.2 || >0.17.2" format = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3987", "uri-template", "webcolors (>=1.11)"] format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "uri-template", "webcolors (>=1.11)"] +[[package]] +name = "kombu" +version = "5.3.4" +description = "Messaging library for Python." +optional = false +python-versions = ">=3.8" +files = [ + {file = "kombu-5.3.4-py3-none-any.whl", hash = "sha256:63bb093fc9bb80cfb3a0972336a5cec1fa7ac5f9ef7e8237c6bf8dda9469313e"}, + {file = "kombu-5.3.4.tar.gz", hash = "sha256:0bb2e278644d11dea6272c17974a3dbb9688a949f3bb60aeb5b791329c44fadc"}, +] + +[package.dependencies] +amqp = ">=5.1.1,<6.0.0" +vine = "*" + +[package.extras] +azureservicebus = ["azure-servicebus (>=7.10.0)"] +azurestoragequeues = ["azure-identity (>=1.12.0)", "azure-storage-queue (>=12.6.0)"] +confluentkafka = ["confluent-kafka (>=2.2.0)"] +consul = ["python-consul2"] +librabbitmq = ["librabbitmq (>=2.0.0)"] +mongodb = ["pymongo (>=4.1.1)"] +msgpack = ["msgpack"] +pyro = ["pyro4"] +qpid = ["qpid-python (>=0.26)", "qpid-tools (>=0.26)"] +redis = ["redis (>=4.5.2,!=4.5.5,<6.0.0)"] +slmq = ["softlayer-messaging (>=1.0.3)"] +sqlalchemy = ["sqlalchemy (>=1.4.48,<2.1)"] +sqs = ["boto3 (>=1.26.143)", "pycurl (>=7.43.0.5)", "urllib3 (>=1.26.16)"] +yaml = ["PyYAML (>=3.10)"] +zookeeper = ["kazoo (>=2.8.0)"] + [[package]] name = "lxml" version = "4.9.2" @@ -1426,18 +1622,15 @@ test = ["blinker", "cryptography", "mock", "nose", "pyjwt (>=1.0.0)", "unittest2 [[package]] name = "packaging" -version = "21.3" +version = "23.2" description = "Core utilities for Python packages" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" files = [ - {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"}, - {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"}, + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, ] -[package.dependencies] -pyparsing = ">=2.0.2,<3.0.5 || >3.0.5" - [[package]] name = "pathspec" version = "0.11.1" @@ -1552,6 +1745,20 @@ files = [ flask = "*" prometheus-client = "*" +[[package]] +name = "prompt-toolkit" +version = "3.0.41" +description = "Library for building powerful interactive command lines in Python" +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "prompt_toolkit-3.0.41-py3-none-any.whl", hash = "sha256:f36fe301fafb7470e86aaf90f036eef600a3210be4decf461a5b1ca8403d3cb2"}, + {file = "prompt_toolkit-3.0.41.tar.gz", hash = "sha256:941367d97fc815548822aa26c2a269fdc4eb21e9ec05fc5d447cf09bad5d75f0"}, +] + +[package.dependencies] +wcwidth = "*" + [[package]] name = "psycopg2" version = "2.9.6" @@ -1616,20 +1823,6 @@ dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.4.0)", "pre-commit", "pyte docs = ["sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] -[[package]] -name = "pyparsing" -version = "3.0.9" -description = "pyparsing module - Classes and methods to define and execute parsing grammars" -optional = false -python-versions = ">=3.6.8" -files = [ - {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, - {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, -] - -[package.extras] -diagrams = ["jinja2", "railroad-diagrams"] - [[package]] name = "pyrsistent" version = "0.19.3" @@ -1867,6 +2060,24 @@ files = [ {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, ] +[[package]] +name = "redis" +version = "5.0.1" +description = "Python client for Redis database and key-value store" +optional = false +python-versions = ">=3.7" +files = [ + {file = "redis-5.0.1-py3-none-any.whl", hash = "sha256:ed4802971884ae19d640775ba3b03aa2e7bd5e8fb8dfaed2decce4d0fc48391f"}, + {file = "redis-5.0.1.tar.gz", hash = "sha256:0dab495cd5753069d3bc650a0dde8a8f9edde16fc5691b689a566eda58100d0f"}, +] + +[package.dependencies] +async-timeout = {version = ">=4.0.2", markers = "python_full_version <= \"3.11.2\""} + +[package.extras] +hiredis = ["hiredis (>=1.0.0)"] +ocsp = ["cryptography (>=36.0.1)", "pyopenssl (==20.0.1)", "requests (>=2.26.0)"] + [[package]] name = "regex" version = "2023.3.23" @@ -2082,25 +2293,28 @@ files = [ [[package]] name = "safety" -version = "2.3.5" +version = "2.4.0b2" description = "Checks installed dependencies for known vulnerabilities and licenses." optional = false python-versions = "*" files = [ - {file = "safety-2.3.5-py3-none-any.whl", hash = "sha256:2227fcac1b22b53c1615af78872b48348661691450aa25d6704a5504dbd1f7e2"}, - {file = "safety-2.3.5.tar.gz", hash = "sha256:a60c11f8952f412cbb165d70cb1f673a3b43a2ba9a93ce11f97e6a4de834aa3a"}, + {file = "safety-2.4.0b2-py3-none-any.whl", hash = "sha256:63773ce92e17f5f80e7dff4c8a25d8abb7d62d375897b5f3bb4afe9313b100ff"}, + {file = "safety-2.4.0b2.tar.gz", hash = "sha256:9907010c6ca7720861ca7fa1496bdb80449b0619ca136eb7ac7e02bd3516cd4f"}, ] [package.dependencies] Click = ">=8.0.2" dparse = ">=0.6.2" -packaging = ">=21.0,<22.0" +jinja2 = {version = ">=3.1.0", markers = "python_version >= \"3.7\""} +marshmallow = {version = ">=3.15.0", markers = "python_version >= \"3.7\""} +packaging = ">=21.0" requests = "*" "ruamel.yaml" = ">=0.17.21" -setuptools = ">=19.3" +setuptools = {version = ">=65.5.1", markers = "python_version >= \"3.7\""} +urllib3 = ">=1.26.5" [package.extras] -github = ["jinja2 (>=3.1.0)", "pygithub (>=1.43.3)"] +github = ["pygithub (>=1.43.3)"] gitlab = ["python-gitlab (>=1.3.0)"] [[package]] @@ -2676,6 +2890,17 @@ secure = ["certifi", "cryptography (>=1.9)", "idna (>=2.0.0)", "pyopenssl (>=17. socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] zstd = ["zstandard (>=0.18.0)"] +[[package]] +name = "vine" +version = "5.1.0" +description = "Python promises." +optional = false +python-versions = ">=3.6" +files = [ + {file = "vine-5.1.0-py3-none-any.whl", hash = "sha256:40fdf3c48b2cfe1c38a49e9ae2da6fda88e4794c810050a728bd7413811fb1dc"}, + {file = "vine-5.1.0.tar.gz", hash = "sha256:8b62e981d35c41049211cf62a0a1242d8c1ee9bd15bb196ce38aefd6799e61e0"}, +] + [[package]] name = "virtualenv" version = "20.21.0" @@ -2696,6 +2921,17 @@ platformdirs = ">=2.4,<4" docs = ["furo (>=2022.12.7)", "proselint (>=0.13)", "sphinx (>=6.1.3)", "sphinx-argparse (>=0.4)", "sphinxcontrib-towncrier (>=0.2.1a0)", "towncrier (>=22.12)"] test = ["covdefaults (>=2.2.2)", "coverage (>=7.1)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=23)", "pytest (>=7.2.1)", "pytest-env (>=0.8.1)", "pytest-freezegun (>=0.4.2)", "pytest-mock (>=3.10)", "pytest-randomly (>=3.12)", "pytest-timeout (>=2.1)"] +[[package]] +name = "wcwidth" +version = "0.2.10" +description = "Measures the displayed width of unicode strings in a terminal" +optional = false +python-versions = "*" +files = [ + {file = "wcwidth-0.2.10-py2.py3-none-any.whl", hash = "sha256:aec5179002dd0f0d40c456026e74a729661c9d468e1ed64405e3a6c2176ca36f"}, + {file = "wcwidth-0.2.10.tar.gz", hash = "sha256:390c7454101092a6a5e43baad8f83de615463af459201709556b6e4b1c861f97"}, +] + [[package]] name = "werkzeug" version = "2.3.8" @@ -2732,34 +2968,32 @@ email = ["email-validator"] [[package]] name = "xdoctest" -version = "1.1.1" +version = "1.1.2" description = "A rewrite of the builtin doctest module" optional = false python-versions = ">=3.6" files = [ - {file = "xdoctest-1.1.1-py3-none-any.whl", hash = "sha256:d59d4ed91cb92e4430ef0ad1b134a2bef02adff7d2fb9c9f057547bee44081a2"}, - {file = "xdoctest-1.1.1.tar.gz", hash = "sha256:2eac8131bdcdf2781b4e5a62d6de87f044b730cc8db8af142a51bb29c245e779"}, + {file = "xdoctest-1.1.2-py3-none-any.whl", hash = "sha256:ebe133222534f09597cbe461f97cc5f95ad7b36e5d31f3437caffb9baaddbddb"}, + {file = "xdoctest-1.1.2.tar.gz", hash = "sha256:267d3d4e362547fa917d3deabaf6888232bbf43c8d30298faeb957dbfa7e0ba3"}, ] [package.dependencies] colorama = {version = "*", optional = true, markers = "platform_system == \"Windows\" and extra == \"colors\""} Pygments = {version = "*", optional = true, markers = "python_version >= \"3.5.0\" and extra == \"colors\""} -six = "*" [package.extras] -all = ["IPython", "IPython", "Pygments", "Pygments", "attrs", "codecov", "colorama", "debugpy", "debugpy", "debugpy", "debugpy", "debugpy", "ipykernel", "ipykernel", "ipython-genutils", "jedi", "jinja2", "jupyter-client", "jupyter-client", "jupyter-core", "nbconvert", "pyflakes", "pytest", "pytest", "pytest", "pytest-cov", "six", "tomli", "typing"] -all-strict = ["IPython (==7.10.0)", "IPython (==7.23.1)", "Pygments (==2.0.0)", "Pygments (==2.4.1)", "attrs (==19.2.0)", "codecov (==2.0.15)", "colorama (==0.4.1)", "debugpy (==1.0.0)", "debugpy (==1.0.0)", "debugpy (==1.0.0)", "debugpy (==1.3.0)", "debugpy (==1.6.0)", "ipykernel (==5.2.0)", "ipykernel (==6.0.0)", "ipython-genutils (==0.2.0)", "jedi (==0.16)", "jinja2 (==3.0.0)", "jupyter-client (==6.1.5)", "jupyter-client (==7.0.0)", "jupyter-core (==4.7.0)", "nbconvert (==6.0.0)", "pyflakes (==2.2.0)", "pytest (==4.6.0)", "pytest (==4.6.0)", "pytest (==6.2.5)", "pytest-cov (==3.0.0)", "six (==1.11.0)", "tomli (==0.2.0)", "typing (==3.7.4)"] +all = ["IPython (>=7.10.0)", "IPython (>=7.23.1)", "Pygments (>=2.0.0)", "Pygments (>=2.4.1)", "attrs (>=19.2.0)", "colorama (>=0.4.1)", "debugpy (>=1.0.0)", "debugpy (>=1.0.0)", "debugpy (>=1.0.0)", "debugpy (>=1.3.0)", "debugpy (>=1.6.0)", "ipykernel (>=5.2.0)", "ipykernel (>=6.0.0)", "ipykernel (>=6.11.0)", "ipython-genutils (>=0.2.0)", "jedi (>=0.16)", "jinja2 (>=3.0.0)", "jupyter-client (>=6.1.5)", "jupyter-client (>=7.0.0)", "jupyter-core (>=4.7.0)", "nbconvert (>=6.0.0)", "nbconvert (>=6.1.0)", "pyflakes (>=2.2.0)", "pytest (>=4.6.0)", "pytest (>=4.6.0)", "pytest (>=6.2.5)", "pytest-cov (>=3.0.0)", "tomli (>=0.2.0)", "typing (>=3.7.4)"] +all-strict = ["IPython (==7.10.0)", "IPython (==7.23.1)", "Pygments (==2.0.0)", "Pygments (==2.4.1)", "attrs (==19.2.0)", "colorama (==0.4.1)", "debugpy (==1.0.0)", "debugpy (==1.0.0)", "debugpy (==1.0.0)", "debugpy (==1.3.0)", "debugpy (==1.6.0)", "ipykernel (==5.2.0)", "ipykernel (==6.0.0)", "ipykernel (==6.11.0)", "ipython-genutils (==0.2.0)", "jedi (==0.16)", "jinja2 (==3.0.0)", "jupyter-client (==6.1.5)", "jupyter-client (==7.0.0)", "jupyter-core (==4.7.0)", "nbconvert (==6.0.0)", "nbconvert (==6.1.0)", "pyflakes (==2.2.0)", "pytest (==4.6.0)", "pytest (==4.6.0)", "pytest (==6.2.5)", "pytest-cov (==3.0.0)", "tomli (==0.2.0)", "typing (==3.7.4)"] colors = ["Pygments", "Pygments", "colorama"] -jupyter = ["IPython", "IPython", "attrs", "debugpy", "debugpy", "debugpy", "debugpy", "debugpy", "ipykernel", "ipykernel", "ipython-genutils", "jedi", "jinja2", "jupyter-client", "jupyter-client", "jupyter-core", "nbconvert"] -optional = ["IPython", "IPython", "Pygments", "Pygments", "attrs", "colorama", "debugpy", "debugpy", "debugpy", "debugpy", "debugpy", "ipykernel", "ipykernel", "ipython-genutils", "jedi", "jinja2", "jupyter-client", "jupyter-client", "jupyter-core", "nbconvert", "pyflakes", "tomli"] -optional-strict = ["IPython (==7.10.0)", "IPython (==7.23.1)", "Pygments (==2.0.0)", "Pygments (==2.4.1)", "attrs (==19.2.0)", "colorama (==0.4.1)", "debugpy (==1.0.0)", "debugpy (==1.0.0)", "debugpy (==1.0.0)", "debugpy (==1.3.0)", "debugpy (==1.6.0)", "ipykernel (==5.2.0)", "ipykernel (==6.0.0)", "ipython-genutils (==0.2.0)", "jedi (==0.16)", "jinja2 (==3.0.0)", "jupyter-client (==6.1.5)", "jupyter-client (==7.0.0)", "jupyter-core (==4.7.0)", "nbconvert (==6.0.0)", "pyflakes (==2.2.0)", "tomli (==0.2.0)"] -runtime-strict = ["six (==1.11.0)"] -tests = ["codecov", "pytest", "pytest", "pytest", "pytest-cov", "typing"] +jupyter = ["IPython", "IPython", "attrs", "debugpy", "debugpy", "debugpy", "debugpy", "debugpy", "ipykernel", "ipykernel", "ipykernel", "ipython-genutils", "jedi", "jinja2", "jupyter-client", "jupyter-client", "jupyter-core", "nbconvert", "nbconvert"] +optional = ["IPython (>=7.10.0)", "IPython (>=7.23.1)", "Pygments (>=2.0.0)", "Pygments (>=2.4.1)", "attrs (>=19.2.0)", "colorama (>=0.4.1)", "debugpy (>=1.0.0)", "debugpy (>=1.0.0)", "debugpy (>=1.0.0)", "debugpy (>=1.3.0)", "debugpy (>=1.6.0)", "ipykernel (>=5.2.0)", "ipykernel (>=6.0.0)", "ipykernel (>=6.11.0)", "ipython-genutils (>=0.2.0)", "jedi (>=0.16)", "jinja2 (>=3.0.0)", "jupyter-client (>=6.1.5)", "jupyter-client (>=7.0.0)", "jupyter-core (>=4.7.0)", "nbconvert (>=6.0.0)", "nbconvert (>=6.1.0)", "pyflakes (>=2.2.0)", "tomli (>=0.2.0)"] +optional-strict = ["IPython (==7.10.0)", "IPython (==7.23.1)", "Pygments (==2.0.0)", "Pygments (==2.4.1)", "attrs (==19.2.0)", "colorama (==0.4.1)", "debugpy (==1.0.0)", "debugpy (==1.0.0)", "debugpy (==1.0.0)", "debugpy (==1.3.0)", "debugpy (==1.6.0)", "ipykernel (==5.2.0)", "ipykernel (==6.0.0)", "ipykernel (==6.11.0)", "ipython-genutils (==0.2.0)", "jedi (==0.16)", "jinja2 (==3.0.0)", "jupyter-client (==6.1.5)", "jupyter-client (==7.0.0)", "jupyter-core (==4.7.0)", "nbconvert (==6.0.0)", "nbconvert (==6.1.0)", "pyflakes (==2.2.0)", "tomli (==0.2.0)"] +tests = ["pytest (>=4.6.0)", "pytest (>=4.6.0)", "pytest (>=6.2.5)", "pytest-cov (>=3.0.0)", "typing (>=3.7.4)"] tests-binary = ["cmake", "cmake", "ninja", "ninja", "pybind11", "pybind11", "scikit-build", "scikit-build"] tests-binary-strict = ["cmake (==3.21.2)", "cmake (==3.25.0)", "ninja (==1.10.2)", "ninja (==1.11.1)", "pybind11 (==2.10.3)", "pybind11 (==2.7.1)", "scikit-build (==0.11.1)", "scikit-build (==0.16.1)"] -tests-strict = ["codecov (==2.0.15)", "pytest (==4.6.0)", "pytest (==4.6.0)", "pytest (==6.2.5)", "pytest-cov (==3.0.0)", "typing (==3.7.4)"] +tests-strict = ["pytest (==4.6.0)", "pytest (==4.6.0)", "pytest (==6.2.5)", "pytest-cov (==3.0.0)", "typing (==3.7.4)"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.12" -content-hash = "c30e0e07342a1e7b34bed4ae9722c0604f561a83e220b1b423049dde6c61c122" +content-hash = "470406c5ff0f63983a4fffb90c0a9101c1abcf07fb4ea2a9414b8cdd16aa2f60" diff --git a/spiffworkflow-backend/pyproject.toml b/spiffworkflow-backend/pyproject.toml index c9f5c474..a6c277db 100644 --- a/spiffworkflow-backend/pyproject.toml +++ b/spiffworkflow-backend/pyproject.toml @@ -77,6 +77,8 @@ spiff-element-units = "^0.3.1" mysqlclient = "^2.2.0" flask-session = "^0.5.0" flask-oauthlib = "^0.9.6" +celery = {extras = ["redis"], version = "^5.3.5"} +celery-stubs = "^0.1.3" [tool.poetry.dev-dependencies] pytest = "^7.1.2" diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/__init__.py b/spiffworkflow-backend/src/spiffworkflow_backend/__init__.py index bed4cdbe..32ea504b 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/__init__.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/__init__.py @@ -1,22 +1,18 @@ import faulthandler -import json import os -import sys from typing import Any import connexion # type: ignore import flask.app import flask.json import sqlalchemy -from apscheduler.schedulers.background import BackgroundScheduler # type: ignore -from apscheduler.schedulers.base import BaseScheduler # type: ignore from flask.json.provider import DefaultJSONProvider from flask_cors import CORS # type: ignore from flask_mail import Mail # type: ignore -from prometheus_flask_exporter import ConnexionPrometheusMetrics # type: ignore -from werkzeug.exceptions import NotFound import spiffworkflow_backend.load_database_models # noqa: F401 +from spiffworkflow_backend.background_processing.apscheduler import start_apscheduler_if_appropriate +from spiffworkflow_backend.background_processing.celery import init_celery_if_appropriate from spiffworkflow_backend.config import setup_config from spiffworkflow_backend.exceptions.api_error import api_error_blueprint from spiffworkflow_backend.helpers.api_version import V1_API_PATH_PREFIX @@ -27,7 +23,8 @@ from spiffworkflow_backend.routes.authentication_controller import verify_token from spiffworkflow_backend.routes.openid_blueprint.openid_blueprint import openid_blueprint from spiffworkflow_backend.routes.user_blueprint import user_blueprint from spiffworkflow_backend.services.authorization_service import AuthorizationService -from spiffworkflow_backend.services.background_processing_service import BackgroundProcessingService +from spiffworkflow_backend.services.monitoring_service import configure_sentry +from spiffworkflow_backend.services.monitoring_service import setup_prometheus_metrics class MyJSONEncoder(DefaultJSONProvider): @@ -55,60 +52,6 @@ class MyJSONEncoder(DefaultJSONProvider): return super().dumps(obj, **kwargs) -def start_scheduler(app: flask.app.Flask, scheduler_class: BaseScheduler = BackgroundScheduler) -> None: - scheduler = scheduler_class() - - # TODO: polling intervals for messages job - polling_interval_in_seconds = app.config["SPIFFWORKFLOW_BACKEND_BACKGROUND_SCHEDULER_POLLING_INTERVAL_IN_SECONDS"] - not_started_polling_interval_in_seconds = app.config[ - "SPIFFWORKFLOW_BACKEND_BACKGROUND_SCHEDULER_NOT_STARTED_POLLING_INTERVAL_IN_SECONDS" - ] - user_input_required_polling_interval_in_seconds = app.config[ - "SPIFFWORKFLOW_BACKEND_BACKGROUND_SCHEDULER_USER_INPUT_REQUIRED_POLLING_INTERVAL_IN_SECONDS" - ] - # TODO: add job to release locks to simplify other queries - # TODO: add job to delete completed entires - # TODO: add job to run old/low priority instances so they do not get drowned out - - scheduler.add_job( - BackgroundProcessingService(app).process_message_instances_with_app_context, - "interval", - seconds=10, - ) - scheduler.add_job( - BackgroundProcessingService(app).process_not_started_process_instances, - "interval", - seconds=not_started_polling_interval_in_seconds, - ) - scheduler.add_job( - BackgroundProcessingService(app).process_waiting_process_instances, - "interval", - seconds=polling_interval_in_seconds, - ) - scheduler.add_job( - BackgroundProcessingService(app).process_user_input_required_process_instances, - "interval", - seconds=user_input_required_polling_interval_in_seconds, - ) - scheduler.add_job( - BackgroundProcessingService(app).remove_stale_locks, - "interval", - seconds=app.config["MAX_INSTANCE_LOCK_DURATION_IN_SECONDS"], - ) - scheduler.start() - - -def should_start_scheduler(app: flask.app.Flask) -> bool: - if not app.config["SPIFFWORKFLOW_BACKEND_RUN_BACKGROUND_SCHEDULER_IN_CREATE_APP"]: - return False - - # do not start the scheduler twice in flask debug mode but support code reloading - if app.config["ENV_IDENTIFIER"] == "local_development" and os.environ.get("WERKZEUG_RUN_MAIN") == "true": - return False - - return True - - def create_app() -> flask.app.Flask: faulthandler.enable() @@ -121,7 +64,7 @@ def create_app() -> flask.app.Flask: app = connexion_app.app app.config["CONNEXION_APP"] = connexion_app app.config["SESSION_TYPE"] = "filesystem" - _setup_prometheus_metrics(app, connexion_app) + setup_prometheus_metrics(app, connexion_app) setup_config(app) db.init_app(app) @@ -134,9 +77,7 @@ def create_app() -> flask.app.Flask: # preflight options requests will be allowed if they meet the requirements of the url regex. # we will add an Access-Control-Max-Age header to the response to tell the browser it doesn't # need to continually keep asking for the same path. - origins_re = [ - r"^https?:\/\/%s(.*)" % o.replace(".", r"\.") for o in app.config["SPIFFWORKFLOW_BACKEND_CORS_ALLOW_ORIGINS"] - ] + origins_re = [r"^https?:\/\/%s(.*)" % o.replace(".", r"\.") for o in app.config["SPIFFWORKFLOW_BACKEND_CORS_ALLOW_ORIGINS"]] CORS(app, origins=origins_re, max_age=3600, supports_credentials=True) connexion_app.add_api("api.yml", base_path=V1_API_PATH_PREFIX) @@ -146,9 +87,6 @@ def create_app() -> flask.app.Flask: app.json = MyJSONEncoder(app) - if should_start_scheduler(app): - start_scheduler(app) - configure_sentry(app) app.before_request(verify_token) @@ -159,104 +97,7 @@ def create_app() -> flask.app.Flask: # This is particularly helpful for forms that are generated from json schemas. app.json.sort_keys = False + start_apscheduler_if_appropriate(app) + init_celery_if_appropriate(app) + return app # type: ignore - - -def get_version_info_data() -> dict[str, Any]: - version_info_data_dict = {} - if os.path.isfile("version_info.json"): - with open("version_info.json") as f: - version_info_data_dict = json.load(f) - return version_info_data_dict - - -def _setup_prometheus_metrics(app: flask.app.Flask, connexion_app: connexion.apps.flask_app.FlaskApp) -> None: - metrics = ConnexionPrometheusMetrics(connexion_app) - app.config["PROMETHEUS_METRICS"] = metrics - version_info_data = get_version_info_data() - if len(version_info_data) > 0: - # prometheus does not allow periods in key names - version_info_data_normalized = {k.replace(".", "_"): v for k, v in version_info_data.items()} - metrics.info("version_info", "Application Version Info", **version_info_data_normalized) - - -def traces_sampler(sampling_context: Any) -> Any: - # always inherit - if sampling_context["parent_sampled"] is not None: - return sampling_context["parent_sampled"] - - if "wsgi_environ" in sampling_context: - wsgi_environ = sampling_context["wsgi_environ"] - path_info = wsgi_environ.get("PATH_INFO") - request_method = wsgi_environ.get("REQUEST_METHOD") - - # tasks_controller.task_submit - # this is the current pain point as of 31 jan 2023. - if path_info and ( - (path_info.startswith("/v1.0/tasks/") and request_method == "PUT") - or (path_info.startswith("/v1.0/task-data/") and request_method == "GET") - ): - return 1 - - # Default sample rate for all others (replaces traces_sample_rate) - return 0.01 - - -def configure_sentry(app: flask.app.Flask) -> None: - import sentry_sdk - from sentry_sdk.integrations.flask import FlaskIntegration - - # get rid of NotFound errors - def before_send(event: Any, hint: Any) -> Any: - if "exc_info" in hint: - _exc_type, exc_value, _tb = hint["exc_info"] - # NotFound is mostly from web crawlers - if isinstance(exc_value, NotFound): - return None - return event - - sentry_errors_sample_rate = app.config.get("SPIFFWORKFLOW_BACKEND_SENTRY_ERRORS_SAMPLE_RATE") - if sentry_errors_sample_rate is None: - raise Exception("SPIFFWORKFLOW_BACKEND_SENTRY_ERRORS_SAMPLE_RATE is not set somehow") - - sentry_traces_sample_rate = app.config.get("SPIFFWORKFLOW_BACKEND_SENTRY_TRACES_SAMPLE_RATE") - if sentry_traces_sample_rate is None: - raise Exception("SPIFFWORKFLOW_BACKEND_SENTRY_TRACES_SAMPLE_RATE is not set somehow") - - sentry_env_identifier = app.config["ENV_IDENTIFIER"] - if app.config.get("SPIFFWORKFLOW_BACKEND_SENTRY_ENV_IDENTIFIER"): - sentry_env_identifier = app.config.get("SPIFFWORKFLOW_BACKEND_SENTRY_ENV_IDENTIFIER") - - sentry_configs = { - "dsn": app.config.get("SPIFFWORKFLOW_BACKEND_SENTRY_DSN"), - "integrations": [ - FlaskIntegration(), - ], - "environment": sentry_env_identifier, - # sample_rate is the errors sample rate. we usually set it to 1 (100%) - # so we get all errors in sentry. - "sample_rate": float(sentry_errors_sample_rate), - # Set traces_sample_rate to capture a certain percentage - # of transactions for performance monitoring. - # We recommend adjusting this value to less than 1(00%) in production. - "traces_sample_rate": float(sentry_traces_sample_rate), - "traces_sampler": traces_sampler, - # The profiles_sample_rate setting is relative to the traces_sample_rate setting. - "before_send": before_send, - } - - # https://docs.sentry.io/platforms/python/configuration/releases - version_info_data = get_version_info_data() - if len(version_info_data) > 0: - git_commit = version_info_data.get("org.opencontainers.image.revision") or version_info_data.get("git_commit") - if git_commit is not None: - sentry_configs["release"] = git_commit - - if app.config.get("SPIFFWORKFLOW_BACKEND_SENTRY_PROFILING_ENABLED"): - # profiling doesn't work on windows, because of an issue like https://github.com/nvdv/vprof/issues/62 - # but also we commented out profiling because it was causing segfaults (i guess it is marked experimental) - profiles_sample_rate = 0 if sys.platform.startswith("win") else 1 - if profiles_sample_rate > 0: - sentry_configs["_experiments"] = {"profiles_sample_rate": profiles_sample_rate} - - sentry_sdk.init(**sentry_configs) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/api.yml b/spiffworkflow-backend/src/spiffworkflow_backend/api.yml index c6407200..7f45b23a 100755 --- a/spiffworkflow-backend/src/spiffworkflow_backend/api.yml +++ b/spiffworkflow-backend/src/spiffworkflow_backend/api.yml @@ -1200,10 +1200,10 @@ paths: description: The unique id of an existing process instance. schema: type: integer - - name: do_engine_steps + - name: force_run in: query required: false - description: Defaults to true, can be set to false if you are just looking at the workflow not completeing it. + description: Force the process instance to run even if it has already been started. schema: type: boolean post: @@ -1728,6 +1728,29 @@ paths: items: $ref: "#/components/schemas/Task" + /tasks/progress/{process_instance_id}: + parameters: + - name: process_instance_id + in: path + required: true + description: The unique id of an existing process instance. + schema: + type: integer + get: + tags: + - Process Instances + operationId: spiffworkflow_backend.routes.tasks_controller.process_instance_progress + summary: returns the list of instructions that have been queued for a process instance. + responses: + "200": + description: list of task instructions + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/TaskInstructionsForEndUser" + /users/search: parameters: - name: username_prefix @@ -2191,6 +2214,27 @@ paths: $ref: "#/components/schemas/Task" + /tasks/{process_instance_id}/instruction: + parameters: + - name: process_instance_id + in: path + required: true + description: The unique id of an existing process instance. + schema: + type: integer + get: + tags: + - Tasks + operationId: spiffworkflow_backend.routes.tasks_controller.task_with_instruction + summary: Gets the next task and its instructions + responses: + "200": + description: One task + content: + application/json: + schema: + $ref: "#/components/schemas/Task" + /tasks/{process_instance_id}/{task_guid}: parameters: - name: task_guid @@ -3040,6 +3084,16 @@ components: documentation: "# Heading 1\n\nMarkdown documentation text goes here" type: form state: ready + TaskInstructionsForEndUser: + properties: + task_guid: + type: string + process_instance_id: + type: integer + instruction: + type: string + timestamp: + type: number TaskAllowsGuest: properties: allows_guest: diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/background_processing/__init__.py b/spiffworkflow-backend/src/spiffworkflow_backend/background_processing/__init__.py new file mode 100644 index 00000000..ac1b1b9c --- /dev/null +++ b/spiffworkflow-backend/src/spiffworkflow_backend/background_processing/__init__.py @@ -0,0 +1,3 @@ +CELERY_TASK_PROCESS_INSTANCE_RUN = ( + "spiffworkflow_backend.background_processing.celery_tasks.process_instance_task.celery_task_process_instance_run" +) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/background_processing/apscheduler.py b/spiffworkflow-backend/src/spiffworkflow_backend/background_processing/apscheduler.py new file mode 100644 index 00000000..b15ee7f5 --- /dev/null +++ b/spiffworkflow-backend/src/spiffworkflow_backend/background_processing/apscheduler.py @@ -0,0 +1,103 @@ +import os + +import flask.wrappers +from apscheduler.schedulers.background import BackgroundScheduler # type: ignore +from apscheduler.schedulers.base import BaseScheduler # type: ignore + +from spiffworkflow_backend.background_processing.background_processing_service import BackgroundProcessingService + + +def should_start_apscheduler(app: flask.app.Flask) -> bool: + if not app.config["SPIFFWORKFLOW_BACKEND_RUN_BACKGROUND_SCHEDULER_IN_CREATE_APP"]: + return False + + # do not start the scheduler twice in flask debug mode but support code reloading + if app.config["ENV_IDENTIFIER"] == "local_development" and os.environ.get("WERKZEUG_RUN_MAIN") == "true": + return False + + return True + + +def start_apscheduler_if_appropriate(app: flask.app.Flask, scheduler_class: BaseScheduler = BackgroundScheduler) -> None: + if not should_start_apscheduler(app): + return None + + start_apscheduler(app, scheduler_class) + + +def start_apscheduler(app: flask.app.Flask, scheduler_class: BaseScheduler = BackgroundScheduler) -> None: + scheduler = scheduler_class() + + if app.config["SPIFFWORKFLOW_BACKEND_CELERY_ENABLED"]: + _add_jobs_for_celery_based_configuration(app, scheduler) + else: + _add_jobs_for_non_celery_based_configuration(app, scheduler) + + _add_jobs_relevant_for_all_celery_configurations(app, scheduler) + + scheduler.start() + + +def _add_jobs_for_celery_based_configuration(app: flask.app.Flask, scheduler: BaseScheduler) -> None: + future_task_execution_interval_in_seconds = app.config[ + "SPIFFWORKFLOW_BACKEND_BACKGROUND_SCHEDULER_FUTURE_TASK_EXECUTION_INTERVAL_IN_SECONDS" + ] + + scheduler.add_job( + BackgroundProcessingService(app).process_future_tasks, + "interval", + seconds=future_task_execution_interval_in_seconds, + ) + + +def _add_jobs_for_non_celery_based_configuration(app: flask.app.Flask, scheduler: BaseScheduler) -> None: + # TODO: polling intervals for messages job + polling_interval_in_seconds = app.config["SPIFFWORKFLOW_BACKEND_BACKGROUND_SCHEDULER_POLLING_INTERVAL_IN_SECONDS"] + user_input_required_polling_interval_in_seconds = app.config[ + "SPIFFWORKFLOW_BACKEND_BACKGROUND_SCHEDULER_USER_INPUT_REQUIRED_POLLING_INTERVAL_IN_SECONDS" + ] + # TODO: add job to release locks to simplify other queries + # TODO: add job to delete completed entires + # TODO: add job to run old/low priority instances so they do not get drowned out + + # we should be able to remove these once we switch over to future tasks for non-celery configuration + scheduler.add_job( + BackgroundProcessingService(app).process_waiting_process_instances, + "interval", + seconds=polling_interval_in_seconds, + ) + scheduler.add_job( + BackgroundProcessingService(app).process_running_process_instances, + "interval", + seconds=polling_interval_in_seconds, + ) + scheduler.add_job( + BackgroundProcessingService(app).process_user_input_required_process_instances, + "interval", + seconds=user_input_required_polling_interval_in_seconds, + ) + + +def _add_jobs_relevant_for_all_celery_configurations(app: flask.app.Flask, scheduler: BaseScheduler) -> None: + not_started_polling_interval_in_seconds = app.config[ + "SPIFFWORKFLOW_BACKEND_BACKGROUND_SCHEDULER_NOT_STARTED_POLLING_INTERVAL_IN_SECONDS" + ] + + # TODO: see if we can queue with celery instead on celery based configuration + scheduler.add_job( + BackgroundProcessingService(app).process_message_instances_with_app_context, + "interval", + seconds=10, + ) + + # when you create a process instance via the API and do not use the run API method, this would pick up the instance. + scheduler.add_job( + BackgroundProcessingService(app).process_not_started_process_instances, + "interval", + seconds=not_started_polling_interval_in_seconds, + ) + scheduler.add_job( + BackgroundProcessingService(app).remove_stale_locks, + "interval", + seconds=app.config["MAX_INSTANCE_LOCK_DURATION_IN_SECONDS"], + ) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/services/background_processing_service.py b/spiffworkflow-backend/src/spiffworkflow_backend/background_processing/background_processing_service.py similarity index 53% rename from spiffworkflow-backend/src/spiffworkflow_backend/services/background_processing_service.py rename to spiffworkflow-backend/src/spiffworkflow_backend/background_processing/background_processing_service.py index f8676fda..a444c219 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/services/background_processing_service.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/background_processing/background_processing_service.py @@ -1,6 +1,15 @@ -import flask +import time +import flask +from sqlalchemy import and_ + +from spiffworkflow_backend.background_processing.celery_tasks.process_instance_task_producer import ( + queue_future_task_if_appropriate, +) +from spiffworkflow_backend.models.future_task import FutureTaskModel +from spiffworkflow_backend.models.process_instance import ProcessInstanceModel from spiffworkflow_backend.models.process_instance import ProcessInstanceStatus +from spiffworkflow_backend.models.task import TaskModel # noqa: F401 from spiffworkflow_backend.services.message_service import MessageService from spiffworkflow_backend.services.process_instance_lock_service import ProcessInstanceLockService from spiffworkflow_backend.services.process_instance_service import ProcessInstanceService @@ -24,6 +33,12 @@ class BackgroundProcessingService: ProcessInstanceLockService.set_thread_local_locking_context("bg:waiting") ProcessInstanceService.do_waiting(ProcessInstanceStatus.waiting.value) + def process_running_process_instances(self) -> None: + """Since this runs in a scheduler, we need to specify the app context as well.""" + with self.app.app_context(): + ProcessInstanceLockService.set_thread_local_locking_context("bg:running") + ProcessInstanceService.do_waiting(ProcessInstanceStatus.running.value) + def process_user_input_required_process_instances(self) -> None: """Since this runs in a scheduler, we need to specify the app context as well.""" with self.app.app_context(): @@ -40,3 +55,26 @@ class BackgroundProcessingService: """If something has been locked for a certain amount of time it is probably stale so unlock it.""" with self.app.app_context(): ProcessInstanceLockService.remove_stale_locks() + + def process_future_tasks(self) -> None: + """If something has been locked for a certain amount of time it is probably stale so unlock it.""" + with self.app.app_context(): + future_task_lookahead_in_seconds = self.app.config[ + "SPIFFWORKFLOW_BACKEND_BACKGROUND_SCHEDULER_FUTURE_TASK_LOOKAHEAD_IN_SECONDS" + ] + lookahead = time.time() + future_task_lookahead_in_seconds + future_tasks = FutureTaskModel.query.filter( + and_( + FutureTaskModel.completed == False, # noqa: E712 + FutureTaskModel.run_at_in_seconds < lookahead, + ) + ).all() + for future_task in future_tasks: + process_instance = ( + ProcessInstanceModel.query.join(TaskModel, TaskModel.process_instance_id == ProcessInstanceModel.id) + .filter(TaskModel.guid == future_task.guid) + .first() + ) + queue_future_task_if_appropriate( + process_instance, eta_in_seconds=future_task.run_at_in_seconds, task_guid=future_task.guid + ) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/background_processing/celery.py b/spiffworkflow-backend/src/spiffworkflow_backend/background_processing/celery.py new file mode 100644 index 00000000..56c7c830 --- /dev/null +++ b/spiffworkflow-backend/src/spiffworkflow_backend/background_processing/celery.py @@ -0,0 +1,34 @@ +import flask.wrappers +from celery import Celery +from celery import Task + + +def init_celery_if_appropriate(app: flask.app.Flask) -> None: + if app.config["SPIFFWORKFLOW_BACKEND_CELERY_ENABLED"]: + celery_app = celery_init_app(app) + app.celery_app = celery_app + + +def celery_init_app(app: flask.app.Flask) -> Celery: + class FlaskTask(Task): + def __call__(self, *args: object, **kwargs: object) -> object: + with app.app_context(): + return self.run(*args, **kwargs) # type: ignore + + celery_configs = { + "broker_url": app.config["SPIFFWORKFLOW_BACKEND_CELERY_BROKER_URL"], + "result_backend": app.config["SPIFFWORKFLOW_BACKEND_CELERY_RESULT_BACKEND"], + "task_ignore_result": True, + "task_serializer": "json", + "result_serializer": "json", + "accept_content": ["json"], + "enable_utc": True, + } + + celery_app = Celery(app.name) + celery_app.Task = FlaskTask # type: ignore + celery_app.config_from_object(celery_configs) + celery_app.conf.update(app.config) + celery_app.set_default() + app.celery_app = celery_app + return celery_app diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/background_processing/celery_tasks/__init__.py b/spiffworkflow-backend/src/spiffworkflow_backend/background_processing/celery_tasks/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/background_processing/celery_tasks/process_instance_task.py b/spiffworkflow-backend/src/spiffworkflow_backend/background_processing/celery_tasks/process_instance_task.py new file mode 100644 index 00000000..f10b4ff5 --- /dev/null +++ b/spiffworkflow-backend/src/spiffworkflow_backend/background_processing/celery_tasks/process_instance_task.py @@ -0,0 +1,53 @@ +from billiard import current_process # type: ignore +from celery import shared_task +from flask import current_app + +from spiffworkflow_backend.background_processing.celery_tasks.process_instance_task_producer import ( + queue_process_instance_if_appropriate, +) +from spiffworkflow_backend.models.db import db +from spiffworkflow_backend.models.future_task import FutureTaskModel +from spiffworkflow_backend.models.process_instance import ProcessInstanceModel +from spiffworkflow_backend.services.process_instance_lock_service import ProcessInstanceLockService +from spiffworkflow_backend.services.process_instance_queue_service import ProcessInstanceIsAlreadyLockedError +from spiffworkflow_backend.services.process_instance_queue_service import ProcessInstanceQueueService +from spiffworkflow_backend.services.process_instance_service import ProcessInstanceService +from spiffworkflow_backend.services.workflow_execution_service import TaskRunnability + +ten_minutes = 60 * 10 + + +@shared_task(ignore_result=False, time_limit=ten_minutes) +def celery_task_process_instance_run(process_instance_id: int, task_guid: str | None = None) -> None: + proc_index = current_process().index + ProcessInstanceLockService.set_thread_local_locking_context("celery:worker", additional_processing_identifier=proc_index) + process_instance = ProcessInstanceModel.query.filter_by(id=process_instance_id).first() + try: + with ProcessInstanceQueueService.dequeued(process_instance, additional_processing_identifier=proc_index): + ProcessInstanceService.run_process_instance_with_processor( + process_instance, execution_strategy_name="run_current_ready_tasks", additional_processing_identifier=proc_index + ) + processor, task_runnability = ProcessInstanceService.run_process_instance_with_processor( + process_instance, + execution_strategy_name="queue_instructions_for_end_user", + additional_processing_identifier=proc_index, + ) + if task_guid is not None: + future_task = FutureTaskModel.query.filter_by(completed=False, guid=task_guid).first() + if future_task is not None: + future_task.completed = True + db.session.add(future_task) + db.session.commit() + if task_runnability == TaskRunnability.has_ready_tasks: + queue_process_instance_if_appropriate(process_instance) + except ProcessInstanceIsAlreadyLockedError: + pass + except Exception as e: + db.session.rollback() # in case the above left the database with a bad transaction + error_message = ( + f"Error running process_instance {process_instance.id}" + f"({process_instance.process_model_identifier}). {str(e)}" + ) + current_app.logger.error(error_message) + db.session.add(process_instance) + db.session.commit() + raise e diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/background_processing/celery_tasks/process_instance_task_producer.py b/spiffworkflow-backend/src/spiffworkflow_backend/background_processing/celery_tasks/process_instance_task_producer.py new file mode 100644 index 00000000..05c73d1b --- /dev/null +++ b/spiffworkflow-backend/src/spiffworkflow_backend/background_processing/celery_tasks/process_instance_task_producer.py @@ -0,0 +1,38 @@ +import time + +import celery +from flask import current_app + +from spiffworkflow_backend.background_processing import CELERY_TASK_PROCESS_INSTANCE_RUN +from spiffworkflow_backend.models.process_instance import ProcessInstanceModel + + +def queue_enabled_for_process_model(process_instance: ProcessInstanceModel) -> bool: + # TODO: check based on the process model itself as well + return current_app.config["SPIFFWORKFLOW_BACKEND_CELERY_ENABLED"] is True + + +def queue_future_task_if_appropriate(process_instance: ProcessInstanceModel, eta_in_seconds: float, task_guid: str) -> bool: + if queue_enabled_for_process_model(process_instance): + buffer = 1 + countdown = eta_in_seconds - time.time() + buffer + args_to_celery = {"process_instance_id": process_instance.id, "task_guid": task_guid} + # add buffer to countdown to avoid rounding issues and race conditions with spiff. the situation we want to avoid is where + # we think the timer said to run it at 6:34:11, and we initialize the SpiffWorkflow library, + # expecting the timer to be ready, but the library considered it ready a little after that time + # (maybe due to subsecond stuff, maybe because of clock skew within the cluster of computers running spiff) + # celery_task_process_instance_run.apply_async(kwargs=args_to_celery, countdown=countdown + 1) # type: ignore + + celery.current_app.send_task(CELERY_TASK_PROCESS_INSTANCE_RUN, kwargs=args_to_celery, countdown=countdown) + return True + + return False + + +# if waiting, check all waiting tasks and see if theyt are timers. if they are timers, it's not runnable. +def queue_process_instance_if_appropriate(process_instance: ProcessInstanceModel) -> bool: + if queue_enabled_for_process_model(process_instance): + celery.current_app.send_task(CELERY_TASK_PROCESS_INSTANCE_RUN, (process_instance.id,)) + return True + + return False diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/background_processing/celery_worker.py b/spiffworkflow-backend/src/spiffworkflow_backend/background_processing/celery_worker.py new file mode 100644 index 00000000..c63d1c04 --- /dev/null +++ b/spiffworkflow-backend/src/spiffworkflow_backend/background_processing/celery_worker.py @@ -0,0 +1,7 @@ +from spiffworkflow_backend import create_app +from spiffworkflow_backend.background_processing.celery_tasks.process_instance_task import ( + celery_task_process_instance_run, # noqa: F401 +) + +the_flask_app = create_app() +setting_variable_to_make_celery_happy_no_idea_how_this_works = the_flask_app.celery_app diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/config/__init__.py b/spiffworkflow-backend/src/spiffworkflow_backend/config/__init__.py index 90f3b948..e519599c 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/config/__init__.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/config/__init__.py @@ -90,9 +90,7 @@ def _set_up_tenant_specific_fields_as_list_of_strings(app: Flask) -> None: else: app.config["SPIFFWORKFLOW_BACKEND_OPEN_ID_TENANT_SPECIFIC_FIELDS"] = tenant_specific_fields.split(",") if len(app.config["SPIFFWORKFLOW_BACKEND_OPEN_ID_TENANT_SPECIFIC_FIELDS"]) > 3: - raise ConfigurationError( - "SPIFFWORKFLOW_BACKEND_OPEN_ID_TENANT_SPECIFIC_FIELDS can have a maximum of 3 fields" - ) + raise ConfigurationError("SPIFFWORKFLOW_BACKEND_OPEN_ID_TENANT_SPECIFIC_FIELDS can have a maximum of 3 fields") def _check_extension_api_configs(app: Flask) -> None: @@ -239,6 +237,13 @@ def setup_config(app: Flask) -> None: } ] + if app.config["SPIFFWORKFLOW_BACKEND_CELERY_ENABLED"]: + app.config["SPIFFWORKFLOW_BACKEND_ENGINE_STEP_DEFAULT_STRATEGY_BACKGROUND"] = "queue_instructions_for_end_user" + app.config["SPIFFWORKFLOW_BACKEND_ENGINE_STEP_DEFAULT_STRATEGY_WEB"] = "queue_instructions_for_end_user" + else: + app.config["SPIFFWORKFLOW_BACKEND_ENGINE_STEP_DEFAULT_STRATEGY_BACKGROUND"] = "greedy" + app.config["SPIFFWORKFLOW_BACKEND_ENGINE_STEP_DEFAULT_STRATEGY_WEB"] = "run_until_user_message" + thread_local_data = threading.local() app.config["THREAD_LOCAL_DATA"] = thread_local_data _set_up_tenant_specific_fields_as_list_of_strings(app) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/config/default.py b/spiffworkflow-backend/src/spiffworkflow_backend/config/default.py index 826d3aea..01948163 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/config/default.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/config/default.py @@ -49,9 +49,16 @@ config_from_env("SPIFFWORKFLOW_BACKEND_RUN_BACKGROUND_SCHEDULER_IN_CREATE_APP", config_from_env("SPIFFWORKFLOW_BACKEND_BACKGROUND_SCHEDULER_ALLOW_OPTIMISTIC_CHECKS", default=True) config_from_env("SPIFFWORKFLOW_BACKEND_BACKGROUND_SCHEDULER_POLLING_INTERVAL_IN_SECONDS", default=10) config_from_env("SPIFFWORKFLOW_BACKEND_BACKGROUND_SCHEDULER_NOT_STARTED_POLLING_INTERVAL_IN_SECONDS", default=30) -config_from_env( - "SPIFFWORKFLOW_BACKEND_BACKGROUND_SCHEDULER_USER_INPUT_REQUIRED_POLLING_INTERVAL_IN_SECONDS", default=120 -) +config_from_env("SPIFFWORKFLOW_BACKEND_BACKGROUND_SCHEDULER_USER_INPUT_REQUIRED_POLLING_INTERVAL_IN_SECONDS", default=120) + +### background with celery +config_from_env("SPIFFWORKFLOW_BACKEND_CELERY_ENABLED", default=False) +config_from_env("SPIFFWORKFLOW_BACKEND_CELERY_BROKER_URL", default="redis://localhost") +config_from_env("SPIFFWORKFLOW_BACKEND_CELERY_RESULT_BACKEND", default="redis://localhost") + +# give a little overlap to ensure we do not miss items although the query will handle it either way +config_from_env("SPIFFWORKFLOW_BACKEND_BACKGROUND_SCHEDULER_FUTURE_TASK_LOOKAHEAD_IN_SECONDS", default=301) +config_from_env("SPIFFWORKFLOW_BACKEND_BACKGROUND_SCHEDULER_FUTURE_TASK_EXECUTION_INTERVAL_IN_SECONDS", default=300) ### frontend config_from_env("SPIFFWORKFLOW_BACKEND_URL_FOR_FRONTEND", default="http://localhost:7001") @@ -147,10 +154,6 @@ config_from_env("SPIFFWORKFLOW_BACKEND_GIT_USER_EMAIL") config_from_env("SPIFFWORKFLOW_BACKEND_GITHUB_WEBHOOK_SECRET") config_from_env("SPIFFWORKFLOW_BACKEND_GIT_SSH_PRIVATE_KEY_PATH") -### engine -config_from_env("SPIFFWORKFLOW_BACKEND_ENGINE_STEP_DEFAULT_STRATEGY_BACKGROUND", default="greedy") -config_from_env("SPIFFWORKFLOW_BACKEND_ENGINE_STEP_DEFAULT_STRATEGY_WEB", default="run_until_user_message") - ### element units # disabling until we fix the "no such directory" error so we do not keep sending cypress errors config_from_env("SPIFFWORKFLOW_BACKEND_ELEMENT_UNITS_CACHE_DIR", default="src/instance/element-unit-cache") diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/config/normalized_environment.py b/spiffworkflow-backend/src/spiffworkflow_backend/config/normalized_environment.py index 9cfcb208..42bec328 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/config/normalized_environment.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/config/normalized_environment.py @@ -8,9 +8,7 @@ def normalized_environment(key_values: os._Environ) -> dict: results = _parse_environment(key_values) if isinstance(results, dict): return results - raise Exception( - f"results from parsing environment variables was not a dict. This is troubling. Results were: {results}" - ) + raise Exception(f"results from parsing environment variables was not a dict. This is troubling. Results were: {results}") # source originally from: https://charemza.name/blog/posts/software-engineering/devops/structured-data-in-environment-variables/ @@ -78,9 +76,7 @@ def _parse_environment(key_values: os._Environ | dict) -> list | dict: ) def items_with_first_component(items: Iterable, first_component: str) -> dict: - return { - get_later_components(key): value for key, value in items if get_first_component(key) == first_component - } + return {get_later_components(key): value for key, value in items if get_first_component(key) == first_component} nested_structured_dict = { **without_more_components, @@ -101,8 +97,6 @@ def _parse_environment(key_values: os._Environ | dict) -> list | dict: return all(is_int(key) for key, value in nested_structured_dict.items()) def list_sorted_by_int_key() -> list: - return [ - value for key, value in sorted(nested_structured_dict.items(), key=lambda key_value: int(key_value[0])) - ] + return [value for key, value in sorted(nested_structured_dict.items(), key=lambda key_value: int(key_value[0]))] return list_sorted_by_int_key() if all_keys_are_ints() else nested_structured_dict diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/config/qa2.py b/spiffworkflow-backend/src/spiffworkflow_backend/config/qa2.py index 250dba7f..f99d3ad7 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/config/qa2.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/config/qa2.py @@ -1,9 +1,7 @@ """qa2 just here as an example of path based routing for apps.""" from os import environ -SPIFFWORKFLOW_BACKEND_PERMISSIONS_FILE_NAME = environ.get( - "SPIFFWORKFLOW_BACKEND_PERMISSIONS_FILE_NAME", default="qa1.yml" -) +SPIFFWORKFLOW_BACKEND_PERMISSIONS_FILE_NAME = environ.get("SPIFFWORKFLOW_BACKEND_PERMISSIONS_FILE_NAME", default="qa1.yml") SPIFFWORKFLOW_BACKEND_URL_FOR_FRONTEND = "https://qa2.spiffworkflow.org" SPIFFWORKFLOW_BACKEND_OPEN_ID_SERVER_URL = "https://qa2.spiffworkflow.org/keycloak/realms/spiffworkflow" SPIFFWORKFLOW_BACKEND_URL = "https://qa2.spiffworkflow.org/api" diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/data_migrations/process_instance_migrator.py b/spiffworkflow-backend/src/spiffworkflow_backend/data_migrations/process_instance_migrator.py index 343f93db..f476b85e 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/data_migrations/process_instance_migrator.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/data_migrations/process_instance_migrator.py @@ -57,9 +57,7 @@ class ProcessInstanceMigrator: @classmethod @benchmark_log_func - def run_version( - cls, data_migration_version_class: DataMigrationBase, process_instance: ProcessInstanceModel - ) -> None: + def run_version(cls, data_migration_version_class: DataMigrationBase, process_instance: ProcessInstanceModel) -> None: if process_instance.spiff_serializer_version < data_migration_version_class.version(): data_migration_version_class.run(process_instance) process_instance.spiff_serializer_version = data_migration_version_class.version() diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/data_migrations/version_1_3.py b/spiffworkflow-backend/src/spiffworkflow_backend/data_migrations/version_1_3.py index f93730cf..ff7a6ce4 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/data_migrations/version_1_3.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/data_migrations/version_1_3.py @@ -53,9 +53,7 @@ class VersionOneThree: def process_task_definition(self, task_definition: TaskDefinitionModel) -> None: task_definition.typename = task_definition.typename.replace("_BoundaryEventParent", "BoundaryEventSplit") - task_definition.bpmn_identifier = task_definition.bpmn_identifier.replace( - "BoundaryEventParent", "BoundaryEventSplit" - ) + task_definition.bpmn_identifier = task_definition.bpmn_identifier.replace("BoundaryEventParent", "BoundaryEventSplit") properties_json = copy.copy(task_definition.properties_json) properties_json.pop("main_child_task_spec", None) @@ -65,9 +63,7 @@ class VersionOneThree: # mostly for ExclusiveGateways if "cond_task_specs" in properties_json and properties_json["cond_task_specs"] is not None: for cond_task_spec in properties_json["cond_task_specs"]: - cond_task_spec["task_spec"] = cond_task_spec["task_spec"].replace( - "BoundaryEventParent", "BoundaryEventSplit" - ) + cond_task_spec["task_spec"] = cond_task_spec["task_spec"].replace("BoundaryEventParent", "BoundaryEventSplit") if "default_task_spec" in properties_json and properties_json["default_task_spec"] is not None: properties_json["default_task_spec"] = properties_json["default_task_spec"].replace( "BoundaryEventParent", "BoundaryEventSplit" @@ -208,9 +204,7 @@ class VersionOneThree: something_changed = False if "escalation_code" in properties_json["event_definition"]: - properties_json["event_definition"]["code"] = properties_json["event_definition"].pop( - "escalation_code" - ) + properties_json["event_definition"]["code"] = properties_json["event_definition"].pop("escalation_code") something_changed = True if "error_code" in properties_json["event_definition"]: properties_json["event_definition"]["code"] = properties_json["event_definition"].pop("error_code") @@ -225,9 +219,7 @@ class VersionOneThree: if current_app.config.get("SPIFFWORKFLOW_BACKEND_DATABASE_TYPE") == "postgres": task_models = ( db.session.query(TaskModel) - .filter( - TaskModel.properties_json.op("->>")("last_state_changed") == None # type: ignore # noqa: E711 - ) + .filter(TaskModel.properties_json.op("->>")("last_state_changed") == None) # type: ignore # noqa: E711 .all() ) else: diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/data_migrations/version_2.py b/spiffworkflow-backend/src/spiffworkflow_backend/data_migrations/version_2.py index 308e65e6..59571182 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/data_migrations/version_2.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/data_migrations/version_2.py @@ -31,9 +31,7 @@ class Version2(DataMigrationBase): task_service.save_objects_to_database(save_process_instance_events=False) except Exception as ex: - current_app.logger.warning( - f"Failed to migrate process_instance '{process_instance.id}'. The error was {str(ex)}" - ) + current_app.logger.warning(f"Failed to migrate process_instance '{process_instance.id}'. The error was {str(ex)}") @classmethod def update_spiff_task_parents(cls, spiff_task: SpiffTask, task_service: TaskService) -> None: diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/data_migrations/version_3.py b/spiffworkflow-backend/src/spiffworkflow_backend/data_migrations/version_3.py index 31dd7fef..2576d363 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/data_migrations/version_3.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/data_migrations/version_3.py @@ -38,6 +38,4 @@ class Version3(DataMigrationBase): db.session.add(bpmn_process) except Exception as ex: - current_app.logger.warning( - f"Failed to migrate process_instance '{process_instance.id}'. The error was {str(ex)}" - ) + current_app.logger.warning(f"Failed to migrate process_instance '{process_instance.id}'. The error was {str(ex)}") diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/data_stores/json.py b/spiffworkflow-backend/src/spiffworkflow_backend/data_stores/json.py index c0925769..dce1c574 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/data_stores/json.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/data_stores/json.py @@ -117,9 +117,7 @@ class JSONFileDataStore(BpmnDataStoreSpecification): # type: ignore location = _data_store_location_for_task(my_task, self.bpmn_id) if location is None: raise Exception(f"Unable to read from data store '{self.bpmn_id}' using location '{location}'.") - contents = FileSystemService.contents_of_json_file_at_relative_path( - location, _data_store_filename(self.bpmn_id) - ) + contents = FileSystemService.contents_of_json_file_at_relative_path(location, _data_store_filename(self.bpmn_id)) my_task.data[self.bpmn_id] = contents def set(self, my_task: SpiffTask) -> None: diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/data_stores/kkv.py b/spiffworkflow-backend/src/spiffworkflow_backend/data_stores/kkv.py index 8ab0ec0e..9b6eae91 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/data_stores/kkv.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/data_stores/kkv.py @@ -38,11 +38,7 @@ class KKVDataStore(BpmnDataStoreSpecification, DataStoreCRUD): # type: ignore } def _get_model(self, top_level_key: str, secondary_key: str) -> KKVDataStoreModel | None: - model = ( - db.session.query(KKVDataStoreModel) - .filter_by(top_level_key=top_level_key, secondary_key=secondary_key) - .first() - ) + model = db.session.query(KKVDataStoreModel).filter_by(top_level_key=top_level_key, secondary_key=secondary_key).first() return model def _delete_all_for_top_level_key(self, top_level_key: str) -> None: diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/data_stores/typeahead.py b/spiffworkflow-backend/src/spiffworkflow_backend/data_stores/typeahead.py index 6a3f7922..21b2c3f6 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/data_stores/typeahead.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/data_stores/typeahead.py @@ -25,9 +25,7 @@ class TypeaheadDataStore(BpmnDataStoreSpecification, DataStoreCRUD): # type: ig @staticmethod def query_data_store(name: str) -> Any: - return TypeaheadModel.query.filter_by(category=name).order_by( - TypeaheadModel.category, TypeaheadModel.search_term - ) + return TypeaheadModel.query.filter_by(category=name).order_by(TypeaheadModel.category, TypeaheadModel.search_term) @staticmethod def build_response_item(model: Any) -> dict[str, Any]: diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/exceptions/api_error.py b/spiffworkflow-backend/src/spiffworkflow_backend/exceptions/api_error.py index b82f7ee7..2c959a70 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/exceptions/api_error.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/exceptions/api_error.py @@ -273,9 +273,7 @@ def handle_exception(exception: Exception) -> flask.wrappers.Response: id = capture_exception(exception) if isinstance(exception, ApiError): - current_app.logger.info( - f"Sending ApiError exception to sentry: {exception} with error code {exception.error_code}" - ) + current_app.logger.info(f"Sending ApiError exception to sentry: {exception} with error code {exception.error_code}") organization_slug = current_app.config.get("SPIFFWORKFLOW_BACKEND_SENTRY_ORGANIZATION_SLUG") project_slug = current_app.config.get("SPIFFWORKFLOW_BACKEND_SENTRY_PROJECT_SLUG") @@ -308,9 +306,7 @@ def handle_exception(exception: Exception) -> flask.wrappers.Response: if isinstance(exception, ApiError): api_exception = exception elif isinstance(exception, SpiffWorkflowException): - api_exception = ApiError.from_workflow_exception( - "unexpected_workflow_exception", "Unexpected Workflow Error", exception - ) + api_exception = ApiError.from_workflow_exception("unexpected_workflow_exception", "Unexpected Workflow Error", exception) else: api_exception = ApiError( error_code=error_code, diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/load_database_models.py b/spiffworkflow-backend/src/spiffworkflow_backend/load_database_models.py index 52062985..e20ef9f2 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/load_database_models.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/load_database_models.py @@ -106,6 +106,9 @@ from spiffworkflow_backend.models.user_property import ( from spiffworkflow_backend.models.service_account import ( ServiceAccountModel, ) # noqa: F401 +from spiffworkflow_backend.models.future_task import ( + FutureTaskModel, +) # noqa: F401 from spiffworkflow_backend.models.feature_flag import ( FeatureFlagModel, ) # noqa: F401 diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/db.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/db.py index 3b6f8837..12e0ae6d 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/db.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/db.py @@ -70,9 +70,7 @@ class SpiffworkflowBaseDBModel(db.Model): # type: ignore raise -def update_created_modified_on_create_listener( - mapper: Mapper, _connection: Connection, target: SpiffworkflowBaseDBModel -) -> None: +def update_created_modified_on_create_listener(mapper: Mapper, _connection: Connection, target: SpiffworkflowBaseDBModel) -> None: """Event listener that runs before a record is updated, and sets the create/modified field accordingly.""" if "created_at_in_seconds" in mapper.columns.keys(): target.created_at_in_seconds = round(time.time()) @@ -80,9 +78,7 @@ def update_created_modified_on_create_listener( target.updated_at_in_seconds = round(time.time()) -def update_modified_on_update_listener( - mapper: Mapper, _connection: Connection, target: SpiffworkflowBaseDBModel -) -> None: +def update_modified_on_update_listener(mapper: Mapper, _connection: Connection, target: SpiffworkflowBaseDBModel) -> None: """Event listener that runs before a record is updated, and sets the modified field accordingly.""" if "updated_at_in_seconds" in mapper.columns.keys(): if db.session.is_modified(target, include_collections=False): diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/future_task.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/future_task.py new file mode 100644 index 00000000..d9d418ad --- /dev/null +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/future_task.py @@ -0,0 +1,48 @@ +import time +from dataclasses import dataclass + +from flask import current_app +from sqlalchemy.dialects.mysql import insert as mysql_insert +from sqlalchemy.dialects.postgresql import insert as postgres_insert +from sqlalchemy.dialects.sqlite import insert as sqlite_insert + +from spiffworkflow_backend.models.db import SpiffworkflowBaseDBModel +from spiffworkflow_backend.models.db import db + + +@dataclass +class FutureTaskModel(SpiffworkflowBaseDBModel): + __tablename__ = "future_task" + + guid: str = db.Column(db.String(36), primary_key=True) + run_at_in_seconds: int = db.Column(db.Integer, nullable=False, index=True) + completed: bool = db.Column(db.Boolean, default=False, nullable=False, index=True) + + updated_at_in_seconds: int = db.Column(db.Integer, nullable=False) + + @classmethod + def insert_or_update(cls, guid: str, run_at_in_seconds: int) -> None: + task_info = [ + { + "guid": guid, + "run_at_in_seconds": run_at_in_seconds, + "updated_at_in_seconds": round(time.time()), + } + ] + on_duplicate_key_stmt = None + if current_app.config["SPIFFWORKFLOW_BACKEND_DATABASE_TYPE"] == "mysql": + insert_stmt = mysql_insert(FutureTaskModel).values(task_info) + on_duplicate_key_stmt = insert_stmt.on_duplicate_key_update( + run_at_in_seconds=insert_stmt.inserted.run_at_in_seconds, updated_at_in_seconds=round(time.time()) + ) + else: + insert_stmt = None + if current_app.config["SPIFFWORKFLOW_BACKEND_DATABASE_TYPE"] == "sqlite": + insert_stmt = sqlite_insert(FutureTaskModel).values(task_info) + else: + insert_stmt = postgres_insert(FutureTaskModel).values(task_info) + on_duplicate_key_stmt = insert_stmt.on_conflict_do_update( + index_elements=["guid"], + set_={"run_at_in_seconds": run_at_in_seconds, "updated_at_in_seconds": round(time.time())}, + ) + db.session.execute(on_duplicate_key_stmt) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/human_task.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/human_task.py index 635a3155..dadac51f 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/human_task.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/human_task.py @@ -24,9 +24,7 @@ class HumanTaskModel(SpiffworkflowBaseDBModel): __tablename__ = "human_task" id: int = db.Column(db.Integer, primary_key=True) - process_instance_id: int = db.Column( - ForeignKey(ProcessInstanceModel.id), nullable=False, index=True # type: ignore - ) + process_instance_id: int = db.Column(ForeignKey(ProcessInstanceModel.id), nullable=False, index=True) # type: ignore lane_assignment_id: int | None = db.Column(ForeignKey(GroupModel.id), index=True) completed_by_user_id: int = db.Column(ForeignKey(UserModel.id), nullable=True, index=True) # type: ignore diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/json_data.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/json_data.py index b0c6551a..c1344f01 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/json_data.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/json_data.py @@ -61,9 +61,7 @@ class JsonDataModel(SpiffworkflowBaseDBModel): return cls.find_object_by_hash(hash).data @classmethod - def insert_or_update_json_data_records( - cls, json_data_hash_to_json_data_dict_mapping: dict[str, JsonDataDict] - ) -> None: + def insert_or_update_json_data_records(cls, json_data_hash_to_json_data_dict_mapping: dict[str, JsonDataDict]) -> None: list_of_dicts = [*json_data_hash_to_json_data_dict_mapping.values()] if len(list_of_dicts) > 0: on_duplicate_key_stmt = None diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/message_instance.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/message_instance.py index e342f7a8..07938d8a 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/message_instance.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/message_instance.py @@ -51,15 +51,11 @@ class MessageInstanceModel(SpiffworkflowBaseDBModel): status: str = db.Column(db.String(20), nullable=False, default="ready", index=True) user_id: int = db.Column(ForeignKey(UserModel.id), nullable=True, index=True) # type: ignore user = relationship("UserModel") - counterpart_id: int = db.Column( - db.Integer - ) # Not enforcing self-referential foreign key so we can delete messages. + counterpart_id: int = db.Column(db.Integer) # Not enforcing self-referential foreign key so we can delete messages. failure_cause: str = db.Column(db.Text()) updated_at_in_seconds: int = db.Column(db.Integer) created_at_in_seconds: int = db.Column(db.Integer) - correlation_rules = relationship( - "MessageInstanceCorrelationRuleModel", back_populates="message_instance", cascade="delete" - ) + correlation_rules = relationship("MessageInstanceCorrelationRuleModel", back_populates="message_instance", cascade="delete") @validates("message_type") def validate_message_type(self, key: str, value: Any) -> Any: diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/process_group.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/process_group.py index b5ec8feb..2537387b 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/process_group.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/process_group.py @@ -66,12 +66,8 @@ class ProcessGroupSchema(Schema): "process_groups", ] - process_models = marshmallow.fields.List( - marshmallow.fields.Nested("ProcessModelInfoSchema", dump_only=True, required=False) - ) - process_groups = marshmallow.fields.List( - marshmallow.fields.Nested("ProcessGroupSchema", dump_only=True, required=False) - ) + process_models = marshmallow.fields.List(marshmallow.fields.Nested("ProcessModelInfoSchema", dump_only=True, required=False)) + process_groups = marshmallow.fields.List(marshmallow.fields.Nested("ProcessGroupSchema", dump_only=True, required=False)) @post_load def make_process_group(self, data: dict[str, str | bool | int], **kwargs: dict) -> ProcessGroup: diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/process_instance.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/process_instance.py index 0408c4df..ef0b4c0a 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/process_instance.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/process_instance.py @@ -30,13 +30,14 @@ class ProcessInstanceCannotBeDeletedError(Exception): class ProcessInstanceStatus(SpiffEnum): - not_started = "not_started" - user_input_required = "user_input_required" - waiting = "waiting" complete = "complete" error = "error" + not_started = "not_started" + running = "running" suspended = "suspended" terminated = "terminated" + user_input_required = "user_input_required" + waiting = "waiting" class ProcessInstanceModel(SpiffworkflowBaseDBModel): @@ -58,9 +59,7 @@ class ProcessInstanceModel(SpiffworkflowBaseDBModel): active_human_tasks = relationship( "HumanTaskModel", - primaryjoin=( - "and_(HumanTaskModel.process_instance_id==ProcessInstanceModel.id, HumanTaskModel.completed == False)" - ), + primaryjoin="and_(HumanTaskModel.process_instance_id==ProcessInstanceModel.id, HumanTaskModel.completed == False)", ) # type: ignore bpmn_process = relationship(BpmnProcessModel, cascade="delete") @@ -103,10 +102,13 @@ class ProcessInstanceModel(SpiffworkflowBaseDBModel): # full, none persistence_level: str = "full" + actions: dict | None = None + def serialized(self) -> dict[str, Any]: """Return object data in serializeable format.""" return { "id": self.id, + "actions": self.actions, "bpmn_version_control_identifier": self.bpmn_version_control_identifier, "bpmn_version_control_type": self.bpmn_version_control_type, "bpmn_xml_file_contents_retrieval_error": self.bpmn_xml_file_contents_retrieval_error, @@ -127,9 +129,7 @@ class ProcessInstanceModel(SpiffworkflowBaseDBModel): def serialized_with_metadata(self) -> dict[str, Any]: process_instance_attributes = self.serialized() process_instance_attributes["process_metadata"] = self.process_metadata - process_instance_attributes["process_model_with_diagram_identifier"] = ( - self.process_model_with_diagram_identifier - ) + process_instance_attributes["process_model_with_diagram_identifier"] = self.process_model_with_diagram_identifier return process_instance_attributes @validates("status") @@ -146,6 +146,9 @@ class ProcessInstanceModel(SpiffworkflowBaseDBModel): def has_terminal_status(self) -> bool: return self.status in self.terminal_statuses() + def is_immediately_runnable(self) -> bool: + return self.status in self.immediately_runnable_statuses() + @classmethod def terminal_statuses(cls) -> list[str]: return ["complete", "error", "terminated"] @@ -157,7 +160,11 @@ class ProcessInstanceModel(SpiffworkflowBaseDBModel): @classmethod def active_statuses(cls) -> list[str]: - return ["not_started", "user_input_required", "waiting"] + return cls.immediately_runnable_statuses() + ["user_input_required", "waiting"] + + @classmethod + def immediately_runnable_statuses(cls) -> list[str]: + return ["not_started", "running"] class ProcessInstanceModelSchema(Schema): diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/process_instance_file_data.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/process_instance_file_data.py index e2d0fcb4..22a6c337 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/process_instance_file_data.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/process_instance_file_data.py @@ -13,9 +13,7 @@ class ProcessInstanceFileDataModel(SpiffworkflowBaseDBModel): __tablename__ = "process_instance_file_data" id: int = db.Column(db.Integer, primary_key=True) - process_instance_id: int = db.Column( - ForeignKey(ProcessInstanceModel.id), nullable=False, index=True # type: ignore - ) + process_instance_id: int = db.Column(ForeignKey(ProcessInstanceModel.id), nullable=False, index=True) # type: ignore identifier: str = db.Column(db.String(255), nullable=False) list_index: int | None = db.Column(db.Integer, nullable=True) mimetype: str = db.Column(db.String(255), nullable=False) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/process_instance_metadata.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/process_instance_metadata.py index ba58c7f8..cca3c211 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/process_instance_metadata.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/process_instance_metadata.py @@ -13,9 +13,7 @@ class ProcessInstanceMetadataModel(SpiffworkflowBaseDBModel): __table_args__ = (db.UniqueConstraint("process_instance_id", "key", name="process_instance_metadata_unique"),) id: int = db.Column(db.Integer, primary_key=True) - process_instance_id: int = db.Column( - ForeignKey(ProcessInstanceModel.id), nullable=False, index=True # type: ignore - ) + process_instance_id: int = db.Column(ForeignKey(ProcessInstanceModel.id), nullable=False, index=True) # type: ignore key: str = db.Column(db.String(255), nullable=False, index=True) value: str = db.Column(db.String(255), nullable=False) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/process_instance_queue.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/process_instance_queue.py index f56eee8e..a16177a2 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/process_instance_queue.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/process_instance_queue.py @@ -12,9 +12,7 @@ class ProcessInstanceQueueModel(SpiffworkflowBaseDBModel): __tablename__ = "process_instance_queue" id: int = db.Column(db.Integer, primary_key=True) - process_instance_id: int = db.Column( - ForeignKey(ProcessInstanceModel.id), unique=True, nullable=False # type: ignore - ) + process_instance_id: int = db.Column(ForeignKey(ProcessInstanceModel.id), unique=True, nullable=False) # type: ignore priority: int = db.Column(db.Integer) locked_by: str | None = db.Column(db.String(80), index=True, nullable=True) locked_at_in_seconds: int | None = db.Column(db.Integer, index=True, nullable=True) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/process_instance_report.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/process_instance_report.py index f9d09548..9770c192 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/process_instance_report.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/process_instance_report.py @@ -110,9 +110,7 @@ class ProcessInstanceReportModel(SpiffworkflowBaseDBModel): ).first() if process_instance_report is not None: - raise ProcessInstanceReportAlreadyExistsError( - f"Process instance report with identifier already exists: {identifier}" - ) + raise ProcessInstanceReportAlreadyExistsError(f"Process instance report with identifier already exists: {identifier}") report_metadata_dict = typing.cast(dict[str, Any], report_metadata) json_data_hash = JsonDataModel.create_and_insert_json_data_from_dict(report_metadata_dict) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/reference_cache.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/reference_cache.py index 7462a565..c506e28e 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/reference_cache.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/reference_cache.py @@ -64,9 +64,7 @@ class ReferenceCacheModel(SpiffworkflowBaseDBModel): """A cache of information about all the Processes and Decisions defined in all files.""" __tablename__ = "reference_cache" - __table_args__ = ( - UniqueConstraint("generation_id", "identifier", "relative_location", "type", name="reference_cache_uniq"), - ) + __table_args__ = (UniqueConstraint("generation_id", "identifier", "relative_location", "type", name="reference_cache_uniq"),) id: int = db.Column(db.Integer, primary_key=True) generation_id: int = db.Column(ForeignKey(CacheGenerationModel.id), nullable=False, index=True) # type: ignore diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/task.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/task.py index 3038f681..2b32dac8 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/task.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/task.py @@ -155,6 +155,7 @@ class Task: error_message: str | None = None, assigned_user_group_identifier: str | None = None, potential_owner_usernames: str | None = None, + process_model_uses_queued_execution: bool | None = None, ): self.id = id self.name = name @@ -167,6 +168,7 @@ class Task: self.lane = lane self.parent = parent self.event_definition = event_definition + self.process_model_uses_queued_execution = process_model_uses_queued_execution self.data = data if self.data is None: @@ -228,6 +230,7 @@ class Task: "error_message": self.error_message, "assigned_user_group_identifier": self.assigned_user_group_identifier, "potential_owner_usernames": self.potential_owner_usernames, + "process_model_uses_queued_execution": self.process_model_uses_queued_execution, } @classmethod diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/task_draft_data.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/task_draft_data.py index 24c5e216..0fd16c1e 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/models/task_draft_data.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/task_draft_data.py @@ -39,9 +39,7 @@ class TaskDraftDataModel(SpiffworkflowBaseDBModel): ), ) - process_instance_id: int = db.Column( - ForeignKey(ProcessInstanceModel.id), nullable=False, index=True # type: ignore - ) + process_instance_id: int = db.Column(ForeignKey(ProcessInstanceModel.id), nullable=False, index=True) # type: ignore # a colon delimited path of bpmn_process_definition_ids for a given task task_definition_id_path: str = db.Column(db.String(255), nullable=False, index=True) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/models/task_instructions_for_end_user.py b/spiffworkflow-backend/src/spiffworkflow_backend/models/task_instructions_for_end_user.py new file mode 100644 index 00000000..737428a1 --- /dev/null +++ b/spiffworkflow-backend/src/spiffworkflow_backend/models/task_instructions_for_end_user.py @@ -0,0 +1,65 @@ +from __future__ import annotations + +import time +from dataclasses import dataclass + +from flask import current_app +from sqlalchemy import ForeignKey +from sqlalchemy import desc +from sqlalchemy.dialects.mysql import insert as mysql_insert +from sqlalchemy.dialects.postgresql import insert as postgres_insert + +from spiffworkflow_backend.models.db import SpiffworkflowBaseDBModel +from spiffworkflow_backend.models.db import db + + +@dataclass +class TaskInstructionsForEndUserModel(SpiffworkflowBaseDBModel): + __tablename__ = "task_instructions_for_end_user" + + task_guid: str = db.Column(db.String(36), primary_key=True) + instruction: str = db.Column(db.Text(), nullable=False) + process_instance_id: int = db.Column(ForeignKey("process_instance.id"), nullable=False, index=True) + has_been_retrieved: bool = db.Column(db.Boolean, nullable=False, default=False, index=True) + + # we need this to maintain order + timestamp: float = db.Column(db.DECIMAL(17, 6), nullable=False, index=True) + + @classmethod + def insert_or_update_record(cls, task_guid: str, process_instance_id: int, instruction: str) -> None: + record = [ + { + "task_guid": task_guid, + "process_instance_id": process_instance_id, + "instruction": instruction, + "timestamp": time.time(), + } + ] + on_duplicate_key_stmt = None + if current_app.config["SPIFFWORKFLOW_BACKEND_DATABASE_TYPE"] == "mysql": + insert_stmt = mysql_insert(TaskInstructionsForEndUserModel).values(record) + on_duplicate_key_stmt = insert_stmt.prefix_with("IGNORE") + # on_duplicate_key_stmt = insert_stmt.on_duplicate_key_update(instruction=insert_stmt.inserted.instruction) + else: + insert_stmt = postgres_insert(TaskInstructionsForEndUserModel).values(record) + on_duplicate_key_stmt = insert_stmt.on_conflict_do_nothing(index_elements=["task_guid"]) + db.session.execute(on_duplicate_key_stmt) + + @classmethod + def entries_for_process_instance(cls, process_instance_id: int) -> list[TaskInstructionsForEndUserModel]: + entries: list[TaskInstructionsForEndUserModel] = ( + cls.query.filter_by(process_instance_id=process_instance_id, has_been_retrieved=False) + .order_by(desc(TaskInstructionsForEndUserModel.timestamp)) # type: ignore + .all() + ) + return entries + + @classmethod + def retrieve_and_clear(cls, process_instance_id: int) -> list[TaskInstructionsForEndUserModel]: + entries = cls.entries_for_process_instance(process_instance_id) + # convert to list[dict] here so we can remove the records from the db right after + for e in entries: + e.has_been_retrieved = True + db.session.add(e) + db.session.commit() + return entries diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/routes/active_users_controller.py b/spiffworkflow-backend/src/spiffworkflow_backend/routes/active_users_controller.py index 0350ba20..1022e264 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/routes/active_users_controller.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/routes/active_users_controller.py @@ -13,9 +13,7 @@ from spiffworkflow_backend.models.user import UserModel def active_user_updates(last_visited_identifier: str) -> Response: - active_user = ActiveUserModel.query.filter_by( - user_id=g.user.id, last_visited_identifier=last_visited_identifier - ).first() + active_user = ActiveUserModel.query.filter_by(user_id=g.user.id, last_visited_identifier=last_visited_identifier).first() if active_user is None: active_user = ActiveUserModel( user_id=g.user.id, last_visited_identifier=last_visited_identifier, last_seen_in_seconds=round(time.time()) @@ -39,9 +37,7 @@ def active_user_updates(last_visited_identifier: str) -> Response: def active_user_unregister(last_visited_identifier: str) -> flask.wrappers.Response: - active_user = ActiveUserModel.query.filter_by( - user_id=g.user.id, last_visited_identifier=last_visited_identifier - ).first() + active_user = ActiveUserModel.query.filter_by(user_id=g.user.id, last_visited_identifier=last_visited_identifier).first() if active_user is not None: db.session.delete(active_user) db.session.commit() diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/routes/authentication_controller.py b/spiffworkflow-backend/src/spiffworkflow_backend/routes/authentication_controller.py index 3a1e232d..cfaeff5c 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/routes/authentication_controller.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/routes/authentication_controller.py @@ -136,16 +136,12 @@ def login_return(code: str, state: str, session_state: str = "") -> Response | N state_dict = ast.literal_eval(base64.b64decode(state).decode("utf-8")) state_redirect_url = state_dict["redirect_url"] authentication_identifier = state_dict["authentication_identifier"] - auth_token_object = AuthenticationService().get_auth_token_object( - code, authentication_identifier=authentication_identifier - ) + auth_token_object = AuthenticationService().get_auth_token_object(code, authentication_identifier=authentication_identifier) if "id_token" in auth_token_object: id_token = auth_token_object["id_token"] user_info = _parse_id_token(id_token) - if AuthenticationService.validate_id_or_access_token( - id_token, authentication_identifier=authentication_identifier - ): + if AuthenticationService.validate_id_or_access_token(id_token, authentication_identifier=authentication_identifier): if user_info and "error" not in user_info: user_model = AuthorizationService.create_user_from_sign_in(user_info) g.user = user_model.id @@ -180,9 +176,7 @@ def login_return(code: str, state: str, session_state: str = "") -> Response | N def login_with_access_token(access_token: str, authentication_identifier: str) -> Response: user_info = _parse_id_token(access_token) - if AuthenticationService.validate_id_or_access_token( - access_token, authentication_identifier=authentication_identifier - ): + if AuthenticationService.validate_id_or_access_token(access_token, authentication_identifier=authentication_identifier): if user_info and "error" not in user_info: AuthorizationService.create_user_from_sign_in(user_info) else: @@ -262,9 +256,7 @@ def _set_new_access_token_in_cookie( response.set_cookie("id_token", tld.new_id_token, domain=domain_for_frontend_cookie) if hasattr(tld, "new_authentication_identifier") and tld.new_authentication_identifier: - response.set_cookie( - "authentication_identifier", tld.new_authentication_identifier, domain=domain_for_frontend_cookie - ) + response.set_cookie("authentication_identifier", tld.new_authentication_identifier, domain=domain_for_frontend_cookie) if hasattr(tld, "user_has_logged_out") and tld.user_has_logged_out: response.set_cookie("id_token", "", max_age=0, domain=domain_for_frontend_cookie) @@ -347,9 +339,7 @@ def _get_user_model_from_token(token: str) -> UserModel | None: try: user_model = _get_user_from_decoded_internal_token(decoded_token) except Exception as e: - current_app.logger.error( - f"Exception in verify_token getting user from decoded internal token. {e}" - ) + current_app.logger.error(f"Exception in verify_token getting user from decoded internal token. {e}") # if the user is forced logged out then stop processing the token if _force_logout_user_if_necessary(user_model): @@ -359,9 +349,7 @@ def _get_user_model_from_token(token: str) -> UserModel | None: user_info = None authentication_identifier = _get_authentication_identifier_from_request() try: - if AuthenticationService.validate_id_or_access_token( - token, authentication_identifier=authentication_identifier - ): + if AuthenticationService.validate_id_or_access_token(token, authentication_identifier=authentication_identifier): user_info = decoded_token except TokenExpiredError as token_expired_error: # Try to refresh the token @@ -437,9 +425,7 @@ def _get_user_from_decoded_internal_token(decoded_token: dict) -> UserModel | No parts = sub.split("::") service = parts[0].split(":")[1] service_id = parts[1].split(":")[1] - user: UserModel = ( - UserModel.query.filter(UserModel.service == service).filter(UserModel.service_id == service_id).first() - ) + user: UserModel = UserModel.query.filter(UserModel.service == service).filter(UserModel.service_id == service_id).first() if user: return user user = UserService.create_user(service_id, service, service_id) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/routes/data_store_controller.py b/spiffworkflow-backend/src/spiffworkflow_backend/routes/data_store_controller.py index f83ec5cd..1e652c61 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/routes/data_store_controller.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/routes/data_store_controller.py @@ -43,9 +43,7 @@ def _build_response(data_store_class: Any, name: str, page: int, per_page: int) return make_response(jsonify(response_json), 200) -def data_store_item_list( - data_store_type: str, name: str, page: int = 1, per_page: int = 100 -) -> flask.wrappers.Response: +def data_store_item_list(data_store_type: str, name: str, page: int = 1, per_page: int = 100) -> flask.wrappers.Response: """Returns a list of the items in a data store.""" if data_store_type == "typeahead": diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/routes/debug_controller.py b/spiffworkflow-backend/src/spiffworkflow_backend/routes/debug_controller.py index b46a0eef..0df120b6 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/routes/debug_controller.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/routes/debug_controller.py @@ -3,8 +3,8 @@ from flask import make_response from flask import request from flask.wrappers import Response -from spiffworkflow_backend import get_version_info_data from spiffworkflow_backend.services.authentication_service import AuthenticationService +from spiffworkflow_backend.services.monitoring_service import get_version_info_data def test_raise_error() -> Response: diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/routes/extensions_controller.py b/spiffworkflow-backend/src/spiffworkflow_backend/routes/extensions_controller.py index ddf57811..c44e4714 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/routes/extensions_controller.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/routes/extensions_controller.py @@ -189,9 +189,7 @@ def _run_extension( if ui_schema_action: if "results_markdown_filename" in ui_schema_action: - file_contents = SpecFileService.get_data( - process_model, ui_schema_action["results_markdown_filename"] - ).decode("utf-8") + file_contents = SpecFileService.get_data(process_model, ui_schema_action["results_markdown_filename"]).decode("utf-8") form_contents = JinjaService.render_jinja_template(file_contents, task_data=task_data) result["rendered_results_markdown"] = form_contents diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/routes/openid_blueprint/openid_blueprint.py b/spiffworkflow-backend/src/spiffworkflow_backend/routes/openid_blueprint/openid_blueprint.py index abeaf658..023619d7 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/routes/openid_blueprint/openid_blueprint.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/routes/openid_blueprint/openid_blueprint.py @@ -87,9 +87,7 @@ def token() -> Response | dict: code = request.values.get("code") if code is None: - return Response( - json.dumps({"error": "missing_code_value_in_token_request"}), status=400, mimetype="application/json" - ) + return Response(json.dumps({"error": "missing_code_value_in_token_request"}), status=400, mimetype="application/json") """We just stuffed the user name on the front of the code, so grab it.""" user_name, secret_hash = code.split(":") diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/routes/process_api_blueprint.py b/spiffworkflow-backend/src/spiffworkflow_backend/routes/process_api_blueprint.py index 4f024f79..777d6cca 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/routes/process_api_blueprint.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/routes/process_api_blueprint.py @@ -9,10 +9,14 @@ from flask import jsonify from flask import make_response from flask import request from flask.wrappers import Response +from sqlalchemy import and_ +from sqlalchemy import or_ from spiffworkflow_backend.exceptions.api_error import ApiError from spiffworkflow_backend.exceptions.process_entity_not_found_error import ProcessEntityNotFoundError from spiffworkflow_backend.models.db import db +from spiffworkflow_backend.models.human_task import HumanTaskModel +from spiffworkflow_backend.models.human_task_user import HumanTaskUserModel from spiffworkflow_backend.models.permission_assignment import PermissionAssignmentModel from spiffworkflow_backend.models.principal import PrincipalModel from spiffworkflow_backend.models.process_instance import ProcessInstanceModel @@ -265,3 +269,55 @@ def _find_principal_or_raise() -> PrincipalModel: ) ) return principal # type: ignore + + +def _find_process_instance_for_me_or_raise( + process_instance_id: int, + include_actions: bool = False, +) -> ProcessInstanceModel: + process_instance: ProcessInstanceModel | None = ( + ProcessInstanceModel.query.filter_by(id=process_instance_id) + .outerjoin(HumanTaskModel) + .outerjoin( + HumanTaskUserModel, + and_( + HumanTaskModel.id == HumanTaskUserModel.human_task_id, + HumanTaskUserModel.user_id == g.user.id, + ), + ) + .filter( + or_( + # you were allowed to complete it + HumanTaskUserModel.id.is_not(None), + # or you completed it (which admins can do even if it wasn't assigned via HumanTaskUserModel) + HumanTaskModel.completed_by_user_id == g.user.id, + # or you started it + ProcessInstanceModel.process_initiator_id == g.user.id, + ) + ) + .first() + ) + + if process_instance is None: + raise ( + ApiError( + error_code="process_instance_cannot_be_found", + message=f"Process instance with id {process_instance_id} cannot be found that is associated with you.", + status_code=400, + ) + ) + + if include_actions: + modified_process_model_identifier = ProcessModelInfo.modify_process_identifier_for_path_param( + process_instance.process_model_identifier + ) + target_uri = f"/v1.0/process-instances/for-me/{modified_process_model_identifier}/{process_instance.id}" + has_permission = AuthorizationService.user_has_permission( + user=g.user, + permission="read", + target_uri=target_uri, + ) + if has_permission: + process_instance.actions = {"read": {"path": target_uri, "method": "GET"}} + + return process_instance diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/routes/process_groups_controller.py b/spiffworkflow-backend/src/spiffworkflow_backend/routes/process_groups_controller.py index f0c0dffe..8ecd9034 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/routes/process_groups_controller.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/routes/process_groups_controller.py @@ -118,7 +118,5 @@ def process_group_show( def process_group_move(modified_process_group_identifier: str, new_location: str) -> flask.wrappers.Response: original_process_group_id = _un_modify_modified_process_model_id(modified_process_group_identifier) new_process_group = ProcessModelService.process_group_move(original_process_group_id, new_location) - _commit_and_push_to_git( - f"User: {g.user.username} moved process group {original_process_group_id} to {new_process_group.id}" - ) + _commit_and_push_to_git(f"User: {g.user.username} moved process group {original_process_group_id} to {new_process_group.id}") return make_response(jsonify(new_process_group), 200) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/routes/process_instance_events_controller.py b/spiffworkflow-backend/src/spiffworkflow_backend/routes/process_instance_events_controller.py index 1e45fbdb..ba1c1801 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/routes/process_instance_events_controller.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/routes/process_instance_events_controller.py @@ -69,9 +69,7 @@ def log_list( log_query = log_query.filter(ProcessInstanceEventModel.event_type == event_type) logs = ( - log_query.order_by( - ProcessInstanceEventModel.timestamp.desc(), ProcessInstanceEventModel.id.desc() # type: ignore - ) + log_query.order_by(ProcessInstanceEventModel.timestamp.desc(), ProcessInstanceEventModel.id.desc()) # type: ignore .outerjoin(UserModel, UserModel.id == ProcessInstanceEventModel.user_id) .add_columns( TaskModel.guid.label("spiff_task_guid"), # type: ignore diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/routes/process_instances_controller.py b/spiffworkflow-backend/src/spiffworkflow_backend/routes/process_instances_controller.py index 3dcf469f..a7fee495 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/routes/process_instances_controller.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/routes/process_instances_controller.py @@ -1,4 +1,6 @@ -"""APIs for dealing with process groups, process models, and process instances.""" +# black and ruff are in competition with each other in import formatting so ignore ruff +# ruff: noqa: I001 + import json from typing import Any @@ -8,17 +10,20 @@ from flask import g from flask import jsonify from flask import make_response from flask.wrappers import Response -from sqlalchemy import and_ from sqlalchemy import or_ from sqlalchemy.orm import aliased +from spiffworkflow_backend.background_processing.celery_tasks.process_instance_task_producer import ( + queue_enabled_for_process_model, +) +from spiffworkflow_backend.background_processing.celery_tasks.process_instance_task_producer import ( + queue_process_instance_if_appropriate, +) from spiffworkflow_backend.data_migrations.process_instance_migrator import ProcessInstanceMigrator from spiffworkflow_backend.exceptions.api_error import ApiError from spiffworkflow_backend.models.bpmn_process import BpmnProcessModel from spiffworkflow_backend.models.bpmn_process_definition import BpmnProcessDefinitionModel from spiffworkflow_backend.models.db import db -from spiffworkflow_backend.models.human_task import HumanTaskModel -from spiffworkflow_backend.models.human_task_user import HumanTaskUserModel from spiffworkflow_backend.models.json_data import JsonDataModel # noqa: F401 from spiffworkflow_backend.models.process_instance import ProcessInstanceApiSchema from spiffworkflow_backend.models.process_instance import ProcessInstanceCannotBeDeletedError @@ -33,6 +38,7 @@ from spiffworkflow_backend.models.reference_cache import ReferenceNotFoundError from spiffworkflow_backend.models.task import TaskModel from spiffworkflow_backend.models.task_definition import TaskDefinitionModel from spiffworkflow_backend.routes.process_api_blueprint import _find_process_instance_by_id_or_raise +from spiffworkflow_backend.routes.process_api_blueprint import _find_process_instance_for_me_or_raise from spiffworkflow_backend.routes.process_api_blueprint import _get_process_model from spiffworkflow_backend.routes.process_api_blueprint import _un_modify_modified_process_model_id from spiffworkflow_backend.services.authorization_service import AuthorizationService @@ -49,30 +55,6 @@ from spiffworkflow_backend.services.process_instance_service import ProcessInsta from spiffworkflow_backend.services.process_model_service import ProcessModelService from spiffworkflow_backend.services.task_service import TaskService -# from spiffworkflow_backend.services.process_instance_report_service import ( -# ProcessInstanceReportFilter, -# ) - - -def _process_instance_create( - process_model_identifier: str, -) -> ProcessInstanceModel: - process_model = _get_process_model(process_model_identifier) - if process_model.primary_file_name is None: - raise ApiError( - error_code="process_model_missing_primary_bpmn_file", - message=( - f"Process Model '{process_model_identifier}' does not have a primary" - " bpmn file. One must be set in order to instantiate this model." - ), - status_code=400, - ) - - process_instance = ProcessInstanceService.create_process_instance_from_process_model_identifier( - process_model_identifier, g.user - ) - return process_instance - def process_instance_create( modified_process_model_identifier: str, @@ -87,55 +69,18 @@ def process_instance_create( ) -def _process_instance_run( - process_instance: ProcessInstanceModel, -) -> None: - if process_instance.status != "not_started": - raise ApiError( - error_code="process_instance_not_runnable", - message=f"Process Instance ({process_instance.id}) is currently running or has already run.", - status_code=400, - ) - - processor = None - try: - if not ProcessInstanceQueueService.is_enqueued_to_run_in_the_future(process_instance): - processor = ProcessInstanceService.run_process_instance_with_processor(process_instance) - except ( - ApiError, - ProcessInstanceIsNotEnqueuedError, - ProcessInstanceIsAlreadyLockedError, - ) as e: - ErrorHandlingService.handle_error(process_instance, e) - raise e - except Exception as e: - ErrorHandlingService.handle_error(process_instance, e) - # FIXME: this is going to point someone to the wrong task - it's misinformation for errors in sub-processes. - # we need to recurse through all last tasks if the last task is a call activity or subprocess. - if processor is not None: - task = processor.bpmn_process_instance.last_task - raise ApiError.from_task( - error_code="unknown_exception", - message=f"An unknown error occurred. Original error: {e}", - status_code=400, - task=task, - ) from e - raise e - - if not current_app.config["SPIFFWORKFLOW_BACKEND_RUN_BACKGROUND_SCHEDULER_IN_CREATE_APP"]: - MessageService.correlate_all_message_instances() - - def process_instance_run( modified_process_model_identifier: str, process_instance_id: int, + force_run: bool = False, ) -> flask.wrappers.Response: process_instance = _find_process_instance_by_id_or_raise(process_instance_id) - _process_instance_run(process_instance) + _process_instance_run(process_instance, force_run=force_run) process_instance_api = ProcessInstanceService.processor_to_process_instance_api(process_instance) - process_instance_metadata = ProcessInstanceApiSchema().dump(process_instance_api) - return Response(json.dumps(process_instance_metadata), status=200, mimetype="application/json") + process_instance_api_dict = ProcessInstanceApiSchema().dump(process_instance_api) + process_instance_api_dict["process_model_uses_queued_execution"] = queue_enabled_for_process_model(process_instance) + return Response(json.dumps(process_instance_api_dict), status=200, mimetype="application/json") def process_instance_terminate( @@ -189,6 +134,9 @@ def process_instance_resume( try: with ProcessInstanceQueueService.dequeued(process_instance): processor.resume() + # the process instance will be in waiting since we just successfully resumed it. + # tell the celery worker to get busy. + queue_process_instance_if_appropriate(process_instance) except ( ProcessInstanceIsNotEnqueuedError, ProcessInstanceIsAlreadyLockedError, @@ -245,10 +193,7 @@ def process_instance_report_show( if report_hash is None and report_id is None and report_identifier is None: raise ApiError( error_code="report_key_missing", - message=( - "A report key is needed to lookup a report. Either choose a report_hash, report_id, or" - " report_identifier." - ), + message="A report key is needed to lookup a report. Either choose a report_hash, report_id, or report_identifier.", ) response_result: Report | ProcessInstanceReportModel | None = None if report_hash is not None: @@ -275,9 +220,7 @@ def process_instance_report_column_list( ) -> flask.wrappers.Response: table_columns = ProcessInstanceReportService.builtin_column_options() system_report_column_options = ProcessInstanceReportService.system_report_column_options() - columns_for_metadata_strings = ProcessInstanceReportService.process_instance_metadata_as_columns( - process_model_identifier - ) + columns_for_metadata_strings = ProcessInstanceReportService.process_instance_metadata_as_columns(process_model_identifier) return make_response(jsonify(table_columns + system_report_column_options + columns_for_metadata_strings), 200) @@ -307,9 +250,7 @@ def process_instance_show( ) -def process_instance_delete( - process_instance_id: int, modified_process_model_identifier: str -) -> flask.wrappers.Response: +def process_instance_delete(process_instance_id: int, modified_process_model_identifier: str) -> flask.wrappers.Response: process_instance = _find_process_instance_by_id_or_raise(process_instance_id) if not process_instance.has_terminal_status(): @@ -433,8 +374,7 @@ def process_instance_task_list( raise ApiError( error_code="bpmn_process_not_found", message=( - f"Cannot find a bpmn process with guid '{bpmn_process_guid}' for process instance" - f" '{process_instance.id}'" + f"Cannot find a bpmn process with guid '{bpmn_process_guid}' for process instance '{process_instance.id}'" ), status_code=400, ) @@ -473,9 +413,7 @@ def process_instance_task_list( task_models_of_parent_bpmn_processes, ) = TaskService.task_models_of_parent_bpmn_processes(to_task_model) task_models_of_parent_bpmn_processes_guids = [p.guid for p in task_models_of_parent_bpmn_processes if p.guid] - if to_task_model.runtime_info and ( - "instance" in to_task_model.runtime_info or "iteration" in to_task_model.runtime_info - ): + if to_task_model.runtime_info and ("instance" in to_task_model.runtime_info or "iteration" in to_task_model.runtime_info): to_task_model_parent = [to_task_model.properties_json["parent"]] else: to_task_model_parent = [] @@ -500,8 +438,7 @@ def process_instance_task_list( ) .outerjoin( direct_parent_bpmn_process_definition_alias, - direct_parent_bpmn_process_definition_alias.id - == direct_parent_bpmn_process_alias.bpmn_process_definition_id, + direct_parent_bpmn_process_definition_alias.id == direct_parent_bpmn_process_alias.bpmn_process_definition_id, ) .join( BpmnProcessDefinitionModel, @@ -554,9 +491,7 @@ def process_instance_task_list( most_recent_tasks[row_key] = task_model if task_model.typename in ["SubWorkflowTask", "CallActivity"]: relevant_subprocess_guids.add(task_model.guid) - elif task_model.runtime_info and ( - "instance" in task_model.runtime_info or "iteration" in task_model.runtime_info - ): + elif task_model.runtime_info and ("instance" in task_model.runtime_info or "iteration" in task_model.runtime_info): # This handles adding all instances of a MI and iterations of loop tasks additional_tasks.append(task_model) @@ -573,9 +508,7 @@ def process_instance_task_list( if to_task_model.guid == task_model["guid"] and task_model["state"] == "COMPLETED": TaskService.reset_task_model_dict(task_model, state="READY") elif ( - end_in_seconds is None - or to_task_model.end_in_seconds is None - or to_task_model.end_in_seconds < end_in_seconds + end_in_seconds is None or to_task_model.end_in_seconds is None or to_task_model.end_in_seconds < end_in_seconds ) and task_model["guid"] in task_models_of_parent_bpmn_processes_guids: TaskService.reset_task_model_dict(task_model, state="WAITING") return make_response(jsonify(task_models_dict), 200) @@ -672,9 +605,7 @@ def _get_process_instance( process_model_with_diagram = None name_of_file_with_diagram = None if process_identifier: - spec_reference = ( - ReferenceCacheModel.basic_query().filter_by(identifier=process_identifier, type="process").first() - ) + spec_reference = ReferenceCacheModel.basic_query().filter_by(identifier=process_identifier, type="process").first() if spec_reference is None: raise ReferenceNotFoundError(f"Could not find given process identifier in the cache: {process_identifier}") @@ -702,39 +633,64 @@ def _get_process_instance( return make_response(jsonify(process_instance_as_dict), 200) -def _find_process_instance_for_me_or_raise( - process_instance_id: int, -) -> ProcessInstanceModel: - process_instance: ProcessInstanceModel | None = ( - ProcessInstanceModel.query.filter_by(id=process_instance_id) - .outerjoin(HumanTaskModel) - .outerjoin( - HumanTaskUserModel, - and_( - HumanTaskModel.id == HumanTaskUserModel.human_task_id, - HumanTaskUserModel.user_id == g.user.id, - ), +def _process_instance_run( + process_instance: ProcessInstanceModel, + force_run: bool = False, +) -> None: + if process_instance.status != "not_started" and not force_run: + raise ApiError( + error_code="process_instance_not_runnable", + message=f"Process Instance ({process_instance.id}) is currently running or has already run.", + status_code=400, ) - .filter( - or_( - # you were allowed to complete it - HumanTaskUserModel.id.is_not(None), - # or you completed it (which admins can do even if it wasn't assigned via HumanTaskUserModel) - HumanTaskModel.completed_by_user_id == g.user.id, - # or you started it - ProcessInstanceModel.process_initiator_id == g.user.id, - ) - ) - .first() - ) - if process_instance is None: - raise ( - ApiError( - error_code="process_instance_cannot_be_found", - message=f"Process instance with id {process_instance_id} cannot be found that is associated with you.", + processor = None + task_runnability = None + try: + if queue_enabled_for_process_model(process_instance): + queue_process_instance_if_appropriate(process_instance) + elif not ProcessInstanceQueueService.is_enqueued_to_run_in_the_future(process_instance): + processor, task_runnability = ProcessInstanceService.run_process_instance_with_processor(process_instance) + except ( + ApiError, + ProcessInstanceIsNotEnqueuedError, + ProcessInstanceIsAlreadyLockedError, + ) as e: + ErrorHandlingService.handle_error(process_instance, e) + raise e + except Exception as e: + ErrorHandlingService.handle_error(process_instance, e) + # FIXME: this is going to point someone to the wrong task - it's misinformation for errors in sub-processes. + # we need to recurse through all last tasks if the last task is a call activity or subprocess. + if processor is not None: + task = processor.bpmn_process_instance.last_task + raise ApiError.from_task( + error_code="unknown_exception", + message=f"An unknown error occurred. Original error: {e}", status_code=400, - ) + task=task, + ) from e + raise e + + if not current_app.config["SPIFFWORKFLOW_BACKEND_RUN_BACKGROUND_SCHEDULER_IN_CREATE_APP"]: + MessageService.correlate_all_message_instances() + + +def _process_instance_create( + process_model_identifier: str, +) -> ProcessInstanceModel: + process_model = _get_process_model(process_model_identifier) + if process_model.primary_file_name is None: + raise ApiError( + error_code="process_model_missing_primary_bpmn_file", + message=( + f"Process Model '{process_model_identifier}' does not have a primary" + " bpmn file. One must be set in order to instantiate this model." + ), + status_code=400, ) + process_instance = ProcessInstanceService.create_process_instance_from_process_model_identifier( + process_model_identifier, g.user + ) return process_instance diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/routes/process_models_controller.py b/spiffworkflow-backend/src/spiffworkflow_backend/routes/process_models_controller.py index 9fc7fe55..cef714c9 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/routes/process_models_controller.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/routes/process_models_controller.py @@ -161,9 +161,7 @@ def process_model_show(modified_process_model_identifier: str, include_file_refe # if the user got here then they can read the process model available_actions = {"read": {"path": f"/process-models/{modified_process_model_identifier}", "method": "GET"}} if GitService.check_for_publish_configs(raise_on_missing=False): - available_actions = { - "publish": {"path": f"/process-model-publish/{modified_process_model_identifier}", "method": "POST"} - } + available_actions = {"publish": {"path": f"/process-model-publish/{modified_process_model_identifier}", "method": "POST"}} process_model.actions = available_actions return make_response(jsonify(process_model), 200) @@ -172,21 +170,16 @@ def process_model_show(modified_process_model_identifier: str, include_file_refe def process_model_move(modified_process_model_identifier: str, new_location: str) -> flask.wrappers.Response: original_process_model_id = _un_modify_modified_process_model_id(modified_process_model_identifier) new_process_model = ProcessModelService.process_model_move(original_process_model_id, new_location) - _commit_and_push_to_git( - f"User: {g.user.username} moved process model {original_process_model_id} to {new_process_model.id}" - ) + _commit_and_push_to_git(f"User: {g.user.username} moved process model {original_process_model_id} to {new_process_model.id}") return make_response(jsonify(new_process_model), 200) -def process_model_publish( - modified_process_model_identifier: str, branch_to_update: str | None = None -) -> flask.wrappers.Response: +def process_model_publish(modified_process_model_identifier: str, branch_to_update: str | None = None) -> flask.wrappers.Response: if branch_to_update is None: branch_to_update = current_app.config["SPIFFWORKFLOW_BACKEND_GIT_PUBLISH_TARGET_BRANCH"] if branch_to_update is None: raise MissingGitConfigsError( - "Missing config for SPIFFWORKFLOW_BACKEND_GIT_PUBLISH_TARGET_BRANCH. " - "This is required for publishing process models" + "Missing config for SPIFFWORKFLOW_BACKEND_GIT_PUBLISH_TARGET_BRANCH. This is required for publishing process models" ) process_model_identifier = _un_modify_modified_process_model_id(modified_process_model_identifier) pr_url = GitService().publish(process_model_identifier, branch_to_update) @@ -267,9 +260,7 @@ def process_model_file_delete(modified_process_model_identifier: str, file_name: ) ) from exception - _commit_and_push_to_git( - f"User: {g.user.username} deleted process model file {process_model_identifier}/{file_name}" - ) + _commit_and_push_to_git(f"User: {g.user.username} deleted process model file {process_model_identifier}/{file_name}") return Response(json.dumps({"ok": True}), status=200, mimetype="application/json") @@ -331,9 +322,7 @@ def process_model_test_run( # "natural_language_text": "Create a bug tracker process model \ # with a bug-details form that collects summary, description, and priority" # } -def process_model_create_with_natural_language( - modified_process_group_id: str, body: dict[str, str] -) -> flask.wrappers.Response: +def process_model_create_with_natural_language(modified_process_group_id: str, body: dict[str, str]) -> flask.wrappers.Response: pattern = re.compile( r"Create a (?P.*?) process model with a (?P.*?) form that" r" collects (?P.*)" ) @@ -391,9 +380,7 @@ def process_model_create_with_natural_language( with open(bpmn_template_file, encoding="utf-8") as f: bpmn_template_contents = f.read() - bpmn_template_contents = bpmn_template_contents.replace( - "natural_language_process_id_template", bpmn_process_identifier - ) + bpmn_template_contents = bpmn_template_contents.replace("natural_language_process_id_template", bpmn_process_identifier) bpmn_template_contents = bpmn_template_contents.replace("form-identifier-id-template", form_identifier) form_uischema_json: dict = {"ui:order": columns} @@ -427,9 +414,7 @@ def process_model_create_with_natural_language( str.encode(json.dumps(form_uischema_json)), ) - _commit_and_push_to_git( - f"User: {g.user.username} created process model via natural language: {process_model_info.id}" - ) + _commit_and_push_to_git(f"User: {g.user.username} created process model via natural language: {process_model_info.id}") default_report_metadata = ProcessInstanceReportService.system_metadata_map("default") if default_report_metadata is None: diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/routes/script_unit_tests_controller.py b/spiffworkflow-backend/src/spiffworkflow_backend/routes/script_unit_tests_controller.py index 292de688..01095ceb 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/routes/script_unit_tests_controller.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/routes/script_unit_tests_controller.py @@ -19,9 +19,7 @@ from spiffworkflow_backend.services.script_unit_test_runner import ScriptUnitTes from spiffworkflow_backend.services.spec_file_service import SpecFileService -def script_unit_test_create( - modified_process_model_identifier: str, body: dict[str, str | bool | int] -) -> flask.wrappers.Response: +def script_unit_test_create(modified_process_model_identifier: str, body: dict[str, str | bool | int]) -> flask.wrappers.Response: bpmn_task_identifier = _get_required_parameter_or_raise("bpmn_task_identifier", body) input_json = _get_required_parameter_or_raise("input_json", body) expected_output_json = _get_required_parameter_or_raise("expected_output_json", body) @@ -92,9 +90,7 @@ def script_unit_test_create( return Response(json.dumps({"ok": True}), status=202, mimetype="application/json") -def script_unit_test_run( - modified_process_model_identifier: str, body: dict[str, str | bool | int] -) -> flask.wrappers.Response: +def script_unit_test_run(modified_process_model_identifier: str, body: dict[str, str | bool | int]) -> flask.wrappers.Response: # FIXME: We should probably clear this somewhere else but this works current_app.config["THREAD_LOCAL_DATA"].process_instance_id = None @@ -102,7 +98,5 @@ def script_unit_test_run( input_json = _get_required_parameter_or_raise("input_json", body) expected_output_json = _get_required_parameter_or_raise("expected_output_json", body) - result = ScriptUnitTestRunner.run_with_script_and_pre_post_contexts( - python_script, input_json, expected_output_json - ) + result = ScriptUnitTestRunner.run_with_script_and_pre_post_contexts(python_script, input_json, expected_output_json) return make_response(jsonify(result), 200) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/routes/service_tasks_controller.py b/spiffworkflow-backend/src/spiffworkflow_backend/routes/service_tasks_controller.py index e50a9ee3..2323dbdf 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/routes/service_tasks_controller.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/routes/service_tasks_controller.py @@ -68,9 +68,7 @@ def authentication_callback( verify_token(token, force_run=True) remote_app = OAuthService.remote_app(service, token) response = remote_app.authorized_response() - SecretService.update_secret( - f"{service}_{auth_method}", response["access_token"], g.user.id, create_if_not_exists=True - ) + SecretService.update_secret(f"{service}_{auth_method}", response["access_token"], g.user.id, create_if_not_exists=True) else: verify_token(request.args.get("token"), force_run=True) response = request.args["response"] diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/routes/tasks_controller.py b/spiffworkflow-backend/src/spiffworkflow_backend/routes/tasks_controller.py index 479f7364..d59eb888 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/routes/tasks_controller.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/routes/tasks_controller.py @@ -26,6 +26,12 @@ from sqlalchemy import func from sqlalchemy.orm import aliased from sqlalchemy.orm.util import AliasedClass +from spiffworkflow_backend.background_processing.celery_tasks.process_instance_task_producer import ( + queue_enabled_for_process_model, +) +from spiffworkflow_backend.background_processing.celery_tasks.process_instance_task_producer import ( + queue_process_instance_if_appropriate, +) from spiffworkflow_backend.data_migrations.process_instance_migrator import ProcessInstanceMigrator from spiffworkflow_backend.exceptions.api_error import ApiError from spiffworkflow_backend.exceptions.error import HumanTaskAlreadyCompletedError @@ -48,9 +54,11 @@ from spiffworkflow_backend.models.task import Task from spiffworkflow_backend.models.task import TaskModel from spiffworkflow_backend.models.task_draft_data import TaskDraftDataDict from spiffworkflow_backend.models.task_draft_data import TaskDraftDataModel +from spiffworkflow_backend.models.task_instructions_for_end_user import TaskInstructionsForEndUserModel from spiffworkflow_backend.models.user import UserModel from spiffworkflow_backend.routes.process_api_blueprint import _find_principal_or_raise from spiffworkflow_backend.routes.process_api_blueprint import _find_process_instance_by_id_or_raise +from spiffworkflow_backend.routes.process_api_blueprint import _find_process_instance_for_me_or_raise from spiffworkflow_backend.routes.process_api_blueprint import _get_process_model from spiffworkflow_backend.services.authorization_service import AuthorizationService from spiffworkflow_backend.services.error_handling_service import ErrorHandlingService @@ -91,9 +99,7 @@ def task_allows_guest( # this is currently not used by the Frontend -def task_list_my_tasks( - process_instance_id: int | None = None, page: int = 1, per_page: int = 100 -) -> flask.wrappers.Response: +def task_list_my_tasks(process_instance_id: int | None = None, page: int = 1, per_page: int = 100) -> flask.wrappers.Response: principal = _find_principal_or_raise() assigned_user = aliased(UserModel) process_initiator_user = aliased(UserModel) @@ -263,8 +269,7 @@ def task_data_update( if process_instance: if process_instance.status != "suspended": raise ProcessInstanceTaskDataCannotBeUpdatedError( - "The process instance needs to be suspended to update the task-data." - f" It is currently: {process_instance.status}" + f"The process instance needs to be suspended to update the task-data. It is currently: {process_instance.status}" ) task_model = TaskModel.query.filter_by(guid=task_guid).first() @@ -360,9 +365,7 @@ def task_assign( ) task_model = _get_task_model_from_guid_or_raise(task_guid, process_instance_id) - human_tasks = HumanTaskModel.query.filter_by( - process_instance_id=process_instance.id, task_id=task_model.guid - ).all() + human_tasks = HumanTaskModel.query.filter_by(process_instance_id=process_instance.id, task_id=task_model.guid).all() if len(human_tasks) > 1: raise ApiError( @@ -463,15 +466,11 @@ def task_show( ) relative_path = os.path.relpath(bpmn_file_full_path, start=FileSystemService.root_path()) process_model_relative_path = os.path.dirname(relative_path) - process_model_with_form = ProcessModelService.get_process_model_from_relative_path( - process_model_relative_path - ) + process_model_with_form = ProcessModelService.get_process_model_from_relative_path(process_model_relative_path) form_schema_file_name = "" form_ui_schema_file_name = "" - task_model.signal_buttons = TaskService.get_ready_signals_with_button_labels( - process_instance_id, task_model.guid - ) + task_model.signal_buttons = TaskService.get_ready_signals_with_button_labels(process_instance_id, task_model.guid) if "properties" in extensions: properties = extensions["properties"] @@ -493,10 +492,7 @@ def task_show( raise ( ApiError( error_code="missing_form_file", - message=( - f"Cannot find a form file for process_instance_id: {process_instance_id}, task_guid:" - f" {task_guid}" - ), + message=f"Cannot find a form file for process_instance_id: {process_instance_id}, task_guid: {task_guid}", status_code=400, ) ) @@ -541,6 +537,50 @@ def task_submit( return _task_submit_shared(process_instance_id, task_guid, body) +def process_instance_progress( + process_instance_id: int, +) -> flask.wrappers.Response: + response: dict[str, Task | ProcessInstanceModel | list] = {} + process_instance = _find_process_instance_for_me_or_raise(process_instance_id, include_actions=True) + + principal = _find_principal_or_raise() + next_human_task_assigned_to_me = _next_human_task_for_user(process_instance_id, principal.user_id) + if next_human_task_assigned_to_me: + response["task"] = HumanTaskModel.to_task(next_human_task_assigned_to_me) + # this may not catch all times we should redirect to instance show page + elif not process_instance.is_immediately_runnable(): + # any time we assign this process_instance, the frontend progress page will redirect to process instance show + response["process_instance"] = process_instance + + user_instructions = TaskInstructionsForEndUserModel.retrieve_and_clear(process_instance.id) + response["instructions"] = user_instructions + + return make_response(jsonify(response), 200) + + +def task_with_instruction(process_instance_id: int) -> Response: + process_instance = _find_process_instance_by_id_or_raise(process_instance_id) + processor = ProcessInstanceProcessor(process_instance) + spiff_task = processor.next_task() + task = None + if spiff_task is not None: + task = ProcessInstanceService.spiff_task_to_api_task(processor, spiff_task) + try: + instructions = _render_instructions(spiff_task) + except Exception as exception: + raise ApiError( + error_code="engine_steps_error", + message=f"Failed to complete an automated task. Error was: {str(exception)}", + status_code=400, + ) from exception + task.properties = {"instructionsForEndUser": instructions} + return make_response(jsonify({"task": task}), 200) + + +def _render_instructions(spiff_task: SpiffTask) -> str: + return JinjaService.render_instructions_for_end_user(spiff_task) + + def _interstitial_stream( process_instance: ProcessInstanceModel, execute_tasks: bool = True, @@ -551,12 +591,6 @@ def _interstitial_stream( state=TaskState.WAITING | TaskState.STARTED | TaskState.READY | TaskState.ERROR ) - def render_instructions(spiff_task: SpiffTask) -> str: - task_model = TaskModel.query.filter_by(guid=str(spiff_task.id)).first() - if task_model is None: - return "" - return JinjaService.render_instructions_for_end_user(task_model) - # do not attempt to get task instructions if process instance is suspended or was terminated if process_instance.status in ["suspended", "terminated"]: yield _render_data("unrunnable_instance", process_instance) @@ -571,7 +605,7 @@ def _interstitial_stream( # ignore the instructions if they are on the EndEvent for the top level process if not TaskService.is_main_process_end_event(spiff_task): try: - instructions = render_instructions(spiff_task) + instructions = _render_instructions(spiff_task) except Exception as e: api_error = ApiError( error_code="engine_steps_error", @@ -644,21 +678,20 @@ def _interstitial_stream( tasks = get_reportable_tasks(processor) spiff_task = processor.next_task() - if spiff_task is not None: + if spiff_task is not None and spiff_task.id not in reported_ids: task = ProcessInstanceService.spiff_task_to_api_task(processor, spiff_task) - if task.id not in reported_ids: - try: - instructions = render_instructions(spiff_task) - except Exception as e: - api_error = ApiError( - error_code="engine_steps_error", - message=f"Failed to complete an automated task. Error was: {str(e)}", - status_code=400, - ) - yield _render_data("error", api_error) - raise e - task.properties = {"instructionsForEndUser": instructions} - yield _render_data("task", task) + try: + instructions = _render_instructions(spiff_task) + except Exception as e: + api_error = ApiError( + error_code="engine_steps_error", + message=f"Failed to complete an automated task. Error was: {str(e)}", + status_code=400, + ) + yield _render_data("error", api_error) + raise e + task.properties = {"instructionsForEndUser": instructions} + yield _render_data("task", task) def _get_ready_engine_step_count(bpmn_process_instance: BpmnWorkflow) -> int: @@ -876,30 +909,28 @@ def _task_submit_shared( db.session.delete(task_draft_data) db.session.commit() - next_human_task_assigned_to_me = ( - HumanTaskModel.query.filter_by(process_instance_id=process_instance_id, completed=False) - .order_by(asc(HumanTaskModel.id)) # type: ignore - .join(HumanTaskUserModel) - .filter_by(user_id=principal.user_id) - .first() - ) + next_human_task_assigned_to_me = _next_human_task_for_user(process_instance_id, principal.user_id) if next_human_task_assigned_to_me: return make_response(jsonify(HumanTaskModel.to_task(next_human_task_assigned_to_me)), 200) + queue_process_instance_if_appropriate(process_instance) + + # a guest user completed a task, it has a guest_confirmation message to display to them, + # and there is nothing else for them to do spiff_task_extensions = spiff_task.task_spec.extensions if ( "allowGuest" in spiff_task_extensions and spiff_task_extensions["allowGuest"] == "true" and "guestConfirmation" in spiff_task.task_spec.extensions ): - return make_response( - jsonify({"guest_confirmation": spiff_task.task_spec.extensions["guestConfirmation"]}), 200 - ) + return make_response(jsonify({"guest_confirmation": spiff_task.task_spec.extensions["guestConfirmation"]}), 200) if processor.next_task(): task = ProcessInstanceService.spiff_task_to_api_task(processor, processor.next_task()) + task.process_model_uses_queued_execution = queue_enabled_for_process_model(process_instance) return make_response(jsonify(task), 200) + # next_task always returns something, even if the instance is complete, so we never get here return Response( json.dumps( { @@ -961,9 +992,7 @@ def _get_tasks( if user_group_identifier: human_tasks_query = human_tasks_query.filter(GroupModel.identifier == user_group_identifier) else: - human_tasks_query = human_tasks_query.filter( - HumanTaskModel.lane_assignment_id.is_not(None) # type: ignore - ) + human_tasks_query = human_tasks_query.filter(HumanTaskModel.lane_assignment_id.is_not(None)) # type: ignore else: human_tasks_query = human_tasks_query.filter(HumanTaskModel.lane_assignment_id.is_(None)) # type: ignore @@ -1147,15 +1176,15 @@ def _update_form_schema_with_task_data_as_needed(in_dict: dict, task_data: dict) def _get_potential_owner_usernames(assigned_user: AliasedClass) -> Any: - potential_owner_usernames_from_group_concat_or_similar = func.group_concat( - assigned_user.username.distinct() - ).label("potential_owner_usernames") + potential_owner_usernames_from_group_concat_or_similar = func.group_concat(assigned_user.username.distinct()).label( + "potential_owner_usernames" + ) db_type = current_app.config.get("SPIFFWORKFLOW_BACKEND_DATABASE_TYPE") if db_type == "postgres": - potential_owner_usernames_from_group_concat_or_similar = func.string_agg( - assigned_user.username.distinct(), ", " - ).label("potential_owner_usernames") + potential_owner_usernames_from_group_concat_or_similar = func.string_agg(assigned_user.username.distinct(), ", ").label( + "potential_owner_usernames" + ) return potential_owner_usernames_from_group_concat_or_similar @@ -1179,10 +1208,7 @@ def _find_human_task_or_raise( raise ( ApiError( error_code="no_human_task", - message=( - f"Cannot find a task to complete for task id '{task_guid}' and" - f" process instance {process_instance_id}." - ), + message=f"Cannot find a task to complete for task id '{task_guid}' and process instance {process_instance_id}.", status_code=500, ) ) @@ -1206,9 +1232,7 @@ def _munge_form_ui_schema_based_on_hidden_fields_in_task_data(form_ui_schema: di def _get_task_model_from_guid_or_raise(task_guid: str, process_instance_id: int) -> TaskModel: - task_model: TaskModel | None = TaskModel.query.filter_by( - guid=task_guid, process_instance_id=process_instance_id - ).first() + task_model: TaskModel | None = TaskModel.query.filter_by(guid=task_guid, process_instance_id=process_instance_id).first() if task_model is None: raise ApiError( error_code="task_not_found", @@ -1216,3 +1240,14 @@ def _get_task_model_from_guid_or_raise(task_guid: str, process_instance_id: int) status_code=400, ) return task_model + + +def _next_human_task_for_user(process_instance_id: int, user_id: int) -> HumanTaskModel | None: + next_human_task: HumanTaskModel | None = ( + HumanTaskModel.query.filter_by(process_instance_id=process_instance_id, completed=False) + .order_by(asc(HumanTaskModel.id)) # type: ignore + .join(HumanTaskUserModel) + .filter_by(user_id=user_id) + .first() + ) + return next_human_task diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/scripts/get_group_members.py b/spiffworkflow-backend/src/spiffworkflow_backend/scripts/get_group_members.py index 1ec5f2c6..b46b86a6 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/scripts/get_group_members.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/scripts/get_group_members.py @@ -24,9 +24,7 @@ class GetGroupMembers(Script): group_identifier = args[0] group = GroupModel.query.filter_by(identifier=group_identifier).first() if group is None: - raise GroupNotFoundError( - f"Script 'get_group_members' could not find group with identifier '{group_identifier}'." - ) + raise GroupNotFoundError(f"Script 'get_group_members' could not find group with identifier '{group_identifier}'.") usernames = [u.username for u in group.users] return usernames diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/scripts/get_url_for_task_with_bpmn_identifier.py b/spiffworkflow-backend/src/spiffworkflow_backend/scripts/get_url_for_task_with_bpmn_identifier.py index a06d0f44..9e61db20 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/scripts/get_url_for_task_with_bpmn_identifier.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/scripts/get_url_for_task_with_bpmn_identifier.py @@ -30,8 +30,7 @@ class GetUrlForTaskWithBpmnIdentifier(Script): desired_spiff_task = ProcessInstanceProcessor.get_task_by_bpmn_identifier(bpmn_identifier, spiff_task.workflow) if desired_spiff_task is None: raise Exception( - f"Could not find a task with bpmn identifier '{bpmn_identifier}' in" - " get_url_for_task_with_bpmn_identifier" + f"Could not find a task with bpmn identifier '{bpmn_identifier}' in get_url_for_task_with_bpmn_identifier" ) if not desired_spiff_task.task_spec.manual: diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/scripts/markdown_file_download_link.py b/spiffworkflow-backend/src/spiffworkflow_backend/scripts/markdown_file_download_link.py index be28e1e0..81d46205 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/scripts/markdown_file_download_link.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/scripts/markdown_file_download_link.py @@ -30,17 +30,12 @@ class GetMarkdownFileDownloadLink(Script): process_model_identifier = script_attributes_context.process_model_identifier if process_model_identifier is None: raise self.get_proces_model_identifier_is_missing_error("markdown_file_download_link") - modified_process_model_identifier = ProcessModelInfo.modify_process_identifier_for_path_param( - process_model_identifier - ) + modified_process_model_identifier = ProcessModelInfo.modify_process_identifier_for_path_param(process_model_identifier) process_instance_id = script_attributes_context.process_instance_id if process_instance_id is None: raise self.get_proces_instance_id_is_missing_error("save_process_instance_metadata") url = current_app.config["SPIFFWORKFLOW_BACKEND_URL"] - url += ( - f"/v1.0/process-data-file-download/{modified_process_model_identifier}/" - + f"{process_instance_id}/{digest}" - ) + url += f"/v1.0/process-data-file-download/{modified_process_model_identifier}/" + f"{process_instance_id}/{digest}" link = f"[{label}]({url})" return link diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/scripts/script.py b/spiffworkflow-backend/src/spiffworkflow_backend/scripts/script.py index dac349e3..27b92220 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/scripts/script.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/scripts/script.py @@ -123,13 +123,10 @@ class Script: f" running script '{script_function_name}'" ) user = process_instance.process_initiator - has_permission = AuthorizationService.user_has_permission( - user=user, permission="create", target_uri=uri - ) + has_permission = AuthorizationService.user_has_permission(user=user, permission="create", target_uri=uri) if not has_permission: raise ScriptUnauthorizedForUserError( - f"User {user.username} does not have access to run" - f" privileged script '{script_function_name}'" + f"User {user.username} does not have access to run privileged script '{script_function_name}'" ) def run_script_if_allowed(*ar: Any, **kw: Any) -> Any: diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/scripts/set_user_properties.py b/spiffworkflow-backend/src/spiffworkflow_backend/scripts/set_user_properties.py index b6ed4bba..f46575e8 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/scripts/set_user_properties.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/scripts/set_user_properties.py @@ -20,9 +20,7 @@ class SetUserProperties(Script): def run(self, script_attributes_context: ScriptAttributesContext, *args: Any, **kwargs: Any) -> Any: properties = args[0] if not isinstance(properties, dict): - raise InvalidArgsGivenToScriptError( - f"Args to set_user_properties must be a dict. '{properties}' is invalid." - ) + raise InvalidArgsGivenToScriptError(f"Args to set_user_properties must be a dict. '{properties}' is invalid.") # consider using engine-specific insert or update metaphor in future: https://stackoverflow.com/a/68431412/6090676 for property_key, property_value in properties.items(): user_property = UserPropertyModel.query.filter_by(user_id=g.user.id, key=property_key).first() diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/services/authentication_service.py b/spiffworkflow-backend/src/spiffworkflow_backend/services/authentication_service.py index 796dbe51..2f2423e0 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/services/authentication_service.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/services/authentication_service.py @@ -48,9 +48,7 @@ class AuthenticationOptionNotFoundError(Exception): class AuthenticationService: - ENDPOINT_CACHE: dict[str, dict[str, str]] = ( - {} - ) # We only need to find the openid endpoints once, then we can cache them. + ENDPOINT_CACHE: dict[str, dict[str, str]] = {} # We only need to find the openid endpoints once, then we can cache them. @classmethod def authentication_options_for_api(cls) -> list[AuthenticationOptionForApi]: @@ -72,9 +70,7 @@ class AuthenticationService: if config["identifier"] == authentication_identifier: return_config: AuthenticationOption = config return return_config - raise AuthenticationOptionNotFoundError( - f"Could not find a config with identifier '{authentication_identifier}'" - ) + raise AuthenticationOptionNotFoundError(f"Could not find a config with identifier '{authentication_identifier}'") @classmethod def client_id(cls, authentication_identifier: str) -> str: @@ -119,9 +115,7 @@ class AuthenticationService: if redirect_url is None: redirect_url = f"{self.get_backend_url()}/v1.0/logout_return" request_url = ( - self.__class__.open_id_endpoint_for_name( - "end_session_endpoint", authentication_identifier=authentication_identifier - ) + self.__class__.open_id_endpoint_for_name("end_session_endpoint", authentication_identifier=authentication_identifier) + f"?post_logout_redirect_uri={redirect_url}&" + f"id_token_hint={id_token}" ) @@ -135,14 +129,10 @@ class AuthenticationService: ) return state - def get_login_redirect_url( - self, state: str, authentication_identifier: str, redirect_url: str = "/v1.0/login_return" - ) -> str: + def get_login_redirect_url(self, state: str, authentication_identifier: str, redirect_url: str = "/v1.0/login_return") -> str: return_redirect_url = f"{self.get_backend_url()}{redirect_url}" login_redirect_url = ( - self.__class__.open_id_endpoint_for_name( - "authorization_endpoint", authentication_identifier=authentication_identifier - ) + self.open_id_endpoint_for_name("authorization_endpoint", authentication_identifier=authentication_identifier) + f"?state={state}&" + "response_type=code&" + f"client_id={self.client_id(authentication_identifier)}&" @@ -151,9 +141,7 @@ class AuthenticationService: ) return login_redirect_url - def get_auth_token_object( - self, code: str, authentication_identifier: str, redirect_url: str = "/v1.0/login_return" - ) -> dict: + def get_auth_token_object(self, code: str, authentication_identifier: str, redirect_url: str = "/v1.0/login_return") -> dict: backend_basic_auth_string = ( f"{self.client_id(authentication_identifier)}:{self.__class__.secret_key(authentication_identifier)}" ) @@ -169,9 +157,7 @@ class AuthenticationService: "redirect_uri": f"{self.get_backend_url()}{redirect_url}", } - request_url = self.open_id_endpoint_for_name( - "token_endpoint", authentication_identifier=authentication_identifier - ) + request_url = self.open_id_endpoint_for_name("token_endpoint", authentication_identifier=authentication_identifier) response = requests.post(request_url, data=data, headers=headers, timeout=HTTP_REQUEST_TIMEOUT_SECONDS) auth_token_object: dict = json.loads(response.text) @@ -203,15 +189,13 @@ class AuthenticationService: if iss != cls.server_url(authentication_identifier): current_app.logger.error( - f"TOKEN INVALID because ISS '{iss}' does not match server url" - f" '{cls.server_url(authentication_identifier)}'" + f"TOKEN INVALID because ISS '{iss}' does not match server url '{cls.server_url(authentication_identifier)}'" ) valid = False # aud could be an array or a string elif len(overlapping_aud_values) < 1: current_app.logger.error( - f"TOKEN INVALID because audience '{aud}' does not match client id" - f" '{cls.client_id(authentication_identifier)}'" + f"TOKEN INVALID because audience '{aud}' does not match client id '{cls.client_id(authentication_identifier)}'" ) valid = False elif azp and azp not in ( @@ -219,15 +203,12 @@ class AuthenticationService: "account", ): current_app.logger.error( - f"TOKEN INVALID because azp '{azp}' does not match client id" - f" '{cls.client_id(authentication_identifier)}'" + f"TOKEN INVALID because azp '{azp}' does not match client id '{cls.client_id(authentication_identifier)}'" ) valid = False # make sure issued at time is not in the future elif now + iat_clock_skew_leeway < iat: - current_app.logger.error( - f"TOKEN INVALID because iat '{iat}' is in the future relative to server now '{now}'" - ) + current_app.logger.error(f"TOKEN INVALID because iat '{iat}' is in the future relative to server now '{now}'") valid = False if valid and now > decoded_token["exp"]: @@ -264,9 +245,7 @@ class AuthenticationService: @staticmethod def get_refresh_token(user_id: int) -> str | None: - refresh_token_object: RefreshTokenModel = RefreshTokenModel.query.filter( - RefreshTokenModel.user_id == user_id - ).first() + refresh_token_object: RefreshTokenModel = RefreshTokenModel.query.filter(RefreshTokenModel.user_id == user_id).first() if refresh_token_object: return refresh_token_object.token return None @@ -274,9 +253,7 @@ class AuthenticationService: @classmethod def get_auth_token_from_refresh_token(cls, refresh_token: str, authentication_identifier: str) -> dict: """Converts a refresh token to an Auth Token by calling the openid's auth endpoint.""" - backend_basic_auth_string = ( - f"{cls.client_id(authentication_identifier)}:{cls.secret_key(authentication_identifier)}" - ) + backend_basic_auth_string = f"{cls.client_id(authentication_identifier)}:{cls.secret_key(authentication_identifier)}" backend_basic_auth_bytes = bytes(backend_basic_auth_string, encoding="ascii") backend_basic_auth = base64.b64encode(backend_basic_auth_bytes) headers = { @@ -291,9 +268,7 @@ class AuthenticationService: "client_secret": cls.secret_key(authentication_identifier), } - request_url = cls.open_id_endpoint_for_name( - "token_endpoint", authentication_identifier=authentication_identifier - ) + request_url = cls.open_id_endpoint_for_name("token_endpoint", authentication_identifier=authentication_identifier) response = requests.post(request_url, data=data, headers=headers, timeout=HTTP_REQUEST_TIMEOUT_SECONDS) auth_token_object: dict = json.loads(response.text) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/services/authorization_service.py b/spiffworkflow-backend/src/spiffworkflow_backend/services/authorization_service.py index 089997d3..4fe8f2f7 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/services/authorization_service.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/services/authorization_service.py @@ -188,9 +188,7 @@ class AuthorizationService: def find_or_create_permission_target(cls, uri: str) -> PermissionTargetModel: uri_with_percent = re.sub(r"\*", "%", uri) target_uri_normalized = uri_with_percent.removeprefix(V1_API_PATH_PREFIX) - permission_target: PermissionTargetModel | None = PermissionTargetModel.query.filter_by( - uri=target_uri_normalized - ).first() + permission_target: PermissionTargetModel | None = PermissionTargetModel.query.filter_by(uri=target_uri_normalized).first() if permission_target is None: permission_target = PermissionTargetModel(uri=target_uri_normalized) db.session.add(permission_target) @@ -305,10 +303,7 @@ class AuthorizationService: return None raise NotAuthorizedError( - ( - f"User {g.user.username} is not authorized to perform requested action:" - f" {permission_string} - {request.path}" - ), + f"User {g.user.username} is not authorized to perform requested action: {permission_string} - {request.path}", ) @classmethod @@ -349,8 +344,7 @@ class AuthorizationService: if human_task.completed: raise HumanTaskAlreadyCompletedError( - f"Human task with task guid '{task_guid}' for process instance '{process_instance_id}' has already" - " been completed" + f"Human task with task guid '{task_guid}' for process instance '{process_instance_id}' has already been completed" ) if user not in human_task.potential_owners: @@ -426,16 +420,13 @@ class AuthorizationService: if desired_group_identifiers is not None: if not isinstance(desired_group_identifiers, list): current_app.logger.error( - f"Invalid groups property in token: {desired_group_identifiers}." - "If groups is specified, it must be a list" + f"Invalid groups property in token: {desired_group_identifiers}.If groups is specified, it must be a list" ) else: for desired_group_identifier in desired_group_identifiers: UserService.add_user_to_group_by_group_identifier(user_model, desired_group_identifier) current_group_identifiers = [g.identifier for g in user_model.groups] - groups_to_remove_from_user = [ - item for item in current_group_identifiers if item not in desired_group_identifiers - ] + groups_to_remove_from_user = [item for item in current_group_identifiers if item not in desired_group_identifiers] for gtrfu in groups_to_remove_from_user: if gtrfu != current_app.config["SPIFFWORKFLOW_BACKEND_DEFAULT_USER_GROUP"]: UserService.remove_user_from_group(user_model, gtrfu) @@ -524,17 +515,11 @@ class AuthorizationService: permissions_to_assign.append(PermissionToAssign(permission="read", target_uri="/users/search")) permissions_to_assign.append(PermissionToAssign(permission="read", target_uri="/onboarding")) - permissions_to_assign.append( - PermissionToAssign(permission="read", target_uri="/process-instances/report-metadata") - ) - permissions_to_assign.append( - PermissionToAssign(permission="read", target_uri="/process-instances/find-by-id/*") - ) + permissions_to_assign.append(PermissionToAssign(permission="read", target_uri="/process-instances/report-metadata")) + permissions_to_assign.append(PermissionToAssign(permission="read", target_uri="/process-instances/find-by-id/*")) for permission in ["create", "read", "update", "delete"]: - permissions_to_assign.append( - PermissionToAssign(permission=permission, target_uri="/process-instances/reports/*") - ) + permissions_to_assign.append(PermissionToAssign(permission=permission, target_uri="/process-instances/reports/*")) permissions_to_assign.append(PermissionToAssign(permission=permission, target_uri="/tasks/*")) return permissions_to_assign @@ -551,9 +536,7 @@ class AuthorizationService: permissions_to_assign.append(PermissionToAssign(permission="read", target_uri="/authentications")) permissions_to_assign.append(PermissionToAssign(permission="read", target_uri="/authentication/configuration")) permissions_to_assign.append(PermissionToAssign(permission="read", target_uri="/authentication_begin/*")) - permissions_to_assign.append( - PermissionToAssign(permission="update", target_uri="/authentication/configuration") - ) + permissions_to_assign.append(PermissionToAssign(permission="update", target_uri="/authentication/configuration")) permissions_to_assign.append(PermissionToAssign(permission="create", target_uri="/service-accounts")) @@ -573,9 +556,7 @@ class AuthorizationService: permissions_to_assign.append(PermissionToAssign(permission="create", target_uri="/messages/*")) permissions_to_assign.append(PermissionToAssign(permission="read", target_uri="/messages")) - permissions_to_assign.append( - PermissionToAssign(permission="create", target_uri="/can-run-privileged-script/*") - ) + permissions_to_assign.append(PermissionToAssign(permission="create", target_uri="/can-run-privileged-script/*")) permissions_to_assign.append(PermissionToAssign(permission="create", target_uri="/debug/*")) permissions_to_assign.append(PermissionToAssign(permission="create", target_uri="/send-event/*")) permissions_to_assign.append(PermissionToAssign(permission="create", target_uri="/task-complete/*")) @@ -731,8 +712,7 @@ class AuthorizationService: if current_app.config["SPIFFWORKFLOW_BACKEND_PERMISSIONS_FILE_ABSOLUTE_PATH"] is None: raise ( PermissionsFileNotSetError( - "SPIFFWORKFLOW_BACKEND_PERMISSIONS_FILE_ABSOLUTE_PATH needs to be set in order to import" - " permissions" + "SPIFFWORKFLOW_BACKEND_PERMISSIONS_FILE_ABSOLUTE_PATH needs to be set in order to import permissions" ) ) @@ -761,9 +741,7 @@ class AuthorizationService: uri = permission_config["uri"] actions = cls.get_permissions_from_config(permission_config) for group_identifier in permission_config["groups"]: - group_permissions_by_group[group_identifier]["permissions"].append( - {"actions": actions, "uri": uri} - ) + group_permissions_by_group[group_identifier]["permissions"].append({"actions": actions, "uri": uri}) return list(group_permissions_by_group.values()) @@ -881,9 +859,7 @@ class AuthorizationService: db.session.commit() @classmethod - def refresh_permissions( - cls, group_permissions: list[GroupPermissionsDict], group_permissions_only: bool = False - ) -> None: + def refresh_permissions(cls, group_permissions: list[GroupPermissionsDict], group_permissions_only: bool = False) -> None: """Adds new permission assignments and deletes old ones.""" initial_permission_assignments = ( PermissionAssignmentModel.query.outerjoin( diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/services/element_units_service.py b/spiffworkflow-backend/src/spiffworkflow_backend/services/element_units_service.py index 65296b89..d51df7b3 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/services/element_units_service.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/services/element_units_service.py @@ -43,9 +43,7 @@ class ElementUnitsService: return None @classmethod - def workflow_from_cached_element_unit( - cls, cache_key: str, process_id: str, element_id: str - ) -> BpmnSpecDict | None: + def workflow_from_cached_element_unit(cls, cache_key: str, process_id: str, element_id: str) -> BpmnSpecDict | None: if not cls._enabled(): return None @@ -62,9 +60,7 @@ class ElementUnitsService: current_app.logger.debug(f"Checking element unit cache @ {cache_key} :: '{process_id}' - '{element_id}'") - bpmn_spec_json = spiff_element_units.workflow_from_cached_element_unit( - cache_dir, cache_key, process_id, element_id - ) + bpmn_spec_json = spiff_element_units.workflow_from_cached_element_unit(cache_dir, cache_key, process_id, element_id) return json.loads(bpmn_spec_json) # type: ignore except Exception as e: current_app.logger.exception(e) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/services/file_system_service.py b/spiffworkflow-backend/src/spiffworkflow_backend/services/file_system_service.py index 2eb0c044..cbdeb9c5 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/services/file_system_service.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/services/file_system_service.py @@ -34,9 +34,7 @@ class FileSystemService: PROCESS_MODEL_JSON_FILE = "process_model.json" @classmethod - def walk_files( - cls, start_dir: str, directory_predicate: DirectoryPredicate, file_predicate: FilePredicate - ) -> FileGenerator: + def walk_files(cls, start_dir: str, directory_predicate: DirectoryPredicate, file_predicate: FilePredicate) -> FileGenerator: depth = 0 for root, subdirs, files in os.walk(start_dir): if directory_predicate: @@ -120,9 +118,7 @@ class FileSystemService: def get_data(process_model_info: ProcessModelInfo, file_name: str) -> bytes: full_file_path = FileSystemService.full_file_path(process_model_info, file_name) if not os.path.exists(full_file_path): - raise ProcessModelFileNotFoundError( - f"No file found with name {file_name} in {process_model_info.display_name}" - ) + raise ProcessModelFileNotFoundError(f"No file found with name {file_name} in {process_model_info.display_name}") with open(full_file_path, "rb") as f_handle: spec_file_data = f_handle.read() return spec_file_data diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/services/git_service.py b/spiffworkflow-backend/src/spiffworkflow_backend/services/git_service.py index e5143bf8..078e6a81 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/services/git_service.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/services/git_service.py @@ -172,9 +172,7 @@ class GitService: my_env = os.environ.copy() my_env["GIT_COMMITTER_NAME"] = current_app.config.get("SPIFFWORKFLOW_BACKEND_GIT_USERNAME") or "unknown" - my_env["GIT_COMMITTER_EMAIL"] = ( - current_app.config.get("SPIFFWORKFLOW_BACKEND_GIT_USER_EMAIL") or "unknown@example.org" - ) + my_env["GIT_COMMITTER_EMAIL"] = current_app.config.get("SPIFFWORKFLOW_BACKEND_GIT_USER_EMAIL") or "unknown@example.org" # SSH authentication can be also provided via gitconfig. ssh_key_path = current_app.config.get("SPIFFWORKFLOW_BACKEND_GIT_SSH_PRIVATE_KEY_PATH") @@ -206,9 +204,7 @@ class GitService: @classmethod def handle_web_hook(cls, webhook: dict) -> bool: if "repository" not in webhook or "clone_url" not in webhook["repository"]: - raise InvalidGitWebhookBodyError( - f"Cannot find required keys of 'repository:clone_url' from webhook body: {webhook}" - ) + raise InvalidGitWebhookBodyError(f"Cannot find required keys of 'repository:clone_url' from webhook body: {webhook}") repo = webhook["repository"] valid_clone_urls = [repo["clone_url"], repo["git_url"], repo["ssh_url"]] bpmn_spec_absolute_dir = current_app.config["SPIFFWORKFLOW_BACKEND_BPMN_SPEC_ABSOLUTE_DIR"] @@ -217,8 +213,7 @@ class GitService: ) if config_clone_url not in valid_clone_urls: raise GitCloneUrlMismatchError( - f"Configured clone url does not match the repo URLs from webhook: {config_clone_url} =/=" - f" {valid_clone_urls}" + f"Configured clone url does not match the repo URLs from webhook: {config_clone_url} =/= {valid_clone_urls}" ) # Test webhook requests have a zen koan and hook info. @@ -282,9 +277,7 @@ class GitService: if cls.run_shell_command_as_boolean(command, context_directory=destination_process_root): cls.run_shell_command(["checkout", branch_to_pull_request], context_directory=destination_process_root) else: - cls.run_shell_command( - ["checkout", "-b", branch_to_pull_request], context_directory=destination_process_root - ) + cls.run_shell_command(["checkout", "-b", branch_to_pull_request], context_directory=destination_process_root) # copy files from process model into the new publish branch destination_process_model_path = os.path.join(destination_process_root, process_model_id) @@ -294,8 +287,7 @@ class GitService: # add and commit files to branch_to_pull_request, then push commit_message = ( - f"Request to publish changes to {process_model_id}, " - f"from {g.user.username} on {current_app.config['ENV_IDENTIFIER']}" + f"Request to publish changes to {process_model_id}, from {g.user.username} on {current_app.config['ENV_IDENTIFIER']}" ) cls.commit(commit_message, destination_process_root, branch_to_pull_request) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/services/jinja_service.py b/spiffworkflow-backend/src/spiffworkflow_backend/services/jinja_service.py index fc62c538..3ee4ba22 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/services/jinja_service.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/services/jinja_service.py @@ -43,7 +43,7 @@ class JinjaService: if extensions is None: if isinstance(task, TaskModel): extensions = TaskService.get_extensions_from_task_model(task) - else: + elif hasattr(task.task_spec, "extensions"): extensions = task.task_spec.extensions if extensions and "instructionsForEndUser" in extensions: if extensions["instructionsForEndUser"]: diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/services/logging_service.py b/spiffworkflow-backend/src/spiffworkflow_backend/services/logging_service.py index 01780584..dfd05075 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/services/logging_service.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/services/logging_service.py @@ -49,9 +49,7 @@ class JsonFormatter(logging.Formatter): return "asctime" in self.fmt_dict.values() # we are overriding a method that returns a string and returning a dict, hence the Any - def formatMessage( # noqa: N802, this is overriding a method from python's stdlib - self, record: logging.LogRecord - ) -> Any: + def formatMessage(self, record: logging.LogRecord) -> Any: # noqa: N802, this is overriding a method from python's stdlib """Overwritten to return a dictionary of the relevant LogRecord attributes instead of a string. KeyError is raised if an unknown attribute is provided in the fmt_dict. @@ -90,9 +88,7 @@ def setup_logger(app: Flask) -> None: log_levels = ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] if upper_log_level_string not in log_levels: - raise InvalidLogLevelError( - f"Log level given is invalid: '{upper_log_level_string}'. Valid options are {log_levels}" - ) + raise InvalidLogLevelError(f"Log level given is invalid: '{upper_log_level_string}'. Valid options are {log_levels}") log_level = getattr(logging, upper_log_level_string) spiff_log_level = getattr(logging, upper_log_level_string) @@ -119,9 +115,7 @@ def setup_logger(app: Flask) -> None: spiff_logger_filehandler = None if app.config["SPIFFWORKFLOW_BACKEND_LOG_TO_FILE"]: - spiff_logger_filehandler = logging.FileHandler( - f"{app.instance_path}/../../log/{app.config['ENV_IDENTIFIER']}.log" - ) + spiff_logger_filehandler = logging.FileHandler(f"{app.instance_path}/../../log/{app.config['ENV_IDENTIFIER']}.log") spiff_logger_filehandler.setLevel(spiff_log_level) spiff_logger_filehandler.setFormatter(log_formatter) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/services/monitoring_service.py b/spiffworkflow-backend/src/spiffworkflow_backend/services/monitoring_service.py new file mode 100644 index 00000000..eaa0f98a --- /dev/null +++ b/spiffworkflow-backend/src/spiffworkflow_backend/services/monitoring_service.py @@ -0,0 +1,108 @@ +import json +import os +import sys +from typing import Any + +import connexion # type: ignore +import flask.wrappers +import sentry_sdk +from prometheus_flask_exporter import ConnexionPrometheusMetrics # type: ignore +from sentry_sdk.integrations.flask import FlaskIntegration +from werkzeug.exceptions import NotFound + + +def get_version_info_data() -> dict[str, Any]: + version_info_data_dict = {} + if os.path.isfile("version_info.json"): + with open("version_info.json") as f: + version_info_data_dict = json.load(f) + return version_info_data_dict + + +def setup_prometheus_metrics(app: flask.app.Flask, connexion_app: connexion.apps.flask_app.FlaskApp) -> None: + metrics = ConnexionPrometheusMetrics(connexion_app) + app.config["PROMETHEUS_METRICS"] = metrics + version_info_data = get_version_info_data() + if len(version_info_data) > 0: + # prometheus does not allow periods in key names + version_info_data_normalized = {k.replace(".", "_"): v for k, v in version_info_data.items()} + metrics.info("version_info", "Application Version Info", **version_info_data_normalized) + + +def traces_sampler(sampling_context: Any) -> Any: + # always inherit + if sampling_context["parent_sampled"] is not None: + return sampling_context["parent_sampled"] + + if "wsgi_environ" in sampling_context: + wsgi_environ = sampling_context["wsgi_environ"] + path_info = wsgi_environ.get("PATH_INFO") + request_method = wsgi_environ.get("REQUEST_METHOD") + + # tasks_controller.task_submit + # this is the current pain point as of 31 jan 2023. + if path_info and ( + (path_info.startswith("/v1.0/tasks/") and request_method == "PUT") + or (path_info.startswith("/v1.0/task-data/") and request_method == "GET") + ): + return 1 + + # Default sample rate for all others (replaces traces_sample_rate) + return 0.01 + + +def configure_sentry(app: flask.app.Flask) -> None: + # get rid of NotFound errors + def before_send(event: Any, hint: Any) -> Any: + if "exc_info" in hint: + _exc_type, exc_value, _tb = hint["exc_info"] + # NotFound is mostly from web crawlers + if isinstance(exc_value, NotFound): + return None + return event + + sentry_errors_sample_rate = app.config.get("SPIFFWORKFLOW_BACKEND_SENTRY_ERRORS_SAMPLE_RATE") + if sentry_errors_sample_rate is None: + raise Exception("SPIFFWORKFLOW_BACKEND_SENTRY_ERRORS_SAMPLE_RATE is not set somehow") + + sentry_traces_sample_rate = app.config.get("SPIFFWORKFLOW_BACKEND_SENTRY_TRACES_SAMPLE_RATE") + if sentry_traces_sample_rate is None: + raise Exception("SPIFFWORKFLOW_BACKEND_SENTRY_TRACES_SAMPLE_RATE is not set somehow") + + sentry_env_identifier = app.config["ENV_IDENTIFIER"] + if app.config.get("SPIFFWORKFLOW_BACKEND_SENTRY_ENV_IDENTIFIER"): + sentry_env_identifier = app.config.get("SPIFFWORKFLOW_BACKEND_SENTRY_ENV_IDENTIFIER") + + sentry_configs = { + "dsn": app.config.get("SPIFFWORKFLOW_BACKEND_SENTRY_DSN"), + "integrations": [ + FlaskIntegration(), + ], + "environment": sentry_env_identifier, + # sample_rate is the errors sample rate. we usually set it to 1 (100%) + # so we get all errors in sentry. + "sample_rate": float(sentry_errors_sample_rate), + # Set traces_sample_rate to capture a certain percentage + # of transactions for performance monitoring. + # We recommend adjusting this value to less than 1(00%) in production. + "traces_sample_rate": float(sentry_traces_sample_rate), + "traces_sampler": traces_sampler, + # The profiles_sample_rate setting is relative to the traces_sample_rate setting. + "before_send": before_send, + } + + # https://docs.sentry.io/platforms/python/configuration/releases + version_info_data = get_version_info_data() + if len(version_info_data) > 0: + git_commit = version_info_data.get("org.opencontainers.image.revision") or version_info_data.get("git_commit") + if git_commit is not None: + sentry_configs["release"] = git_commit + + if app.config.get("SPIFFWORKFLOW_BACKEND_SENTRY_PROFILING_ENABLED"): + # profiling doesn't work on windows, because of an issue like https://github.com/nvdv/vprof/issues/62 + # but also we commented out profiling because it was causing segfaults (i guess it is marked experimental) + profiles_sample_rate = 0 if sys.platform.startswith("win") else 1 + if profiles_sample_rate > 0: + sentry_configs["_experiments"] = {"profiles_sample_rate": profiles_sample_rate} + + sentry_sdk.init(**sentry_configs) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/services/process_caller_service.py b/spiffworkflow-backend/src/spiffworkflow_backend/services/process_caller_service.py index 69bc8f06..4bd6e114 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/services/process_caller_service.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/services/process_caller_service.py @@ -25,15 +25,11 @@ class ProcessCallerService: @staticmethod def add_caller(process_id: str, called_process_ids: list[str]) -> None: for called_process_id in called_process_ids: - db.session.add( - ProcessCallerCacheModel(process_identifier=called_process_id, calling_process_identifier=process_id) - ) + db.session.add(ProcessCallerCacheModel(process_identifier=called_process_id, calling_process_identifier=process_id)) @staticmethod def callers(process_ids: list[str]) -> list[str]: records = ( - db.session.query(ProcessCallerCacheModel) - .filter(ProcessCallerCacheModel.process_identifier.in_(process_ids)) - .all() + db.session.query(ProcessCallerCacheModel).filter(ProcessCallerCacheModel.process_identifier.in_(process_ids)).all() ) return sorted({r.calling_process_identifier for r in records}) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_lock_service.py b/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_lock_service.py index fec6650c..b9c0de02 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_lock_service.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_lock_service.py @@ -18,8 +18,11 @@ class ProcessInstanceLockService: """TODO: comment.""" @classmethod - def set_thread_local_locking_context(cls, domain: str) -> None: - current_app.config["THREAD_LOCAL_DATA"].lock_service_context = { + def set_thread_local_locking_context(cls, domain: str, additional_processing_identifier: str | None = None) -> None: + tld = current_app.config["THREAD_LOCAL_DATA"] + if not hasattr(tld, "lock_service_context"): + tld.lock_service_context = {} + tld.lock_service_context[additional_processing_identifier] = { "domain": domain, "uuid": current_app.config["PROCESS_UUID"], "thread_id": threading.get_ident(), @@ -27,45 +30,52 @@ class ProcessInstanceLockService: } @classmethod - def get_thread_local_locking_context(cls) -> dict[str, Any]: + def get_thread_local_locking_context(cls, additional_processing_identifier: str | None = None) -> dict[str, Any]: tld = current_app.config["THREAD_LOCAL_DATA"] if not hasattr(tld, "lock_service_context"): - cls.set_thread_local_locking_context("web") - return tld.lock_service_context # type: ignore + cls.set_thread_local_locking_context("web", additional_processing_identifier=additional_processing_identifier) + return tld.lock_service_context[additional_processing_identifier] # type: ignore @classmethod - def locked_by(cls) -> str: - ctx = cls.get_thread_local_locking_context() - return f"{ctx['domain']}:{ctx['uuid']}:{ctx['thread_id']}" + def locked_by(cls, additional_processing_identifier: str | None = None) -> str: + ctx = cls.get_thread_local_locking_context(additional_processing_identifier=additional_processing_identifier) + return f"{ctx['domain']}:{ctx['uuid']}:{ctx['thread_id']}:{additional_processing_identifier}" @classmethod - def lock(cls, process_instance_id: int, queue_entry: ProcessInstanceQueueModel) -> None: - ctx = cls.get_thread_local_locking_context() + def lock( + cls, process_instance_id: int, queue_entry: ProcessInstanceQueueModel, additional_processing_identifier: str | None = None + ) -> None: + ctx = cls.get_thread_local_locking_context(additional_processing_identifier=additional_processing_identifier) ctx["locks"][process_instance_id] = queue_entry @classmethod - def lock_many(cls, queue_entries: list[ProcessInstanceQueueModel]) -> list[int]: - ctx = cls.get_thread_local_locking_context() + def lock_many( + cls, queue_entries: list[ProcessInstanceQueueModel], additional_processing_identifier: str | None = None + ) -> list[int]: + ctx = cls.get_thread_local_locking_context(additional_processing_identifier=additional_processing_identifier) new_locks = {entry.process_instance_id: entry for entry in queue_entries} new_lock_ids = list(new_locks.keys()) ctx["locks"].update(new_locks) return new_lock_ids @classmethod - def unlock(cls, process_instance_id: int) -> ProcessInstanceQueueModel: - queue_model = cls.try_unlock(process_instance_id) + def unlock(cls, process_instance_id: int, additional_processing_identifier: str | None = None) -> ProcessInstanceQueueModel: + queue_model = cls.try_unlock(process_instance_id, additional_processing_identifier=additional_processing_identifier) if queue_model is None: raise ExpectedLockNotFoundError(f"Could not find a lock for process instance: {process_instance_id}") return queue_model @classmethod - def try_unlock(cls, process_instance_id: int) -> ProcessInstanceQueueModel | None: - ctx = cls.get_thread_local_locking_context() + def try_unlock( + cls, process_instance_id: int, additional_processing_identifier: str | None = None + ) -> ProcessInstanceQueueModel | None: + ctx = cls.get_thread_local_locking_context(additional_processing_identifier=additional_processing_identifier) return ctx["locks"].pop(process_instance_id, None) # type: ignore @classmethod - def has_lock(cls, process_instance_id: int) -> bool: - ctx = cls.get_thread_local_locking_context() + def has_lock(cls, process_instance_id: int, additional_processing_identifier: str | None = None) -> bool: + ctx = cls.get_thread_local_locking_context(additional_processing_identifier=additional_processing_identifier) + current_app.logger.info(f"THREAD LOCK: {ctx}") return process_instance_id in ctx["locks"] @classmethod diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py b/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py index edee605c..a86db09b 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py @@ -101,6 +101,7 @@ from spiffworkflow_backend.services.workflow_execution_service import ExecutionS from spiffworkflow_backend.services.workflow_execution_service import ExecutionStrategyNotConfiguredError from spiffworkflow_backend.services.workflow_execution_service import SkipOneExecutionStrategy from spiffworkflow_backend.services.workflow_execution_service import TaskModelSavingDelegate +from spiffworkflow_backend.services.workflow_execution_service import TaskRunnability from spiffworkflow_backend.services.workflow_execution_service import WorkflowExecutionService from spiffworkflow_backend.services.workflow_execution_service import execution_strategy_named from spiffworkflow_backend.specs.start_event import StartEvent @@ -422,10 +423,12 @@ class ProcessInstanceProcessor: script_engine: PythonScriptEngine | None = None, workflow_completed_handler: WorkflowCompletedHandler | None = None, process_id_to_run: str | None = None, + additional_processing_identifier: str | None = None, ) -> None: """Create a Workflow Processor based on the serialized information available in the process_instance model.""" self._script_engine = script_engine or self.__class__._default_script_engine self._workflow_completed_handler = workflow_completed_handler + self.additional_processing_identifier = additional_processing_identifier self.setup_processor_with_process_instance( process_instance_model=process_instance_model, validate_only=validate_only, @@ -520,9 +523,7 @@ class ProcessInstanceProcessor: return bpmn_process_instance @staticmethod - def set_script_engine( - bpmn_process_instance: BpmnWorkflow, script_engine: PythonScriptEngine | None = None - ) -> None: + def set_script_engine(bpmn_process_instance: BpmnWorkflow, script_engine: PythonScriptEngine | None = None) -> None: script_engine_to_use = script_engine or ProcessInstanceProcessor._default_script_engine script_engine_to_use.environment.restore_state(bpmn_process_instance) bpmn_process_instance.script_engine = script_engine_to_use @@ -562,15 +563,11 @@ class ProcessInstanceProcessor: bpmn_process_definition.bpmn_identifier, bpmn_process_definition=bpmn_process_definition, ) - task_definitions = TaskDefinitionModel.query.filter_by( - bpmn_process_definition_id=bpmn_process_definition.id - ).all() + task_definitions = TaskDefinitionModel.query.filter_by(bpmn_process_definition_id=bpmn_process_definition.id).all() bpmn_process_definition_dict: dict = bpmn_process_definition.properties_json bpmn_process_definition_dict["task_specs"] = {} for task_definition in task_definitions: - bpmn_process_definition_dict["task_specs"][ - task_definition.bpmn_identifier - ] = task_definition.properties_json + bpmn_process_definition_dict["task_specs"][task_definition.bpmn_identifier] = task_definition.properties_json cls._update_bpmn_definition_mappings( bpmn_definition_to_task_definitions_mappings, bpmn_process_definition.bpmn_identifier, @@ -589,8 +586,7 @@ class ProcessInstanceProcessor: bpmn_process_subprocess_definitions = ( BpmnProcessDefinitionModel.query.join( BpmnProcessDefinitionRelationshipModel, - BpmnProcessDefinitionModel.id - == BpmnProcessDefinitionRelationshipModel.bpmn_process_definition_child_id, + BpmnProcessDefinitionModel.id == BpmnProcessDefinitionRelationshipModel.bpmn_process_definition_child_id, ) .filter_by(bpmn_process_definition_parent_id=bpmn_process_definition.id) .all() @@ -604,18 +600,14 @@ class ProcessInstanceProcessor: bpmn_process_definition=bpmn_subprocess_definition, ) bpmn_process_definition_dict: dict = bpmn_subprocess_definition.properties_json - spiff_bpmn_process_dict["subprocess_specs"][ - bpmn_subprocess_definition.bpmn_identifier - ] = bpmn_process_definition_dict + spiff_bpmn_process_dict["subprocess_specs"][bpmn_subprocess_definition.bpmn_identifier] = bpmn_process_definition_dict spiff_bpmn_process_dict["subprocess_specs"][bpmn_subprocess_definition.bpmn_identifier]["task_specs"] = {} bpmn_subprocess_definition_bpmn_identifiers[bpmn_subprocess_definition.id] = ( bpmn_subprocess_definition.bpmn_identifier ) task_definitions = TaskDefinitionModel.query.filter( - TaskDefinitionModel.bpmn_process_definition_id.in_( # type: ignore - bpmn_subprocess_definition_bpmn_identifiers.keys() - ) + TaskDefinitionModel.bpmn_process_definition_id.in_(bpmn_subprocess_definition_bpmn_identifiers.keys()) # type: ignore ).all() for task_definition in task_definitions: bpmn_subprocess_definition_bpmn_identifier = bpmn_subprocess_definition_bpmn_identifiers[ @@ -729,10 +721,7 @@ class ProcessInstanceProcessor: spiff_bpmn_process_dict["spec"] = element_unit_process_dict["spec"] keys = list(spiff_bpmn_process_dict["subprocess_specs"].keys()) for k in keys: - if ( - k not in subprocess_specs_for_ready_tasks - and k not in element_unit_process_dict["subprocess_specs"] - ): + if k not in subprocess_specs_for_ready_tasks and k not in element_unit_process_dict["subprocess_specs"]: spiff_bpmn_process_dict["subprocess_specs"].pop(k) bpmn_process = process_instance_model.bpmn_process @@ -810,9 +799,7 @@ class ProcessInstanceProcessor: finally: spiff_logger.setLevel(original_spiff_logger_log_level) else: - bpmn_process_instance = ProcessInstanceProcessor.get_bpmn_process_instance_from_workflow_spec( - spec, subprocesses - ) + bpmn_process_instance = ProcessInstanceProcessor.get_bpmn_process_instance_from_workflow_spec(spec, subprocesses) bpmn_process_instance.data[ProcessInstanceProcessor.VALIDATION_PROCESS_KEY] = validate_only return ( @@ -862,9 +849,7 @@ class ProcessInstanceProcessor: ) else: if group_model is None: - raise ( - NoPotentialOwnersForTaskError(f"Could not find a group with name matching lane: {task_lane}") - ) + raise (NoPotentialOwnersForTaskError(f"Could not find a group with name matching lane: {task_lane}")) potential_owner_ids = [i.user_id for i in group_model.user_group_assignments] self.raise_if_no_potential_owners( potential_owner_ids, @@ -924,16 +909,12 @@ class ProcessInstanceProcessor: single_process_hash = sha256(json.dumps(process_bpmn_properties, sort_keys=True).encode("utf8")).hexdigest() full_process_model_hash = None if full_bpmn_spec_dict is not None: - full_process_model_hash = sha256( - json.dumps(full_bpmn_spec_dict, sort_keys=True).encode("utf8") - ).hexdigest() + full_process_model_hash = sha256(json.dumps(full_bpmn_spec_dict, sort_keys=True).encode("utf8")).hexdigest() bpmn_process_definition = BpmnProcessDefinitionModel.query.filter_by( full_process_model_hash=full_process_model_hash ).first() else: - bpmn_process_definition = BpmnProcessDefinitionModel.query.filter_by( - single_process_hash=single_process_hash - ).first() + bpmn_process_definition = BpmnProcessDefinitionModel.query.filter_by(single_process_hash=single_process_hash).first() if bpmn_process_definition is None: task_specs = process_bpmn_properties.pop("task_specs") @@ -974,9 +955,7 @@ class ProcessInstanceProcessor: process_bpmn_identifier, bpmn_process_definition=bpmn_process_definition, ) - task_definitions = TaskDefinitionModel.query.filter_by( - bpmn_process_definition_id=bpmn_process_definition.id - ).all() + task_definitions = TaskDefinitionModel.query.filter_by(bpmn_process_definition_id=bpmn_process_definition.id).all() for task_definition in task_definitions: self._update_bpmn_definition_mappings( self.bpmn_definition_to_task_definitions_mappings, @@ -1067,15 +1046,11 @@ class ProcessInstanceProcessor: db.session.add(self.process_instance_model) db.session.commit() - human_tasks = HumanTaskModel.query.filter_by( - process_instance_id=self.process_instance_model.id, completed=False - ).all() + human_tasks = HumanTaskModel.query.filter_by(process_instance_id=self.process_instance_model.id, completed=False).all() ready_or_waiting_tasks = self.get_all_ready_or_waiting_tasks() process_model_display_name = "" - process_model_info = ProcessModelService.get_process_model( - self.process_instance_model.process_model_identifier - ) + process_model_info = ProcessModelService.get_process_model(self.process_instance_model.process_model_identifier) if process_model_info is not None: process_model_display_name = process_model_info.display_name @@ -1177,15 +1152,13 @@ class ProcessInstanceProcessor: if spiff_task.task_spec.manual: # Executing or not executing a human task results in the same state. current_app.logger.info( - f"Manually skipping Human Task {spiff_task.task_spec.name} of process" - f" instance {self.process_instance_model.id}" + f"Manually skipping Human Task {spiff_task.task_spec.name} of process instance {self.process_instance_model.id}" ) human_task = HumanTaskModel.query.filter_by(task_id=task_id).first() self.complete_task(spiff_task, human_task=human_task, user=user) elif execute: current_app.logger.info( - f"Manually executing Task {spiff_task.task_spec.name} of process" - f" instance {self.process_instance_model.id}" + f"Manually executing Task {spiff_task.task_spec.name} of process instance {self.process_instance_model.id}" ) self.do_engine_steps(save=True, execution_strategy_name="run_current_ready_tasks") else: @@ -1207,9 +1180,7 @@ class ProcessInstanceProcessor: bpmn_definition_to_task_definitions_mappings=self.bpmn_definition_to_task_definitions_mappings, ) task_service.update_all_tasks_from_spiff_tasks(spiff_tasks, [], start_time) - ProcessInstanceTmpService.add_event_to_process_instance( - self.process_instance_model, event_type, task_guid=task_id - ) + ProcessInstanceTmpService.add_event_to_process_instance(self.process_instance_model, event_type, task_guid=task_id) self.save() # Saving the workflow seems to reset the status @@ -1265,18 +1236,12 @@ class ProcessInstanceProcessor: bpmn_process_identifier: str, ) -> str: if bpmn_process_identifier is None: - raise ValueError( - "bpmn_file_full_path_from_bpmn_process_identifier: bpmn_process_identifier is unexpectedly None" - ) + raise ValueError("bpmn_file_full_path_from_bpmn_process_identifier: bpmn_process_identifier is unexpectedly None") - spec_reference = ( - ReferenceCacheModel.basic_query().filter_by(identifier=bpmn_process_identifier, type="process").first() - ) + spec_reference = ReferenceCacheModel.basic_query().filter_by(identifier=bpmn_process_identifier, type="process").first() bpmn_file_full_path = None if spec_reference is None: - bpmn_file_full_path = ProcessInstanceProcessor.backfill_missing_spec_reference_records( - bpmn_process_identifier - ) + bpmn_file_full_path = ProcessInstanceProcessor.backfill_missing_spec_reference_records(bpmn_process_identifier) else: bpmn_file_full_path = os.path.join( FileSystemService.root_path(), @@ -1286,10 +1251,7 @@ class ProcessInstanceProcessor: raise ( ApiError( error_code="could_not_find_bpmn_process_identifier", - message=( - "Could not find the the given bpmn process identifier from any sources:" - f" {bpmn_process_identifier}" - ), + message=f"Could not find the the given bpmn process identifier from any sources: {bpmn_process_identifier}", ) ) return os.path.abspath(bpmn_file_full_path) @@ -1325,9 +1287,7 @@ class ProcessInstanceProcessor: if new_bpmn_files: parser.add_bpmn_files(new_bpmn_files) - ProcessInstanceProcessor.update_spiff_parser_with_all_process_dependency_files( - parser, processed_identifiers - ) + ProcessInstanceProcessor.update_spiff_parser_with_all_process_dependency_files(parser, processed_identifiers) @staticmethod def get_spec( @@ -1384,6 +1344,7 @@ class ProcessInstanceProcessor: if bpmn_process_instance.is_completed(): return ProcessInstanceStatus.complete user_tasks = bpmn_process_instance.get_tasks(state=TaskState.READY, manual=True) + ready_tasks = bpmn_process_instance.get_tasks(state=TaskState.READY) # workflow.waiting_events (includes timers, and timers have a when firing property) @@ -1396,6 +1357,8 @@ class ProcessInstanceProcessor: # return ProcessInstanceStatus.waiting if len(user_tasks) > 0: return ProcessInstanceStatus.user_input_required + elif len(ready_tasks) > 0: + return ProcessInstanceStatus.running else: return ProcessInstanceStatus.waiting @@ -1455,15 +1418,17 @@ class ProcessInstanceProcessor: save: bool = False, execution_strategy_name: str | None = None, execution_strategy: ExecutionStrategy | None = None, - ) -> None: + ) -> TaskRunnability: if self.process_instance_model.persistence_level != "none": - with ProcessInstanceQueueService.dequeued(self.process_instance_model): + with ProcessInstanceQueueService.dequeued( + self.process_instance_model, additional_processing_identifier=self.additional_processing_identifier + ): # TODO: ideally we just lock in the execution service, but not sure # about _add_bpmn_process_definitions and if that needs to happen in # the same lock like it does on main - self._do_engine_steps(exit_at, save, execution_strategy_name, execution_strategy) + return self._do_engine_steps(exit_at, save, execution_strategy_name, execution_strategy) else: - self._do_engine_steps( + return self._do_engine_steps( exit_at, save=False, execution_strategy_name=execution_strategy_name, @@ -1476,7 +1441,7 @@ class ProcessInstanceProcessor: save: bool = False, execution_strategy_name: str | None = None, execution_strategy: ExecutionStrategy | None = None, - ) -> None: + ) -> TaskRunnability: self._add_bpmn_process_definitions() task_model_delegate = TaskModelSavingDelegate( @@ -1502,9 +1467,11 @@ class ProcessInstanceProcessor: execution_strategy, self._script_engine.environment.finalize_result, self.save, + additional_processing_identifier=self.additional_processing_identifier, ) - execution_service.run_and_save(exit_at, save) + task_runnability = execution_service.run_and_save(exit_at, save) self.check_all_tasks() + return task_runnability @classmethod def get_tasks_with_data(cls, bpmn_process_instance: BpmnWorkflow) -> list[SpiffTask]: @@ -1760,9 +1727,7 @@ class ProcessInstanceProcessor: return self.bpmn_process_instance.get_task_from_id(UUID(task_guid)) @classmethod - def get_task_by_bpmn_identifier( - cls, bpmn_task_identifier: str, bpmn_process_instance: BpmnWorkflow - ) -> SpiffTask | None: + def get_task_by_bpmn_identifier(cls, bpmn_task_identifier: str, bpmn_process_instance: BpmnWorkflow) -> SpiffTask | None: all_tasks = bpmn_process_instance.get_tasks(state=TaskState.ANY_MASK) for task in all_tasks: if task.task_spec.name == bpmn_task_identifier: diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_queue_service.py b/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_queue_service.py index 0ab6c8cf..569d4708 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_queue_service.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_queue_service.py @@ -35,22 +35,22 @@ class ProcessInstanceQueueService: @classmethod def enqueue_new_process_instance(cls, process_instance: ProcessInstanceModel, run_at_in_seconds: int) -> None: - queue_entry = ProcessInstanceQueueModel( - process_instance_id=process_instance.id, run_at_in_seconds=run_at_in_seconds - ) + queue_entry = ProcessInstanceQueueModel(process_instance_id=process_instance.id, run_at_in_seconds=run_at_in_seconds) cls._configure_and_save_queue_entry(process_instance, queue_entry) @classmethod - def _enqueue(cls, process_instance: ProcessInstanceModel) -> None: - queue_entry = ProcessInstanceLockService.unlock(process_instance.id) + def _enqueue(cls, process_instance: ProcessInstanceModel, additional_processing_identifier: str | None = None) -> None: + queue_entry = ProcessInstanceLockService.unlock( + process_instance.id, additional_processing_identifier=additional_processing_identifier + ) current_time = round(time.time()) if current_time > queue_entry.run_at_in_seconds: queue_entry.run_at_in_seconds = current_time cls._configure_and_save_queue_entry(process_instance, queue_entry) @classmethod - def _dequeue(cls, process_instance: ProcessInstanceModel) -> None: - locked_by = ProcessInstanceLockService.locked_by() + def _dequeue(cls, process_instance: ProcessInstanceModel, additional_processing_identifier: str | None = None) -> None: + locked_by = ProcessInstanceLockService.locked_by(additional_processing_identifier=additional_processing_identifier) current_time = round(time.time()) db.session.query(ProcessInstanceQueueModel).filter( @@ -84,16 +84,22 @@ class ProcessInstanceQueueService: f"It has already been locked by {queue_entry.locked_by}." ) - ProcessInstanceLockService.lock(process_instance.id, queue_entry) + ProcessInstanceLockService.lock( + process_instance.id, queue_entry, additional_processing_identifier=additional_processing_identifier + ) @classmethod @contextlib.contextmanager - def dequeued(cls, process_instance: ProcessInstanceModel) -> Generator[None, None, None]: - reentering_lock = ProcessInstanceLockService.has_lock(process_instance.id) + def dequeued( + cls, process_instance: ProcessInstanceModel, additional_processing_identifier: str | None = None + ) -> Generator[None, None, None]: + reentering_lock = ProcessInstanceLockService.has_lock( + process_instance.id, additional_processing_identifier=additional_processing_identifier + ) if not reentering_lock: # this can blow up with ProcessInstanceIsNotEnqueuedError or ProcessInstanceIsAlreadyLockedError # that's fine, let it bubble up. and in that case, there's no need to _enqueue / unlock - cls._dequeue(process_instance) + cls._dequeue(process_instance, additional_processing_identifier=additional_processing_identifier) try: yield except Exception as ex: @@ -107,7 +113,7 @@ class ProcessInstanceQueueService: raise ex finally: if not reentering_lock: - cls._enqueue(process_instance) + cls._enqueue(process_instance, additional_processing_identifier=additional_processing_identifier) @classmethod def entries_with_status( diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_report_service.py b/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_report_service.py index 87952d9f..b2317927 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_report_service.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_report_service.py @@ -173,9 +173,7 @@ class ProcessInstanceReportService: ), "system_report_completed_instances": system_report_completed_instances, "system_report_in_progress_instances_initiated_by_me": system_report_in_progress_instances_initiated_by_me, - "system_report_in_progress_instances_with_tasks_for_me": ( - system_report_in_progress_instances_with_tasks_for_me - ), + "system_report_in_progress_instances_with_tasks_for_me": system_report_in_progress_instances_with_tasks_for_me, "system_report_in_progress_instances_with_tasks": system_report_in_progress_instances_with_tasks, } if metadata_key not in temp_system_metadata_map: @@ -184,9 +182,7 @@ class ProcessInstanceReportService: return return_value @classmethod - def process_instance_metadata_as_columns( - cls, process_model_identifier: str | None = None - ) -> list[ReportMetadataColumn]: + def process_instance_metadata_as_columns(cls, process_model_identifier: str | None = None) -> list[ReportMetadataColumn]: columns_for_metadata_query = ( db.session.query(ProcessInstanceMetadataModel.key) .order_by(ProcessInstanceMetadataModel.key) @@ -226,9 +222,7 @@ class ProcessInstanceReportService: report_identifier: str | None = None, ) -> ProcessInstanceReportModel: if report_id is not None: - process_instance_report = ProcessInstanceReportModel.query.filter_by( - id=report_id, created_by_id=user.id - ).first() + process_instance_report = ProcessInstanceReportModel.query.filter_by(id=report_id, created_by_id=user.id).first() if process_instance_report is not None: return process_instance_report # type: ignore @@ -269,14 +263,10 @@ class ProcessInstanceReportService: process_instance_dict = process_instance_row[0].serialized() for metadata_column in metadata_columns: if metadata_column["accessor"] not in process_instance_dict: - process_instance_dict[metadata_column["accessor"]] = process_instance_mapping[ - metadata_column["accessor"] - ] + process_instance_dict[metadata_column["accessor"]] = process_instance_mapping[metadata_column["accessor"]] if "last_milestone_bpmn_name" in process_instance_mapping: - process_instance_dict["last_milestone_bpmn_name"] = process_instance_mapping[ - "last_milestone_bpmn_name" - ] + process_instance_dict["last_milestone_bpmn_name"] = process_instance_mapping["last_milestone_bpmn_name"] results.append(process_instance_dict) return results @@ -305,9 +295,7 @@ class ProcessInstanceReportService: .outerjoin(GroupModel, GroupModel.id == HumanTaskModel.lane_assignment_id) ) if restrict_human_tasks_to_user is not None: - human_task_query = human_task_query.filter( - HumanTaskUserModel.user_id == restrict_human_tasks_to_user.id - ) + human_task_query = human_task_query.filter(HumanTaskUserModel.user_id == restrict_human_tasks_to_user.id) potential_owner_usernames_from_group_concat_or_similar = cls._get_potential_owner_usernames(assigned_user) human_task = ( human_task_query.add_columns( @@ -327,9 +315,9 @@ class ProcessInstanceReportService: @classmethod def _get_potential_owner_usernames(cls, assigned_user: AliasedClass) -> Any: - potential_owner_usernames_from_group_concat_or_similar = func.group_concat( - assigned_user.username.distinct() - ).label("potential_owner_usernames") + potential_owner_usernames_from_group_concat_or_similar = func.group_concat(assigned_user.username.distinct()).label( + "potential_owner_usernames" + ) db_type = current_app.config.get("SPIFFWORKFLOW_BACKEND_DATABASE_TYPE") if db_type == "postgres": @@ -421,13 +409,9 @@ class ProcessInstanceReportService: if human_task_already_joined is False: process_instance_query = process_instance_query.join(HumanTaskModel) # type: ignore if process_status is not None: - non_active_statuses = [ - s for s in process_status.split(",") if s not in ProcessInstanceModel.active_statuses() - ] + non_active_statuses = [s for s in process_status.split(",") if s not in ProcessInstanceModel.active_statuses()] if len(non_active_statuses) == 0: - process_instance_query = process_instance_query.filter( - HumanTaskModel.completed.is_(False) # type: ignore - ) + process_instance_query = process_instance_query.filter(HumanTaskModel.completed.is_(False)) # type: ignore # Check to make sure the task is not only available for the group but the user as well if instances_with_tasks_waiting_for_me is not True: human_task_user_alias = aliased(HumanTaskUserModel) @@ -519,9 +503,7 @@ class ProcessInstanceReportService: and with_relation_to_me is True ): if user is None: - raise ProcessInstanceReportCannotBeRunError( - "A user must be specified to run report with with_relation_to_me" - ) + raise ProcessInstanceReportCannotBeRunError("A user must be specified to run report with with_relation_to_me") process_instance_query = process_instance_query.outerjoin(HumanTaskModel).outerjoin( HumanTaskUserModel, and_( @@ -550,9 +532,7 @@ class ProcessInstanceReportService: raise ProcessInstanceReportCannotBeRunError( "A user must be specified to run report with instances_with_tasks_completed_by_me." ) - process_instance_query = process_instance_query.filter( - ProcessInstanceModel.process_initiator_id != user.id - ) + process_instance_query = process_instance_query.filter(ProcessInstanceModel.process_initiator_id != user.id) process_instance_query = process_instance_query.join( HumanTaskModel, and_( @@ -569,9 +549,7 @@ class ProcessInstanceReportService: raise ProcessInstanceReportCannotBeRunError( "A user must be specified to run report with instances_with_tasks_waiting_for_me." ) - process_instance_query = process_instance_query.filter( - ProcessInstanceModel.process_initiator_id != user.id - ) + process_instance_query = process_instance_query.filter(ProcessInstanceModel.process_initiator_id != user.id) process_instance_query = process_instance_query.join( HumanTaskModel, and_( @@ -605,9 +583,7 @@ class ProcessInstanceReportService: if user_group_identifier is not None: if user is None: - raise ProcessInstanceReportCannotBeRunError( - "A user must be specified to run report with a group identifier." - ) + raise ProcessInstanceReportCannotBeRunError("A user must be specified to run report with a group identifier.") process_instance_query = cls.filter_by_user_group_identifier( process_instance_query=process_instance_query, user_group_identifier=user_group_identifier, @@ -647,9 +623,7 @@ class ProcessInstanceReportService: elif filter_for_column["operator"] == "less_than": join_conditions.append(instance_metadata_alias.value < filter_for_column["field_value"]) elif filter_for_column["operator"] == "contains": - join_conditions.append( - instance_metadata_alias.value.like(f"%{filter_for_column['field_value']}%") - ) + join_conditions.append(instance_metadata_alias.value.like(f"%{filter_for_column['field_value']}%")) elif filter_for_column["operator"] == "is_empty": # we still need to return results if the metadata value is null so make sure it's outer join isouter = True diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_service.py b/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_service.py index e5ec6cb1..712fb6d3 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_service.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_service.py @@ -17,12 +17,12 @@ from SpiffWorkflow.bpmn.specs.event_definitions.timer import TimerEventDefinitio from SpiffWorkflow.task import Task as SpiffTask # type: ignore from SpiffWorkflow.util.task import TaskState # type: ignore -from spiffworkflow_backend import db from spiffworkflow_backend.data_migrations.process_instance_migrator import ProcessInstanceMigrator from spiffworkflow_backend.exceptions.api_error import ApiError from spiffworkflow_backend.exceptions.error import HumanTaskAlreadyCompletedError from spiffworkflow_backend.exceptions.error import HumanTaskNotFoundError from spiffworkflow_backend.exceptions.error import UserDoesNotHaveAccessToTaskError +from spiffworkflow_backend.models.db import db from spiffworkflow_backend.models.group import GroupModel from spiffworkflow_backend.models.human_task import HumanTaskModel from spiffworkflow_backend.models.process_instance import ProcessInstanceApi @@ -42,6 +42,7 @@ from spiffworkflow_backend.services.process_instance_processor import ProcessIns from spiffworkflow_backend.services.process_instance_queue_service import ProcessInstanceIsAlreadyLockedError from spiffworkflow_backend.services.process_instance_queue_service import ProcessInstanceQueueService from spiffworkflow_backend.services.process_model_service import ProcessModelService +from spiffworkflow_backend.services.workflow_execution_service import TaskRunnability from spiffworkflow_backend.services.workflow_service import WorkflowService from spiffworkflow_backend.specs.start_event import StartConfiguration @@ -125,16 +126,12 @@ class ProcessInstanceService: user: UserModel, ) -> ProcessInstanceModel: process_model = ProcessModelService.get_process_model(process_model_identifier) - process_instance_model, (cycle_count, _, duration_in_seconds) = cls.create_process_instance( - process_model, user - ) + process_instance_model, (cycle_count, _, duration_in_seconds) = cls.create_process_instance(process_model, user) cls.register_process_model_cycles(process_model_identifier, cycle_count, duration_in_seconds) return process_instance_model @classmethod - def register_process_model_cycles( - cls, process_model_identifier: str, cycle_count: int, duration_in_seconds: int - ) -> None: + def register_process_model_cycles(cls, process_model_identifier: str, cycle_count: int, duration_in_seconds: int) -> None: # clean up old cycle record if it exists. event if the given cycle_count is 0 the previous version # of the model could have included a cycle timer start event cycles = ProcessModelCycleModel.query.filter( @@ -230,6 +227,7 @@ class ProcessInstanceService: return False + # this is only used from background processor @classmethod def do_waiting(cls, status_value: str) -> None: run_at_in_seconds_threshold = round(time.time()) @@ -268,12 +266,18 @@ class ProcessInstanceService: process_instance: ProcessInstanceModel, status_value: str | None = None, execution_strategy_name: str | None = None, - ) -> ProcessInstanceProcessor | None: + additional_processing_identifier: str | None = None, + ) -> tuple[ProcessInstanceProcessor | None, TaskRunnability]: processor = None - with ProcessInstanceQueueService.dequeued(process_instance): + task_runnability = TaskRunnability.unknown_if_ready_tasks + with ProcessInstanceQueueService.dequeued( + process_instance, additional_processing_identifier=additional_processing_identifier + ): ProcessInstanceMigrator.run(process_instance) processor = ProcessInstanceProcessor( - process_instance, workflow_completed_handler=cls.schedule_next_process_model_cycle + process_instance, + workflow_completed_handler=cls.schedule_next_process_model_cycle, + additional_processing_identifier=additional_processing_identifier, ) # if status_value is user_input_required (we are processing instances with that status from background processor), @@ -281,13 +285,16 @@ class ProcessInstanceService: # otherwise, in all cases, we should optimistically skip it. if status_value and cls.can_optimistically_skip(processor, status_value): current_app.logger.info(f"Optimistically skipped process_instance {process_instance.id}") - return None + return (processor, task_runnability) db.session.refresh(process_instance) if status_value is None or process_instance.status == status_value: - processor.do_engine_steps(save=True, execution_strategy_name=execution_strategy_name) + task_runnability = processor.do_engine_steps( + save=True, + execution_strategy_name=execution_strategy_name, + ) - return processor + return (processor, task_runnability) @staticmethod def processor_to_process_instance_api(process_instance: ProcessInstanceModel) -> ProcessInstanceApi: @@ -334,10 +341,7 @@ class ProcessInstanceService: else: raise ApiError.from_task( error_code="task_lane_user_error", - message=( - "Spiff Task %s lane user dict must have a key called" - " 'value' with the user's uid in it." - ) + message="Spiff Task %s lane user dict must have a key called 'value' with the user's uid in it." % spiff_task.task_spec.name, task=spiff_task, ) @@ -425,9 +429,7 @@ class ProcessInstanceService: models: list[ProcessInstanceFileDataModel], ) -> None: for model in models: - digest_reference = ( - f"data:{model.mimetype};name={model.filename};base64,{cls.FILE_DATA_DIGEST_PREFIX}{model.digest}" - ) + digest_reference = f"data:{model.mimetype};name={model.filename};base64,{cls.FILE_DATA_DIGEST_PREFIX}{model.digest}" if model.list_index is None: data[model.identifier] = digest_reference else: diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/services/process_model_service.py b/spiffworkflow-backend/src/spiffworkflow_backend/services/process_model_service.py index e5b8b7eb..d5a7efc3 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/services/process_model_service.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/services/process_model_service.py @@ -113,9 +113,7 @@ class ProcessModelService(FileSystemService): @classmethod def save_process_model(cls, process_model: ProcessModelInfo) -> None: - process_model_path = os.path.abspath( - os.path.join(FileSystemService.root_path(), process_model.id_for_file_path()) - ) + process_model_path = os.path.abspath(os.path.join(FileSystemService.root_path(), process_model.id_for_file_path())) os.makedirs(process_model_path, exist_ok=True) json_path = os.path.abspath(os.path.join(process_model_path, cls.PROCESS_MODEL_JSON_FILE)) json_data = cls.PROCESS_MODEL_SCHEMA.dump(process_model) @@ -126,9 +124,7 @@ class ProcessModelService(FileSystemService): @classmethod def process_model_delete(cls, process_model_id: str) -> None: - instances = ProcessInstanceModel.query.filter( - ProcessInstanceModel.process_model_identifier == process_model_id - ).all() + instances = ProcessInstanceModel.query.filter(ProcessInstanceModel.process_model_identifier == process_model_id).all() if len(instances) > 0: raise ProcessModelWithInstancesNotDeletableError( f"We cannot delete the model `{process_model_id}`, there are existing instances that depend on it." @@ -218,8 +214,7 @@ class ProcessModelService(FileSystemService): ) -> list[ProcessModelInfo]: if filter_runnable_as_extension and filter_runnable_by_user: raise Exception( - "It is not valid to filter process models by both filter_runnable_by_user and" - " filter_runnable_as_extension" + "It is not valid to filter process models by both filter_runnable_by_user and filter_runnable_as_extension" ) # get the full list (before we filter it by the ones you are allowed to start) @@ -277,13 +272,9 @@ class ProcessModelService(FileSystemService): permitted_process_model_identifiers = [] for process_model_identifier in process_model_identifiers: - modified_process_model_id = ProcessModelInfo.modify_process_identifier_for_path_param( - process_model_identifier - ) + modified_process_model_id = ProcessModelInfo.modify_process_identifier_for_path_param(process_model_identifier) uri = f"{permission_base_uri}/{modified_process_model_id}" - has_permission = AuthorizationService.user_has_permission( - user=user, permission=permission_to_check, target_uri=uri - ) + has_permission = AuthorizationService.user_has_permission(user=user, permission=permission_to_check, target_uri=uri) if has_permission: permitted_process_model_identifiers.append(process_model_identifier) @@ -351,9 +342,7 @@ class ProcessModelService(FileSystemService): for process_group in process_groups: modified_process_group_id = ProcessModelInfo.modify_process_identifier_for_path_param(process_group.id) uri = f"{permission_base_uri}/{modified_process_group_id}" - has_permission = AuthorizationService.user_has_permission( - user=user, permission=permission_to_check, target_uri=uri - ) + has_permission = AuthorizationService.user_has_permission(user=user, permission=permission_to_check, target_uri=uri) if has_permission: new_process_group_list.append(process_group) return new_process_group_list @@ -490,9 +479,7 @@ class ProcessModelService(FileSystemService): if cls.is_process_group(nested_item.path): # This is a nested group process_group.process_groups.append( - cls.find_or_create_process_group( - nested_item.path, find_all_nested_items=find_all_nested_items - ) + cls.find_or_create_process_group(nested_item.path, find_all_nested_items=find_all_nested_items) ) elif ProcessModelService.is_process_model(nested_item.path): process_group.process_models.append( diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/services/process_model_test_runner_service.py b/spiffworkflow-backend/src/spiffworkflow_backend/services/process_model_test_runner_service.py index 9df1b706..e4686471 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/services/process_model_test_runner_service.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/services/process_model_test_runner_service.py @@ -104,9 +104,7 @@ class ProcessModelTestRunnerMostlyPureSpiffDelegate(ProcessModelTestRunnerDelega if sub_parser.process_executable: executable_process = sub_parser.bpmn_id if executable_process is None: - raise BpmnFileMissingExecutableProcessError( - f"Executable process cannot be found in {bpmn_file}. Test cannot run." - ) + raise BpmnFileMissingExecutableProcessError(f"Executable process cannot be found in {bpmn_file}. Test cannot run.") all_related = self._find_related_bpmn_files(bpmn_file) for related_file in all_related: @@ -254,9 +252,7 @@ class ProcessModelTestRunner: f" class '{process_model_test_runner_delegate_class}' does not" ) - self.process_model_test_runner_delegate = process_model_test_runner_delegate_class( - process_model_directory_path - ) + self.process_model_test_runner_delegate = process_model_test_runner_delegate_class(process_model_directory_path) self.test_mappings = self._discover_process_model_test_cases() self.test_case_results: list[TestCaseResult] = [] @@ -388,9 +384,7 @@ class ProcessModelTestRunner: def _get_relative_path_of_bpmn_file(self, bpmn_file: str) -> str: return os.path.relpath(bpmn_file, start=self.process_model_directory_path) - def _exception_to_test_case_error_details( - self, exception: Exception | WorkflowTaskException - ) -> TestCaseErrorDetails: + def _exception_to_test_case_error_details(self, exception: Exception | WorkflowTaskException) -> TestCaseErrorDetails: error_messages = str(exception).split("\n") test_case_error_details = TestCaseErrorDetails(error_messages=error_messages) if isinstance(exception, WorkflowTaskException): diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/services/reference_cache_service.py b/spiffworkflow-backend/src/spiffworkflow_backend/services/reference_cache_service.py index a2987c7c..936f7eb4 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/services/reference_cache_service.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/services/reference_cache_service.py @@ -12,9 +12,7 @@ class ReferenceCacheService: def add_unique_reference_cache_object( cls, reference_objects: dict[str, ReferenceCacheModel], reference_cache: ReferenceCacheModel ) -> None: - reference_cache_unique = ( - f"{reference_cache.identifier}{reference_cache.relative_location}{reference_cache.type}" - ) + reference_cache_unique = f"{reference_cache.identifier}{reference_cache.relative_location}{reference_cache.type}" reference_objects[reference_cache_unique] = reference_cache @classmethod diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/services/service_task_service.py b/spiffworkflow-backend/src/spiffworkflow_backend/services/service_task_service.py index f5c76f5f..ba26a298 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/services/service_task_service.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/services/service_task_service.py @@ -112,10 +112,7 @@ class ServiceTaskDelegate: if code == 500: msg = "500 (Internal Server Error) - The service you called is experiencing technical difficulties." if code == 501: - msg = ( - "501 (Not Implemented) - This service needs to be called with the" - " different method (like POST not GET)." - ) + msg = "501 (Not Implemented) - This service needs to be called with the different method (like POST not GET)." return msg @classmethod @@ -149,11 +146,7 @@ class ServiceTaskDelegate: ) -> None: # v2 support base_error = None - if ( - "error" in parsed_response - and isinstance(parsed_response["error"], dict) - and "error_code" in parsed_response["error"] - ): + if "error" in parsed_response and isinstance(parsed_response["error"], dict) and "error_code" in parsed_response["error"]: base_error = parsed_response["error"] # v1 support or something terrible happened with a v2 connector elif status_code >= 300: diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/services/spec_file_service.py b/spiffworkflow-backend/src/spiffworkflow_backend/services/spec_file_service.py index c3cdddb7..48778e32 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/services/spec_file_service.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/services/spec_file_service.py @@ -146,9 +146,7 @@ class SpecFileService(FileSystemService): try: parser.add_bpmn_xml(cls.get_etree_from_xml_bytes(binary_data), filename=file_name) except Exception as exception: - raise ProcessModelFileInvalidError( - f"Received error trying to parse bpmn xml: {str(exception)}" - ) from exception + raise ProcessModelFileInvalidError(f"Received error trying to parse bpmn xml: {str(exception)}") from exception @classmethod def update_file( @@ -195,17 +193,13 @@ class SpecFileService(FileSystemService): ) if len(called_element_refs) > 0: process_model_identifiers: list[str] = [r.relative_location for r in called_element_refs] - permitted_process_model_identifiers = ( - ProcessModelService.process_model_identifiers_with_permission_for_user( - user=user, - permission_to_check="create", - permission_base_uri="/v1.0/process-instances", - process_model_identifiers=process_model_identifiers, - ) - ) - unpermitted_process_model_identifiers = set(process_model_identifiers) - set( - permitted_process_model_identifiers + permitted_process_model_identifiers = ProcessModelService.process_model_identifiers_with_permission_for_user( + user=user, + permission_to_check="create", + permission_base_uri="/v1.0/process-instances", + process_model_identifiers=process_model_identifiers, ) + unpermitted_process_model_identifiers = set(process_model_identifiers) - set(permitted_process_model_identifiers) if len(unpermitted_process_model_identifiers): raise NotAuthorizedError( "You are not authorized to use one or more processes as a called element:" diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/services/task_service.py b/spiffworkflow-backend/src/spiffworkflow_backend/services/task_service.py index 7b8c173d..170ad983 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/services/task_service.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/services/task_service.py @@ -66,9 +66,7 @@ class TaskModelError(Exception): self.line_number = exception.lineno self.offset = exception.offset elif isinstance(exception, NameError): - self.add_note( - WorkflowException.did_you_mean_from_name_error(exception, list(task_model.get_data().keys())) - ) + self.add_note(WorkflowException.did_you_mean_from_name_error(exception, list(task_model.get_data().keys()))) # If encountered in a sub-workflow, this traces back up the stack, # so we can tell how we got to this particular task, no matter how @@ -163,9 +161,7 @@ class TaskService: """ (parent_subprocess_guid, _parent_subprocess) = self.__class__._task_subprocess(spiff_task) if parent_subprocess_guid is not None: - spiff_task_of_parent_subprocess = spiff_task.workflow.top_workflow.get_task_from_id( - UUID(parent_subprocess_guid) - ) + spiff_task_of_parent_subprocess = spiff_task.workflow.top_workflow.get_task_from_id(UUID(parent_subprocess_guid)) if spiff_task_of_parent_subprocess is not None: self.update_task_model_with_spiff_task( @@ -196,15 +192,11 @@ class TaskService: # we are not sure why task_model.bpmn_process can be None while task_model.bpmn_process_id actually has a valid value bpmn_process = ( - new_bpmn_process - or task_model.bpmn_process - or BpmnProcessModel.query.filter_by(id=task_model.bpmn_process_id).first() + new_bpmn_process or task_model.bpmn_process or BpmnProcessModel.query.filter_by(id=task_model.bpmn_process_id).first() ) self.update_task_model(task_model, spiff_task) - bpmn_process_json_data = self.update_task_data_on_bpmn_process( - bpmn_process, bpmn_process_instance=spiff_task.workflow - ) + bpmn_process_json_data = self.update_task_data_on_bpmn_process(bpmn_process, bpmn_process_instance=spiff_task.workflow) if bpmn_process_json_data is not None: self.json_data_dicts[bpmn_process_json_data["hash"]] = bpmn_process_json_data self.task_models[task_model.guid] = task_model @@ -248,18 +240,14 @@ class TaskService: new_properties_json["success"] = spiff_workflow.success bpmn_process.properties_json = new_properties_json - bpmn_process_json_data = self.update_task_data_on_bpmn_process( - bpmn_process, bpmn_process_instance=spiff_workflow - ) + bpmn_process_json_data = self.update_task_data_on_bpmn_process(bpmn_process, bpmn_process_instance=spiff_workflow) if bpmn_process_json_data is not None: self.json_data_dicts[bpmn_process_json_data["hash"]] = bpmn_process_json_data self.bpmn_processes[bpmn_process.guid or "top_level"] = bpmn_process if spiff_workflow.parent_task_id: - direct_parent_bpmn_process = BpmnProcessModel.query.filter_by( - id=bpmn_process.direct_parent_process_id - ).first() + direct_parent_bpmn_process = BpmnProcessModel.query.filter_by(id=bpmn_process.direct_parent_process_id).first() self.update_bpmn_process(spiff_workflow.parent_workflow, direct_parent_bpmn_process) def update_task_model( @@ -396,9 +384,7 @@ class TaskService: for subprocess_guid in list(subprocesses): subprocess = subprocesses[subprocess_guid] if subprocess == spiff_workflow.parent_workflow: - direct_bpmn_process_parent = BpmnProcessModel.query.filter_by( - guid=str(subprocess_guid) - ).first() + direct_bpmn_process_parent = BpmnProcessModel.query.filter_by(guid=str(subprocess_guid)).first() if direct_bpmn_process_parent is None: raise BpmnProcessNotFoundError( f"Could not find bpmn process with guid: {str(subprocess_guid)} " @@ -406,9 +392,7 @@ class TaskService: ) if direct_bpmn_process_parent is None: - raise BpmnProcessNotFoundError( - f"Could not find a direct bpmn process parent for guid: {bpmn_process_guid}" - ) + raise BpmnProcessNotFoundError(f"Could not find a direct bpmn process parent for guid: {bpmn_process_guid}") bpmn_process.direct_parent_process_id = direct_bpmn_process_parent.id @@ -468,9 +452,7 @@ class TaskService: # Remove all the deleted/pruned tasks from the database. deleted_task_guids = [str(t.id) for t in deleted_spiff_tasks] tasks_to_clear = TaskModel.query.filter(TaskModel.guid.in_(deleted_task_guids)).all() # type: ignore - human_tasks_to_clear = HumanTaskModel.query.filter( - HumanTaskModel.task_id.in_(deleted_task_guids) # type: ignore - ).all() + human_tasks_to_clear = HumanTaskModel.query.filter(HumanTaskModel.task_id.in_(deleted_task_guids)).all() # type: ignore # delete human tasks first to avoid potential conflicts when deleting tasks. # otherwise sqlalchemy returns several warnings. @@ -587,25 +569,19 @@ class TaskService: return (bpmn_processes, task_models) @classmethod - def full_bpmn_process_path( - cls, bpmn_process: BpmnProcessModel, definition_column: str = "bpmn_identifier" - ) -> list[str]: + def full_bpmn_process_path(cls, bpmn_process: BpmnProcessModel, definition_column: str = "bpmn_identifier") -> list[str]: """Returns a list of bpmn process identifiers pointing the given bpmn_process.""" bpmn_process_identifiers: list[str] = [] if bpmn_process.guid: task_model = TaskModel.query.filter_by(guid=bpmn_process.guid).first() if task_model is None: - raise TaskNotFoundError( - f"Cannot find the corresponding task for the bpmn process with guid {bpmn_process.guid}." - ) + raise TaskNotFoundError(f"Cannot find the corresponding task for the bpmn process with guid {bpmn_process.guid}.") ( parent_bpmn_processes, _task_models_of_parent_bpmn_processes, ) = TaskService.task_models_of_parent_bpmn_processes(task_model) for parent_bpmn_process in parent_bpmn_processes: - bpmn_process_identifiers.append( - getattr(parent_bpmn_process.bpmn_process_definition, definition_column) - ) + bpmn_process_identifiers.append(getattr(parent_bpmn_process.bpmn_process_definition, definition_column)) bpmn_process_identifiers.append(getattr(bpmn_process.bpmn_process_definition, definition_column)) return bpmn_process_identifiers @@ -631,9 +607,7 @@ class TaskService: @classmethod def is_main_process_end_event(cls, spiff_task: SpiffTask) -> bool: - return ( - cls.get_task_type_from_spiff_task(spiff_task) == "EndEvent" and spiff_task.workflow.parent_workflow is None - ) + return cls.get_task_type_from_spiff_task(spiff_task) == "EndEvent" and spiff_task.workflow.parent_workflow is None @classmethod def bpmn_process_for_called_activity_or_top_level_process(cls, task_model: TaskModel) -> BpmnProcessModel: @@ -668,16 +642,12 @@ class TaskService: @classmethod def get_ready_signals_with_button_labels(cls, process_instance_id: int, associated_task_guid: str) -> list[dict]: - waiting_tasks: list[TaskModel] = TaskModel.query.filter_by( - state="WAITING", process_instance_id=process_instance_id - ).all() + waiting_tasks: list[TaskModel] = TaskModel.query.filter_by(state="WAITING", process_instance_id=process_instance_id).all() result = [] for task_model in waiting_tasks: task_definition = task_model.task_definition extensions: dict = ( - task_definition.properties_json["extensions"] - if "extensions" in task_definition.properties_json - else {} + task_definition.properties_json["extensions"] if "extensions" in task_definition.properties_json else {} ) event_definition: dict = ( task_definition.properties_json["event_definition"] @@ -748,9 +718,7 @@ class TaskService: spiff_task: SpiffTask, bpmn_definition_to_task_definitions_mappings: dict, ) -> TaskModel: - task_definition = bpmn_definition_to_task_definitions_mappings[spiff_task.workflow.spec.name][ - spiff_task.task_spec.name - ] + task_definition = bpmn_definition_to_task_definitions_mappings[spiff_task.workflow.spec.name][spiff_task.task_spec.name] task_model = TaskModel( guid=str(spiff_task.id), bpmn_process_id=bpmn_process.id, @@ -760,9 +728,7 @@ class TaskService: return task_model @classmethod - def _get_python_env_data_dict_from_spiff_task( - cls, spiff_task: SpiffTask, serializer: BpmnWorkflowSerializer - ) -> dict: + def _get_python_env_data_dict_from_spiff_task(cls, spiff_task: SpiffTask, serializer: BpmnWorkflowSerializer) -> dict: user_defined_state = spiff_task.workflow.script_engine.environment.user_defined_state() # this helps to convert items like datetime objects to be json serializable converted_data: dict = serializer.registry.convert(user_defined_state) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/services/user_service.py b/spiffworkflow-backend/src/spiffworkflow_backend/services/user_service.py index d185c9ed..23d740b3 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/services/user_service.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/services/user_service.py @@ -154,11 +154,7 @@ class UserService: @classmethod def apply_waiting_group_assignments(cls, user: UserModel) -> None: """Only called from create_user which is normally called at sign-in time""" - waiting = ( - UserGroupAssignmentWaitingModel() - .query.filter(UserGroupAssignmentWaitingModel.username == user.username) - .all() - ) + waiting = UserGroupAssignmentWaitingModel().query.filter(UserGroupAssignmentWaitingModel.username == user.username).all() for assignment in waiting: cls.add_user_to_group(user, assignment.group) db.session.delete(assignment) @@ -174,9 +170,7 @@ class UserService: @staticmethod def get_user_by_service_and_service_id(service: str, service_id: str) -> UserModel | None: - user: UserModel = ( - UserModel.query.filter(UserModel.service == service).filter(UserModel.service_id == service_id).first() - ) + user: UserModel = UserModel.query.filter(UserModel.service == service).filter(UserModel.service_id == service_id).first() if user: return user return None @@ -184,9 +178,7 @@ class UserService: @classmethod def add_user_to_human_tasks_if_appropriate(cls, user: UserModel) -> None: group_ids = [g.id for g in user.groups] - human_tasks = HumanTaskModel.query.filter( - HumanTaskModel.lane_assignment_id.in_(group_ids) # type: ignore - ).all() + human_tasks = HumanTaskModel.query.filter(HumanTaskModel.lane_assignment_id.in_(group_ids)).all() # type: ignore for human_task in human_tasks: human_task_user = HumanTaskUserModel(user_id=user.id, human_task_id=human_task.id) db.session.add(human_task_user) @@ -272,9 +264,7 @@ class UserService: db.session.commit() @classmethod - def find_or_create_guest_user( - cls, username: str = SPIFF_GUEST_USER, group_identifier: str = SPIFF_GUEST_GROUP - ) -> UserModel: + def find_or_create_guest_user(cls, username: str = SPIFF_GUEST_USER, group_identifier: str = SPIFF_GUEST_GROUP) -> UserModel: guest_user: UserModel | None = UserModel.query.filter_by( username=username, service="spiff_guest_service", service_id="spiff_guest_service_id" ).first() diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/services/workflow_execution_service.py b/spiffworkflow-backend/src/spiffworkflow_backend/services/workflow_execution_service.py index a93101b4..5bc3cdee 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/services/workflow_execution_service.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/services/workflow_execution_service.py @@ -4,6 +4,7 @@ import concurrent.futures import time from abc import abstractmethod from collections.abc import Callable +from datetime import datetime from typing import Any from uuid import UUID @@ -13,18 +14,26 @@ from flask import g from SpiffWorkflow.bpmn.exceptions import WorkflowTaskException # type: ignore from SpiffWorkflow.bpmn.serializer.workflow import BpmnWorkflowSerializer # type: ignore from SpiffWorkflow.bpmn.specs.event_definitions.message import MessageEventDefinition # type: ignore +from SpiffWorkflow.bpmn.specs.mixins.events.event_types import CatchingEvent # type: ignore from SpiffWorkflow.bpmn.workflow import BpmnWorkflow # type: ignore from SpiffWorkflow.exceptions import SpiffWorkflowException # type: ignore from SpiffWorkflow.task import Task as SpiffTask # type: ignore from SpiffWorkflow.util.task import TaskState # type: ignore +from spiffworkflow_backend.background_processing.celery_tasks.process_instance_task_producer import ( + queue_future_task_if_appropriate, +) from spiffworkflow_backend.exceptions.api_error import ApiError +from spiffworkflow_backend.helpers.spiff_enum import SpiffEnum from spiffworkflow_backend.models.db import db +from spiffworkflow_backend.models.future_task import FutureTaskModel from spiffworkflow_backend.models.message_instance import MessageInstanceModel from spiffworkflow_backend.models.message_instance_correlation import MessageInstanceCorrelationRuleModel from spiffworkflow_backend.models.process_instance import ProcessInstanceModel from spiffworkflow_backend.models.process_instance_event import ProcessInstanceEventType +from spiffworkflow_backend.models.task_instructions_for_end_user import TaskInstructionsForEndUserModel from spiffworkflow_backend.services.assertion_service import safe_assertion +from spiffworkflow_backend.services.jinja_service import JinjaService from spiffworkflow_backend.services.process_instance_lock_service import ProcessInstanceLockService from spiffworkflow_backend.services.process_instance_tmp_service import ProcessInstanceTmpService from spiffworkflow_backend.services.task_service import StartAndEndTimes @@ -51,6 +60,12 @@ class ExecutionStrategyNotConfiguredError(Exception): pass +class TaskRunnability(SpiffEnum): + has_ready_tasks = "has_ready_tasks" + no_ready_tasks = "no_ready_tasks" + unknown_if_ready_tasks = "unknown_if_ready_tasks" + + class EngineStepDelegate: """Interface of sorts for a concrete engine step delegate.""" @@ -81,43 +96,44 @@ SubprocessSpecLoader = Callable[[], dict[str, Any] | None] class ExecutionStrategy: """Interface of sorts for a concrete execution strategy.""" - def __init__( - self, delegate: EngineStepDelegate, subprocess_spec_loader: SubprocessSpecLoader, options: dict | None = None - ): + def __init__(self, delegate: EngineStepDelegate, subprocess_spec_loader: SubprocessSpecLoader, options: dict | None = None): self.delegate = delegate self.subprocess_spec_loader = subprocess_spec_loader self.options = options - def should_break_before(self, tasks: list[SpiffTask]) -> bool: + def should_break_before(self, tasks: list[SpiffTask], process_instance_model: ProcessInstanceModel) -> bool: return False def should_break_after(self, tasks: list[SpiffTask]) -> bool: return False + def should_do_before(self, bpmn_process_instance: BpmnWorkflow, process_instance_model: ProcessInstanceModel) -> None: + pass + def _run( self, spiff_task: SpiffTask, app: flask.app.Flask, - process_instance_id: Any | None, - process_model_identifier: Any | None, user: Any | None, ) -> SpiffTask: with app.app_context(): - app.config["THREAD_LOCAL_DATA"].process_instance_id = process_instance_id - app.config["THREAD_LOCAL_DATA"].process_model_identifier = process_model_identifier g.user = user spiff_task.run() return spiff_task - def spiff_run(self, bpmn_process_instance: BpmnWorkflow, exit_at: None = None) -> None: - # Note + def spiff_run( + self, bpmn_process_instance: BpmnWorkflow, process_instance_model: ProcessInstanceModel, exit_at: None = None + ) -> TaskRunnability: while True: bpmn_process_instance.refresh_waiting_tasks() + self.should_do_before(bpmn_process_instance, process_instance_model) engine_steps = self.get_ready_engine_steps(bpmn_process_instance) - if self.should_break_before(engine_steps): - break num_steps = len(engine_steps) + if self.should_break_before(engine_steps, process_instance_model=process_instance_model): + task_runnability = TaskRunnability.has_ready_tasks if num_steps > 0 else TaskRunnability.no_ready_tasks + break if num_steps == 0: + task_runnability = TaskRunnability.no_ready_tasks break elif num_steps == 1: spiff_task = engine_steps[0] @@ -130,11 +146,6 @@ class ExecutionStrategy: # service tasks at once - many api calls, and then get those responses back without # waiting for each individual task to complete. futures = [] - process_instance_id = None - process_model_identifier = None - if hasattr(current_app.config["THREAD_LOCAL_DATA"], "process_instance_id"): - process_instance_id = current_app.config["THREAD_LOCAL_DATA"].process_instance_id - process_model_identifier = current_app.config["THREAD_LOCAL_DATA"].process_model_identifier user = None if hasattr(g, "user"): user = g.user @@ -147,8 +158,6 @@ class ExecutionStrategy: self._run, spiff_task, current_app._get_current_object(), - process_instance_id, - process_model_identifier, user, ) ) @@ -156,9 +165,12 @@ class ExecutionStrategy: spiff_task = future.result() self.delegate.did_complete_task(spiff_task) if self.should_break_after(engine_steps): + # we could call the stuff at the top of the loop again and find out, but let's not do that unless we need to + task_runnability = TaskRunnability.unknown_if_ready_tasks break self.delegate.after_engine_steps(bpmn_process_instance) + return task_runnability def on_exception(self, bpmn_process_instance: BpmnWorkflow) -> None: self.delegate.on_exception(bpmn_process_instance) @@ -286,6 +298,43 @@ class GreedyExecutionStrategy(ExecutionStrategy): pass +class QueueInstructionsForEndUserExecutionStrategy(ExecutionStrategy): + """When you want to run tasks with instructionsForEndUser but you want to queue them. + + The queue can be used to display the instructions to user later. + """ + + def __init__(self, delegate: EngineStepDelegate, subprocess_spec_loader: SubprocessSpecLoader, options: dict | None = None): + super().__init__(delegate, subprocess_spec_loader, options) + self.tasks_that_have_been_seen: set[str] = set() + + def should_do_before(self, bpmn_process_instance: BpmnWorkflow, process_instance_model: ProcessInstanceModel) -> None: + tasks = bpmn_process_instance.get_tasks(state=TaskState.WAITING | TaskState.READY) + for spiff_task in tasks: + if hasattr(spiff_task.task_spec, "extensions") and spiff_task.task_spec.extensions.get( + "instructionsForEndUser", None + ): + task_guid = str(spiff_task.id) + if task_guid in self.tasks_that_have_been_seen: + continue + instruction = JinjaService.render_instructions_for_end_user(spiff_task) + if instruction != "": + TaskInstructionsForEndUserModel.insert_or_update_record( + task_guid=str(spiff_task.id), + process_instance_id=process_instance_model.id, + instruction=instruction, + ) + self.tasks_that_have_been_seen.add(str(spiff_task.id)) + + def should_break_before(self, tasks: list[SpiffTask], process_instance_model: ProcessInstanceModel) -> bool: + for spiff_task in tasks: + if hasattr(spiff_task.task_spec, "extensions") and spiff_task.task_spec.extensions.get( + "instructionsForEndUser", None + ): + return True + return False + + class RunUntilUserTaskOrMessageExecutionStrategy(ExecutionStrategy): """When you want to run tasks until you hit something to report to the end user. @@ -293,9 +342,11 @@ class RunUntilUserTaskOrMessageExecutionStrategy(ExecutionStrategy): but will stop if it hits instructions after the first task. """ - def should_break_before(self, tasks: list[SpiffTask]) -> bool: - for task in tasks: - if hasattr(task.task_spec, "extensions") and task.task_spec.extensions.get("instructionsForEndUser", None): + def should_break_before(self, tasks: list[SpiffTask], process_instance_model: ProcessInstanceModel) -> bool: + for spiff_task in tasks: + if hasattr(spiff_task.task_spec, "extensions") and spiff_task.task_spec.extensions.get( + "instructionsForEndUser", None + ): return True return False @@ -310,8 +361,11 @@ class RunCurrentReadyTasksExecutionStrategy(ExecutionStrategy): class SkipOneExecutionStrategy(ExecutionStrategy): """When you want to skip over the next task, rather than execute it.""" - def spiff_run(self, bpmn_process_instance: BpmnWorkflow, exit_at: None = None) -> None: + def spiff_run( + self, bpmn_process_instance: BpmnWorkflow, process_instance_model: ProcessInstanceModel, exit_at: None = None + ) -> TaskRunnability: spiff_task = None + engine_steps = [] if self.options and "spiff_task" in self.options.keys(): spiff_task = self.options["spiff_task"] else: @@ -323,13 +377,15 @@ class SkipOneExecutionStrategy(ExecutionStrategy): spiff_task.complete() self.delegate.did_complete_task(spiff_task) self.delegate.after_engine_steps(bpmn_process_instance) + # even if there was just 1 engine_step, and we ran it, we can't know that there is not another one + # that resulted from running that one, hence the unknown_if_ready_tasks + return TaskRunnability.has_ready_tasks if len(engine_steps) > 1 else TaskRunnability.unknown_if_ready_tasks -def execution_strategy_named( - name: str, delegate: EngineStepDelegate, spec_loader: SubprocessSpecLoader -) -> ExecutionStrategy: +def execution_strategy_named(name: str, delegate: EngineStepDelegate, spec_loader: SubprocessSpecLoader) -> ExecutionStrategy: cls = { "greedy": GreedyExecutionStrategy, + "queue_instructions_for_end_user": QueueInstructionsForEndUserExecutionStrategy, "run_until_user_message": RunUntilUserTaskOrMessageExecutionStrategy, "run_current_ready_tasks": RunCurrentReadyTasksExecutionStrategy, "skip_one": SkipOneExecutionStrategy, @@ -352,21 +408,27 @@ class WorkflowExecutionService: execution_strategy: ExecutionStrategy, process_instance_completer: ProcessInstanceCompleter, process_instance_saver: ProcessInstanceSaver, + additional_processing_identifier: str | None = None, ): self.bpmn_process_instance = bpmn_process_instance self.process_instance_model = process_instance_model self.execution_strategy = execution_strategy self.process_instance_completer = process_instance_completer self.process_instance_saver = process_instance_saver + self.additional_processing_identifier = additional_processing_identifier # names of methods that do spiff stuff: # processor.do_engine_steps calls: # run # execution_strategy.spiff_run # spiff.[some_run_task_method] - def run_and_save(self, exit_at: None = None, save: bool = False) -> None: + def run_and_save(self, exit_at: None = None, save: bool = False) -> TaskRunnability: if self.process_instance_model.persistence_level != "none": - with safe_assertion(ProcessInstanceLockService.has_lock(self.process_instance_model.id)) as tripped: + with safe_assertion( + ProcessInstanceLockService.has_lock( + self.process_instance_model.id, additional_processing_identifier=self.additional_processing_identifier + ) + ) as tripped: if tripped: raise AssertionError( "The current thread has not obtained a lock for this process" @@ -376,13 +438,17 @@ class WorkflowExecutionService: self.bpmn_process_instance.refresh_waiting_tasks() # TODO: implicit re-entrant locks here `with_dequeued` - self.execution_strategy.spiff_run(self.bpmn_process_instance, exit_at) + task_runnability = self.execution_strategy.spiff_run( + self.bpmn_process_instance, exit_at=exit_at, process_instance_model=self.process_instance_model + ) if self.bpmn_process_instance.is_completed(): self.process_instance_completer(self.bpmn_process_instance) self.process_bpmn_messages() self.queue_waiting_receive_messages() + self.schedule_waiting_timer_events() + return task_runnability except WorkflowTaskException as wte: ProcessInstanceTmpService.add_event_to_process_instance( self.process_instance_model, @@ -398,10 +464,32 @@ class WorkflowExecutionService: finally: if self.process_instance_model.persistence_level != "none": + # even if a task fails, try to persist all tasks, which will include the error state. self.execution_strategy.add_object_to_db_session(self.bpmn_process_instance) if save: self.process_instance_saver() + def is_happening_soon(self, time_in_seconds: int) -> bool: + # if it is supposed to happen in less than the amount of time we take between polling runs + return time_in_seconds - time.time() < int( + current_app.config["SPIFFWORKFLOW_BACKEND_BACKGROUND_SCHEDULER_FUTURE_TASK_EXECUTION_INTERVAL_IN_SECONDS"] + ) + + def schedule_waiting_timer_events(self) -> None: + # TODO: update to always insert records so we can remove user_input_required and possibly waiting apscheduler jobs + if current_app.config["SPIFFWORKFLOW_BACKEND_CELERY_ENABLED"]: + waiting_tasks = self.bpmn_process_instance.get_tasks(state=TaskState.WAITING, spec_class=CatchingEvent) + for spiff_task in waiting_tasks: + event = spiff_task.task_spec.event_definition.details(spiff_task) + if "Time" in event.event_type: + time_string = event.value + run_at_in_seconds = round(datetime.fromisoformat(time_string).timestamp()) + FutureTaskModel.insert_or_update(guid=str(spiff_task.id), run_at_in_seconds=run_at_in_seconds) + if self.is_happening_soon(run_at_in_seconds): + queue_future_task_if_appropriate( + self.process_instance_model, eta_in_seconds=run_at_in_seconds, task_guid=str(spiff_task.id) + ) + def process_bpmn_messages(self) -> None: # FIXE: get_events clears out the events so if we have other events we care about # this will clear them out as well. @@ -474,10 +562,12 @@ class WorkflowExecutionService: class ProfiledWorkflowExecutionService(WorkflowExecutionService): """A profiled version of the workflow execution service.""" - def run_and_save(self, exit_at: None = None, save: bool = False) -> None: + def run_and_save(self, exit_at: None = None, save: bool = False) -> TaskRunnability: import cProfile from pstats import SortKey + task_runnability = TaskRunnability.unknown_if_ready_tasks with cProfile.Profile() as pr: - super().run_and_save(exit_at=exit_at, save=save) + task_runnability = super().run_and_save(exit_at=exit_at, save=save) pr.print_stats(sort=SortKey.CUMULATIVE) + return task_runnability diff --git a/spiffworkflow-backend/tests/data/script-task-with-instruction/script_task_with_instruction.bpmn b/spiffworkflow-backend/tests/data/script-task-with-instruction/script_task_with_instruction.bpmn new file mode 100644 index 00000000..eae14998 --- /dev/null +++ b/spiffworkflow-backend/tests/data/script-task-with-instruction/script_task_with_instruction.bpmn @@ -0,0 +1,59 @@ + + + + + Flow_0jml23i + + + + Flow_0xzoduo + + + + + + We run script one + + Flow_0jml23i + Flow_0ula2mv + a = 1 + + + + We run script two + + Flow_0ula2mv + Flow_0xzoduo + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/spiffworkflow-backend/tests/data/user-task-with-timer/user_task_with_timer.bpmn b/spiffworkflow-backend/tests/data/user-task-with-timer/user_task_with_timer.bpmn new file mode 100644 index 00000000..ba221943 --- /dev/null +++ b/spiffworkflow-backend/tests/data/user-task-with-timer/user_task_with_timer.bpmn @@ -0,0 +1,69 @@ + + + + + Flow_0903e0h + + + + Flow_1yn50r0 + + + + Flow_0903e0h + Flow_1yn50r0 + + + Flow_1ky2hak + + "PT10M" + + + + Flow_1ky2hak + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/spiffworkflow-backend/tests/spiffworkflow_backend/helpers/base_test.py b/spiffworkflow-backend/tests/spiffworkflow_backend/helpers/base_test.py index 04836821..55d43450 100644 --- a/spiffworkflow-backend/tests/spiffworkflow_backend/helpers/base_test.py +++ b/spiffworkflow-backend/tests/spiffworkflow_backend/helpers/base_test.py @@ -316,9 +316,7 @@ class BaseTest: grant_type: str = "permit", ) -> UserModel: user = BaseTest.find_or_create_user(username=username) - return cls.add_permissions_to_user( - user, target_uri=target_uri, permission_names=permission_names, grant_type=grant_type - ) + return cls.add_permissions_to_user(user, target_uri=target_uri, permission_names=permission_names, grant_type=grant_type) @classmethod def add_permissions_to_user( @@ -518,9 +516,7 @@ class BaseTest: report_metadata=report_metadata, user=user, ) - response = self.post_to_process_instance_list( - client, user, report_metadata=process_instance_report.get_report_metadata() - ) + response = self.post_to_process_instance_list(client, user, report_metadata=process_instance_report.get_report_metadata()) if expect_to_find_instance is True: assert len(response.json["results"]) == 1 diff --git a/spiffworkflow-backend/tests/spiffworkflow_backend/integration/test_authentication.py b/spiffworkflow-backend/tests/spiffworkflow_backend/integration/test_authentication.py index a362fea0..d93227d6 100644 --- a/spiffworkflow-backend/tests/spiffworkflow_backend/integration/test_authentication.py +++ b/spiffworkflow-backend/tests/spiffworkflow_backend/integration/test_authentication.py @@ -2,10 +2,10 @@ import ast import base64 import re import time -from typing import Any from flask.app import Flask from flask.testing import FlaskClient +from pytest_mock.plugin import MockerFixture from spiffworkflow_backend.models.db import db from spiffworkflow_backend.models.user import UserModel from spiffworkflow_backend.services.authentication_service import AuthenticationService @@ -122,7 +122,7 @@ class TestAuthentication(BaseTest): def test_can_login_with_valid_user( self, app: Flask, - mocker: Any, + mocker: MockerFixture, client: FlaskClient, with_db_and_bpmn_file_cleanup: None, ) -> None: diff --git a/spiffworkflow-backend/tests/spiffworkflow_backend/integration/test_for_good_errors.py b/spiffworkflow-backend/tests/spiffworkflow_backend/integration/test_for_good_errors.py index bd6298ce..15f78566 100644 --- a/spiffworkflow-backend/tests/spiffworkflow_backend/integration/test_for_good_errors.py +++ b/spiffworkflow-backend/tests/spiffworkflow_backend/integration/test_for_good_errors.py @@ -87,9 +87,7 @@ class TestForGoodErrors(BaseTest): _dequeued_interstitial_stream(process_instance_id) """Returns the next available user task for a given process instance, if possible.""" - human_tasks = ( - db.session.query(HumanTaskModel).filter(HumanTaskModel.process_instance_id == process_instance_id).all() - ) + human_tasks = db.session.query(HumanTaskModel).filter(HumanTaskModel.process_instance_id == process_instance_id).all() assert len(human_tasks) > 0, "No human tasks found for process." human_task = human_tasks[0] response = client.get( diff --git a/spiffworkflow-backend/tests/spiffworkflow_backend/integration/test_logging_service.py b/spiffworkflow-backend/tests/spiffworkflow_backend/integration/test_logging_service.py index 7cfbbaaa..6e58022c 100644 --- a/spiffworkflow-backend/tests/spiffworkflow_backend/integration/test_logging_service.py +++ b/spiffworkflow-backend/tests/spiffworkflow_backend/integration/test_logging_service.py @@ -28,9 +28,7 @@ class TestLoggingService(BaseTest): process_model_id="misc/category_number_one/simple_form", process_model_source_directory="simple_form", ) - process_instance = self.create_process_instance_from_process_model( - process_model=process_model, user=initiator_user - ) + process_instance = self.create_process_instance_from_process_model(process_model=process_model, user=initiator_user) processor = ProcessInstanceProcessor(process_instance) processor.do_engine_steps(save=True) @@ -39,9 +37,7 @@ class TestLoggingService(BaseTest): assert len(human_task.potential_owners) == 1 assert human_task.potential_owners[0] == initiator_user - spiff_task = processor.__class__.get_task_by_bpmn_identifier( - human_task.task_name, processor.bpmn_process_instance - ) + spiff_task = processor.__class__.get_task_by_bpmn_identifier(human_task.task_name, processor.bpmn_process_instance) ProcessInstanceService.complete_form_task(processor, spiff_task, {"name": "HEY"}, initiator_user, human_task) headers = self.logged_in_headers(with_super_admin_user) @@ -86,9 +82,7 @@ class TestLoggingService(BaseTest): process_model_id="misc/category_number_one/simple_form", process_model_source_directory="simple_form", ) - process_instance = self.create_process_instance_from_process_model( - process_model=process_model, user=initiator_user - ) + process_instance = self.create_process_instance_from_process_model(process_model=process_model, user=initiator_user) processor = ProcessInstanceProcessor(process_instance) processor.do_engine_steps(save=True) @@ -97,9 +91,7 @@ class TestLoggingService(BaseTest): assert len(human_task.potential_owners) == 1 assert human_task.potential_owners[0] == initiator_user - spiff_task = processor.__class__.get_task_by_bpmn_identifier( - human_task.task_name, processor.bpmn_process_instance - ) + spiff_task = processor.__class__.get_task_by_bpmn_identifier(human_task.task_name, processor.bpmn_process_instance) ProcessInstanceService.complete_form_task(processor, spiff_task, {"name": "HEY"}, initiator_user, human_task) process_instance = ProcessInstanceModel.query.filter_by(id=process_instance.id).first() diff --git a/spiffworkflow-backend/tests/spiffworkflow_backend/integration/test_onboarding.py b/spiffworkflow-backend/tests/spiffworkflow_backend/integration/test_onboarding.py index 979246fa..a0bdbc16 100644 --- a/spiffworkflow-backend/tests/spiffworkflow_backend/integration/test_onboarding.py +++ b/spiffworkflow-backend/tests/spiffworkflow_backend/integration/test_onboarding.py @@ -77,7 +77,5 @@ class TestOnboarding(BaseTest): assert len(results.json.keys()) == 4 assert results.json["type"] == "user_input_required" assert results.json["process_instance_id"] is not None - instance = ProcessInstanceModel.query.filter( - ProcessInstanceModel.id == results.json["process_instance_id"] - ).first() + instance = ProcessInstanceModel.query.filter(ProcessInstanceModel.id == results.json["process_instance_id"]).first() assert instance is not None diff --git a/spiffworkflow-backend/tests/spiffworkflow_backend/integration/test_process_api.py b/spiffworkflow-backend/tests/spiffworkflow_backend/integration/test_process_api.py index 4529938c..8e31218b 100644 --- a/spiffworkflow-backend/tests/spiffworkflow_backend/integration/test_process_api.py +++ b/spiffworkflow-backend/tests/spiffworkflow_backend/integration/test_process_api.py @@ -109,9 +109,7 @@ class TestProcessApi(BaseTest): self.add_permissions_to_principal( principal, target_uri="/v1.0/process-groups/deny_group:%", permission_names=["create"], grant_type="deny" ) - self.add_permissions_to_principal( - principal, target_uri="/v1.0/process-groups/test_group:%", permission_names=["create"] - ) + self.add_permissions_to_principal(principal, target_uri="/v1.0/process-groups/test_group:%", permission_names=["create"]) request_body = { "requests_to_check": { "/v1.0/process-groups": ["GET", "POST"], @@ -354,9 +352,7 @@ class TestProcessApi(BaseTest): ) headers = self.logged_in_headers(with_super_admin_user) # create an instance from a model - response = self.create_process_instance_from_process_model_id_with_api( - client, process_model_identifier, headers - ) + response = self.create_process_instance_from_process_model_id_with_api(client, process_model_identifier, headers) data = json.loads(response.get_data(as_text=True)) # make sure the instance has the correct model @@ -374,8 +370,7 @@ class TestProcessApi(BaseTest): assert data["error_code"] == "existing_instances" assert ( data["message"] - == f"We cannot delete the model `{process_model_identifier}`, there are" - " existing instances that depend on it." + == f"We cannot delete the model `{process_model_identifier}`, there are existing instances that depend on it." ) def test_process_model_update( @@ -603,9 +598,7 @@ class TestProcessApi(BaseTest): # When adding a process model with 4 processes and a decision, 5 new records will be in the Cache assert len(ReferenceCacheModel.basic_query().all()) == 6 - user_one = self.create_user_with_permission( - username="user_one", target_uri="/v1.0/process-groups/test_group_one:*" - ) + user_one = self.create_user_with_permission(username="user_one", target_uri="/v1.0/process-groups/test_group_one:*") self.add_permissions_to_user(user=user_one, target_uri="/v1.0/processes", permission_names=["read"]) self.add_permissions_to_user( user=user_one, target_uri="/v1.0/process-instances/test_group_one:*", permission_names=["create"] @@ -774,9 +767,7 @@ class TestProcessApi(BaseTest): for i in range(5): group_id = f"test_process_group_{i}" group_display_name = f"Test Group {i}" - self.create_process_group_with_api( - client, with_super_admin_user, group_id, display_name=group_display_name - ) + self.create_process_group_with_api(client, with_super_admin_user, group_id, display_name=group_display_name) # get all groups response = client.get( @@ -1233,9 +1224,7 @@ class TestProcessApi(BaseTest): with_db_and_bpmn_file_cleanup: None, with_super_admin_user: UserModel, ) -> None: - process_model = self.create_group_and_model_with_bpmn( - client, with_super_admin_user, bpmn_file_name="random_fact.bpmn" - ) + process_model = self.create_group_and_model_with_bpmn(client, with_super_admin_user, bpmn_file_name="random_fact.bpmn") modified_process_model_identifier = process_model.modify_process_identifier_for_path_param(process_model.id) response = client.get( @@ -1320,6 +1309,39 @@ class TestProcessApi(BaseTest): assert response.json["status"] == "complete" assert response.json["process_model_identifier"] == process_model.id + def test_process_instance_run_with_instructions( + self, + app: Flask, + client: FlaskClient, + with_db_and_bpmn_file_cleanup: None, + with_super_admin_user: UserModel, + ) -> None: + # process_model_id = "runs_without_input/sample" + process_model = self.create_group_and_model_with_bpmn( + client=client, + user=with_super_admin_user, + process_group_id="runs_without_input", + process_model_id="sample", + bpmn_file_name=None, + bpmn_file_location="sample", + ) + + headers = self.logged_in_headers(with_super_admin_user) + response = self.create_process_instance_from_process_model_id_with_api(client, process_model.id, headers) + assert response.json is not None + process_instance_id = response.json["id"] + response = client.post( + f"/v1.0/process-instances/{self.modify_process_identifier_for_path_param(process_model.id)}/{process_instance_id}/run", + headers=self.logged_in_headers(with_super_admin_user), + ) + + assert response.status_code == 200 + assert response.json is not None + assert isinstance(response.json["updated_at_in_seconds"], int) + assert response.json["updated_at_in_seconds"] > 0 + assert response.json["status"] == "complete" + assert response.json["process_model_identifier"] == process_model.id + def test_process_instance_show( self, app: Flask, @@ -1337,9 +1359,7 @@ class TestProcessApi(BaseTest): ) modified_process_model_identifier = self.modify_process_identifier_for_path_param(process_model.id) headers = self.logged_in_headers(with_super_admin_user) - create_response = self.create_process_instance_from_process_model_id_with_api( - client, process_model.id, headers - ) + create_response = self.create_process_instance_from_process_model_id_with_api(client, process_model.id, headers) assert create_response.json is not None process_instance_id = create_response.json["id"] client.post( @@ -1377,9 +1397,7 @@ class TestProcessApi(BaseTest): assert spec_reference modified_process_model_identifier = self.modify_process_identifier_for_path_param(process_model.id) headers = self.logged_in_headers(with_super_admin_user) - create_response = self.create_process_instance_from_process_model_id_with_api( - client, process_model.id, headers - ) + create_response = self.create_process_instance_from_process_model_id_with_api(client, process_model.id, headers) assert create_response.json is not None assert create_response.status_code == 201 process_instance_id = create_response.json["id"] @@ -1840,9 +1858,7 @@ class TestProcessApi(BaseTest): "columns": [], "order_by": [], } - response = self.post_to_process_instance_list( - client, with_super_admin_user, report_metadata=report_metadata_body - ) + response = self.post_to_process_instance_list(client, with_super_admin_user, report_metadata=report_metadata_body) results = response.json["results"] assert len(results) == 5 @@ -1865,9 +1881,7 @@ class TestProcessApi(BaseTest): "columns": [], "order_by": [], } - response = self.post_to_process_instance_list( - client, with_super_admin_user, report_metadata=report_metadata_body - ) + response = self.post_to_process_instance_list(client, with_super_admin_user, report_metadata=report_metadata_body) results = response.json["results"] assert len(results) == 1 assert results[0]["status"] == ProcessInstanceStatus[statuses[i]].value @@ -1884,9 +1898,7 @@ class TestProcessApi(BaseTest): "columns": [], "order_by": [], } - response = self.post_to_process_instance_list( - client, with_super_admin_user, report_metadata=report_metadata_body - ) + response = self.post_to_process_instance_list(client, with_super_admin_user, report_metadata=report_metadata_body) results = response.json["results"] assert len(results) == 2 assert results[0]["status"] in ["complete", "not_started"] @@ -1899,9 +1911,7 @@ class TestProcessApi(BaseTest): "columns": [], "order_by": [], } - response = self.post_to_process_instance_list( - client, with_super_admin_user, report_metadata=report_metadata_body - ) + response = self.post_to_process_instance_list(client, with_super_admin_user, report_metadata=report_metadata_body) results = response.json["results"] assert len(results) == 4 for i in range(4): @@ -1921,9 +1931,7 @@ class TestProcessApi(BaseTest): "columns": [], "order_by": [], } - response = self.post_to_process_instance_list( - client, with_super_admin_user, report_metadata=report_metadata_body - ) + response = self.post_to_process_instance_list(client, with_super_admin_user, report_metadata=report_metadata_body) results = response.json["results"] assert len(results) == 2 assert json.loads(results[0]["bpmn_version_control_identifier"]) in (2, 3) @@ -1938,9 +1946,7 @@ class TestProcessApi(BaseTest): "columns": [], "order_by": [], } - response = self.post_to_process_instance_list( - client, with_super_admin_user, report_metadata=report_metadata_body - ) + response = self.post_to_process_instance_list(client, with_super_admin_user, report_metadata=report_metadata_body) results = response.json["results"] assert len(results) == 2 assert json.loads(results[0]["bpmn_version_control_identifier"]) in (1, 2) @@ -1955,9 +1961,7 @@ class TestProcessApi(BaseTest): "columns": [], "order_by": [], } - response = self.post_to_process_instance_list( - client, with_super_admin_user, report_metadata=report_metadata_body - ) + response = self.post_to_process_instance_list(client, with_super_admin_user, report_metadata=report_metadata_body) results = response.json["results"] assert len(results) == 3 for i in range(3): @@ -2865,9 +2869,7 @@ class TestProcessApi(BaseTest): processor = ProcessInstanceProcessor(process_instance) processor.do_engine_steps(save=True) - process_instance_metadata = ProcessInstanceMetadataModel.query.filter_by( - process_instance_id=process_instance.id - ).all() + process_instance_metadata = ProcessInstanceMetadataModel.query.filter_by(process_instance_id=process_instance.id).all() assert len(process_instance_metadata) == 3 report_metadata: ReportMetadata = { @@ -3142,9 +3144,7 @@ class TestProcessApi(BaseTest): processor = ProcessInstanceProcessor(process_instance) processor.do_engine_steps(save=True) - process_instance_metadata = ProcessInstanceMetadataModel.query.filter_by( - process_instance_id=process_instance.id - ).all() + process_instance_metadata = ProcessInstanceMetadataModel.query.filter_by(process_instance_id=process_instance.id).all() assert len(process_instance_metadata) == 2 process_model = load_test_spec( @@ -3168,9 +3168,7 @@ class TestProcessApi(BaseTest): processor = ProcessInstanceProcessor(process_instance) processor.do_engine_steps(save=True) - process_instance_metadata = ProcessInstanceMetadataModel.query.filter_by( - process_instance_id=process_instance.id - ).all() + process_instance_metadata = ProcessInstanceMetadataModel.query.filter_by(process_instance_id=process_instance.id).all() assert len(process_instance_metadata) == 3 response = client.get( diff --git a/spiffworkflow-backend/tests/spiffworkflow_backend/integration/test_tasks_controller.py b/spiffworkflow-backend/tests/spiffworkflow_backend/integration/test_tasks_controller.py index cf7485d1..01353044 100644 --- a/spiffworkflow-backend/tests/spiffworkflow_backend/integration/test_tasks_controller.py +++ b/spiffworkflow-backend/tests/spiffworkflow_backend/integration/test_tasks_controller.py @@ -50,9 +50,7 @@ class TestTasksController(BaseTest): # Call this to assure all engine-steps are fully processed. _dequeued_interstitial_stream(process_instance_id) - human_tasks = ( - db.session.query(HumanTaskModel).filter(HumanTaskModel.process_instance_id == process_instance_id).all() - ) + human_tasks = db.session.query(HumanTaskModel).filter(HumanTaskModel.process_instance_id == process_instance_id).all() { r.bpmn_identifier diff --git a/spiffworkflow-backend/tests/spiffworkflow_backend/scripts/test_get_all_permissions.py b/spiffworkflow-backend/tests/spiffworkflow_backend/scripts/test_get_all_permissions.py index c8f87797..8f53b4df 100644 --- a/spiffworkflow-backend/tests/spiffworkflow_backend/scripts/test_get_all_permissions.py +++ b/spiffworkflow-backend/tests/spiffworkflow_backend/scripts/test_get_all_permissions.py @@ -24,9 +24,7 @@ class TestGetAllPermissions(BaseTest): AuthorizationService.add_permission_from_uri_or_macro( permission="start", target="PG:hey:group", group_identifier="my_test_group" ) - AuthorizationService.add_permission_from_uri_or_macro( - permission="all", target="/tasks", group_identifier="my_test_group" - ) + AuthorizationService.add_permission_from_uri_or_macro(permission="all", target="/tasks", group_identifier="my_test_group") expected_permissions = [ { diff --git a/spiffworkflow-backend/tests/spiffworkflow_backend/scripts/test_get_current_task_info.py b/spiffworkflow-backend/tests/spiffworkflow_backend/scripts/test_get_current_task_info.py index 46066e69..e268b5be 100644 --- a/spiffworkflow-backend/tests/spiffworkflow_backend/scripts/test_get_current_task_info.py +++ b/spiffworkflow-backend/tests/spiffworkflow_backend/scripts/test_get_current_task_info.py @@ -22,9 +22,7 @@ class TestGetCurrentTaskInfo(BaseTest): process_model_id="misc/test-get-current-task-info-script", process_model_source_directory="test-get-current-task-info-script", ) - process_instance = self.create_process_instance_from_process_model( - process_model=process_model, user=initiator_user - ) + process_instance = self.create_process_instance_from_process_model(process_model=process_model, user=initiator_user) processor = ProcessInstanceProcessor(process_instance) processor.do_engine_steps(save=True) @@ -33,9 +31,7 @@ class TestGetCurrentTaskInfo(BaseTest): assert len(human_task.potential_owners) == 1 assert human_task.potential_owners[0] == initiator_user - spiff_task = processor.__class__.get_task_by_bpmn_identifier( - human_task.task_name, processor.bpmn_process_instance - ) + spiff_task = processor.__class__.get_task_by_bpmn_identifier(human_task.task_name, processor.bpmn_process_instance) ProcessInstanceService.complete_form_task(processor, spiff_task, {}, initiator_user, human_task) assert process_instance.status == ProcessInstanceStatus.complete.value assert spiff_task is not None diff --git a/spiffworkflow-backend/tests/spiffworkflow_backend/scripts/test_get_group_members.py b/spiffworkflow-backend/tests/spiffworkflow_backend/scripts/test_get_group_members.py index ed837f50..9b084270 100644 --- a/spiffworkflow-backend/tests/spiffworkflow_backend/scripts/test_get_group_members.py +++ b/spiffworkflow-backend/tests/spiffworkflow_backend/scripts/test_get_group_members.py @@ -33,9 +33,7 @@ class TestGetGroupMembers(BaseTest): bpmn_file_name="get_group_members.bpmn", process_model_source_directory="get_group_members", ) - process_instance = self.create_process_instance_from_process_model( - process_model=process_model, user=initiator_user - ) + process_instance = self.create_process_instance_from_process_model(process_model=process_model, user=initiator_user) processor = ProcessInstanceProcessor(process_instance) processor.do_engine_steps(save=True) diff --git a/spiffworkflow-backend/tests/spiffworkflow_backend/scripts/test_get_last_user_completing_task.py b/spiffworkflow-backend/tests/spiffworkflow_backend/scripts/test_get_last_user_completing_task.py index d3f725d2..43c1bbb4 100644 --- a/spiffworkflow-backend/tests/spiffworkflow_backend/scripts/test_get_last_user_completing_task.py +++ b/spiffworkflow-backend/tests/spiffworkflow_backend/scripts/test_get_last_user_completing_task.py @@ -22,9 +22,7 @@ class TestGetLastUserCompletingTask(BaseTest): # bpmn_file_name="simp.bpmn", process_model_source_directory="simple_form", ) - process_instance = self.create_process_instance_from_process_model( - process_model=process_model, user=initiator_user - ) + process_instance = self.create_process_instance_from_process_model(process_model=process_model, user=initiator_user) processor = ProcessInstanceProcessor(process_instance) processor.do_engine_steps(save=True) @@ -33,16 +31,12 @@ class TestGetLastUserCompletingTask(BaseTest): assert len(human_task.potential_owners) == 1 assert human_task.potential_owners[0] == initiator_user - spiff_task = processor.__class__.get_task_by_bpmn_identifier( - human_task.task_name, processor.bpmn_process_instance - ) + spiff_task = processor.__class__.get_task_by_bpmn_identifier(human_task.task_name, processor.bpmn_process_instance) ProcessInstanceService.complete_form_task(processor, spiff_task, {"name": "HEY"}, initiator_user, human_task) assert len(process_instance.active_human_tasks) == 1 human_task = process_instance.active_human_tasks[0] - spiff_task = processor.__class__.get_task_by_bpmn_identifier( - human_task.task_name, processor.bpmn_process_instance - ) + spiff_task = processor.__class__.get_task_by_bpmn_identifier(human_task.task_name, processor.bpmn_process_instance) ProcessInstanceService.complete_form_task(processor, spiff_task, {}, initiator_user, human_task) assert spiff_task is not None diff --git a/spiffworkflow-backend/tests/spiffworkflow_backend/scripts/test_get_localtime.py b/spiffworkflow-backend/tests/spiffworkflow_backend/scripts/test_get_localtime.py index 98eaf38a..8fa12b65 100644 --- a/spiffworkflow-backend/tests/spiffworkflow_backend/scripts/test_get_localtime.py +++ b/spiffworkflow-backend/tests/spiffworkflow_backend/scripts/test_get_localtime.py @@ -46,16 +46,12 @@ class TestGetLocaltime(BaseTest): bpmn_file_name="get_localtime.bpmn", process_model_source_directory="get_localtime", ) - process_instance = self.create_process_instance_from_process_model( - process_model=process_model, user=initiator_user - ) + process_instance = self.create_process_instance_from_process_model(process_model=process_model, user=initiator_user) processor = ProcessInstanceProcessor(process_instance) processor.do_engine_steps(save=True) human_task = process_instance.active_human_tasks[0] - spiff_task = processor.__class__.get_task_by_bpmn_identifier( - human_task.task_name, processor.bpmn_process_instance - ) + spiff_task = processor.__class__.get_task_by_bpmn_identifier(human_task.task_name, processor.bpmn_process_instance) ProcessInstanceService.complete_form_task( processor, @@ -66,9 +62,7 @@ class TestGetLocaltime(BaseTest): ) human_task = process_instance.active_human_tasks[0] - spiff_task = processor.__class__.get_task_by_bpmn_identifier( - human_task.task_name, processor.bpmn_process_instance - ) + spiff_task = processor.__class__.get_task_by_bpmn_identifier(human_task.task_name, processor.bpmn_process_instance) assert spiff_task diff --git a/spiffworkflow-backend/tests/spiffworkflow_backend/scripts/test_get_process_initiator_user.py b/spiffworkflow-backend/tests/spiffworkflow_backend/scripts/test_get_process_initiator_user.py index 988a755f..4f403fca 100644 --- a/spiffworkflow-backend/tests/spiffworkflow_backend/scripts/test_get_process_initiator_user.py +++ b/spiffworkflow-backend/tests/spiffworkflow_backend/scripts/test_get_process_initiator_user.py @@ -21,9 +21,7 @@ class TestGetProcessInitiatorUser(BaseTest): process_model_id="misc/category_number_one/simple_form", process_model_source_directory="simple_form", ) - process_instance = self.create_process_instance_from_process_model( - process_model=process_model, user=initiator_user - ) + process_instance = self.create_process_instance_from_process_model(process_model=process_model, user=initiator_user) processor = ProcessInstanceProcessor(process_instance) processor.do_engine_steps(save=True) @@ -32,9 +30,7 @@ class TestGetProcessInitiatorUser(BaseTest): assert len(human_task.potential_owners) == 1 assert human_task.potential_owners[0] == initiator_user - spiff_task = processor.__class__.get_task_by_bpmn_identifier( - human_task.task_name, processor.bpmn_process_instance - ) + spiff_task = processor.__class__.get_task_by_bpmn_identifier(human_task.task_name, processor.bpmn_process_instance) ProcessInstanceService.complete_form_task(processor, spiff_task, {"name": "HEY"}, initiator_user, human_task) assert spiff_task is not None diff --git a/spiffworkflow-backend/tests/spiffworkflow_backend/scripts/test_get_url_for_task_with_bpmn_identifier.py b/spiffworkflow-backend/tests/spiffworkflow_backend/scripts/test_get_url_for_task_with_bpmn_identifier.py index b71bf942..05eb86b4 100644 --- a/spiffworkflow-backend/tests/spiffworkflow_backend/scripts/test_get_url_for_task_with_bpmn_identifier.py +++ b/spiffworkflow-backend/tests/spiffworkflow_backend/scripts/test_get_url_for_task_with_bpmn_identifier.py @@ -22,9 +22,7 @@ class TestGetUrlForTaskWithBpmnIdentifier(BaseTest): process_model_id="misc/test-get-url-for-task-with-bpmn-identifier", process_model_source_directory="test-get-url-for-task-with-bpmn-identifier", ) - process_instance = self.create_process_instance_from_process_model( - process_model=process_model, user=initiator_user - ) + process_instance = self.create_process_instance_from_process_model(process_model=process_model, user=initiator_user) processor = ProcessInstanceProcessor(process_instance) processor.do_engine_steps(save=True) @@ -33,9 +31,7 @@ class TestGetUrlForTaskWithBpmnIdentifier(BaseTest): assert len(human_task.potential_owners) == 1 assert human_task.potential_owners[0] == initiator_user - spiff_task = processor.__class__.get_task_by_bpmn_identifier( - human_task.task_name, processor.bpmn_process_instance - ) + spiff_task = processor.__class__.get_task_by_bpmn_identifier(human_task.task_name, processor.bpmn_process_instance) ProcessInstanceService.complete_form_task(processor, spiff_task, {}, initiator_user, human_task) assert process_instance.status == ProcessInstanceStatus.complete.value assert spiff_task is not None diff --git a/spiffworkflow-backend/tests/spiffworkflow_backend/scripts/test_refresh_permissions.py b/spiffworkflow-backend/tests/spiffworkflow_backend/scripts/test_refresh_permissions.py index 9bd51701..49400fce 100644 --- a/spiffworkflow-backend/tests/spiffworkflow_backend/scripts/test_refresh_permissions.py +++ b/spiffworkflow-backend/tests/spiffworkflow_backend/scripts/test_refresh_permissions.py @@ -24,9 +24,7 @@ class TestRefreshPermissions(BaseTest): process_model_id="refresh_permissions", process_model_source_directory="script_refresh_permissions", ) - process_instance = self.create_process_instance_from_process_model( - process_model=process_model, user=basic_user - ) + process_instance = self.create_process_instance_from_process_model(process_model=process_model, user=basic_user) processor = ProcessInstanceProcessor(process_instance) @@ -34,9 +32,7 @@ class TestRefreshPermissions(BaseTest): processor.do_engine_steps(save=True) assert "ScriptUnauthorizedForUserError" in str(exception) - process_instance = self.create_process_instance_from_process_model( - process_model=process_model, user=privileged_user - ) + process_instance = self.create_process_instance_from_process_model(process_model=process_model, user=privileged_user) processor = ProcessInstanceProcessor(process_instance) processor.do_engine_steps(save=True) assert process_instance.status == "complete" diff --git a/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_authorization_service.py b/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_authorization_service.py index 3f6a4681..9152388f 100644 --- a/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_authorization_service.py +++ b/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_authorization_service.py @@ -68,21 +68,15 @@ class TestAuthorizationService(BaseTest): process_model_source_directory="model_with_lanes", ) - process_instance = self.create_process_instance_from_process_model( - process_model=process_model, user=initiator_user - ) + process_instance = self.create_process_instance_from_process_model(process_model=process_model, user=initiator_user) processor = ProcessInstanceProcessor(process_instance) processor.do_engine_steps(save=True) human_task = process_instance.active_human_tasks[0] - spiff_task = processor.__class__.get_task_by_bpmn_identifier( - human_task.task_name, processor.bpmn_process_instance - ) + spiff_task = processor.__class__.get_task_by_bpmn_identifier(human_task.task_name, processor.bpmn_process_instance) ProcessInstanceService.complete_form_task(processor, spiff_task, {}, initiator_user, human_task) human_task = process_instance.active_human_tasks[0] - spiff_task = processor.__class__.get_task_by_bpmn_identifier( - human_task.task_name, processor.bpmn_process_instance - ) + spiff_task = processor.__class__.get_task_by_bpmn_identifier(human_task.task_name, processor.bpmn_process_instance) finance_user = AuthorizationService.create_user_from_sign_in( { "username": "testuser2", @@ -143,9 +137,7 @@ class TestAuthorizationService(BaseTest): ("/task-data/some-process-group:some-process-model:*", "update"), ] ) - permissions_to_assign = AuthorizationService.explode_permissions( - "all", "PG:/some-process-group/some-process-model" - ) + permissions_to_assign = AuthorizationService.explode_permissions("all", "PG:/some-process-group/some-process-model") permissions_to_assign_tuples = sorted([(p.target_uri, p.permission) for p in permissions_to_assign]) assert permissions_to_assign_tuples == expected_permissions @@ -177,9 +169,7 @@ class TestAuthorizationService(BaseTest): ("/process-instances/some-process-group:some-process-model:*", "create"), ] ) - permissions_to_assign = AuthorizationService.explode_permissions( - "start", "PG:/some-process-group/some-process-model" - ) + permissions_to_assign = AuthorizationService.explode_permissions("start", "PG:/some-process-group/some-process-model") permissions_to_assign_tuples = sorted([(p.target_uri, p.permission) for p in permissions_to_assign]) assert permissions_to_assign_tuples == expected_permissions @@ -229,9 +219,7 @@ class TestAuthorizationService(BaseTest): ("/task-data/some-process-group:some-process-model/*", "update"), ] ) - permissions_to_assign = AuthorizationService.explode_permissions( - "all", "PM:/some-process-group/some-process-model" - ) + permissions_to_assign = AuthorizationService.explode_permissions("all", "PM:/some-process-group/some-process-model") permissions_to_assign_tuples = sorted([(p.target_uri, p.permission) for p in permissions_to_assign]) assert permissions_to_assign_tuples == expected_permissions @@ -263,9 +251,7 @@ class TestAuthorizationService(BaseTest): ("/process-instances/some-process-group:some-process-model/*", "create"), ] ) - permissions_to_assign = AuthorizationService.explode_permissions( - "start", "PM:/some-process-group/some-process-model" - ) + permissions_to_assign = AuthorizationService.explode_permissions("start", "PM:/some-process-group/some-process-model") permissions_to_assign_tuples = sorted([(p.target_uri, p.permission) for p in permissions_to_assign]) assert permissions_to_assign_tuples == expected_permissions @@ -615,9 +601,7 @@ class TestAuthorizationService(BaseTest): UserService.add_user_to_group(user, user_group) AuthorizationService.add_permission_from_uri_or_macro(user_group.identifier, "read", "PG:hey") AuthorizationService.add_permission_from_uri_or_macro(user_group.identifier, "DENY:read", "PG:hey:yo") - AuthorizationService.add_permission_from_uri_or_macro( - user_group.identifier, "DENY:read", "/process-groups/hey:new" - ) + AuthorizationService.add_permission_from_uri_or_macro(user_group.identifier, "DENY:read", "/process-groups/hey:new") self.assert_user_has_permission(user, "read", "/v1.0/process-groups/hey") diff --git a/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_dot_notation.py b/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_dot_notation.py index 869b7b27..adf93a82 100644 --- a/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_dot_notation.py +++ b/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_dot_notation.py @@ -36,9 +36,7 @@ class TestDotNotation(BaseTest): "invoice.invoiceAmount": "1000.00", "invoice.dueDate": "09/30/2022", } - ProcessInstanceService.complete_form_task( - processor, user_task, form_data, process_instance.process_initiator, human_task - ) + ProcessInstanceService.complete_form_task(processor, user_task, form_data, process_instance.process_initiator, human_task) expected = { "contibutorName": "Elizabeth", diff --git a/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_future_task.py b/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_future_task.py new file mode 100644 index 00000000..654b69d4 --- /dev/null +++ b/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_future_task.py @@ -0,0 +1,47 @@ +import time + +import pytest +from flask.app import Flask +from pytest_mock.plugin import MockerFixture +from spiffworkflow_backend.models.db import db +from spiffworkflow_backend.models.future_task import FutureTaskModel +from spiffworkflow_backend.services.process_instance_processor import ProcessInstanceProcessor + +from tests.spiffworkflow_backend.helpers.base_test import BaseTest +from tests.spiffworkflow_backend.helpers.test_data import load_test_spec + + +class TestFutureTask(BaseTest): + def test_can_add_record_from_bpmn_timer_event( + self, + app: Flask, + mocker: MockerFixture, + with_db_and_bpmn_file_cleanup: None, + ) -> None: + with self.app_config_mock(app, "SPIFFWORKFLOW_BACKEND_CELERY_ENABLED", True): + process_model = load_test_spec( + process_model_id="test_group/user-task-with-timer", + process_model_source_directory="user-task-with-timer", + ) + process_instance = self.create_process_instance_from_process_model(process_model=process_model) + processor = ProcessInstanceProcessor(process_instance) + mock = mocker.patch("celery.current_app.send_task") + processor.do_engine_steps(save=True) + + # this one is not happening soon. it will get picked up by the "every five minutes" job + assert mock.call_count == 0 + + assert process_instance.status == "user_input_required" + + future_tasks = FutureTaskModel.query.all() + assert len(future_tasks) == 1 + future_task = future_tasks[0] + ten_minutes_from_now = 10 * 60 + time.time() + + # give a 2 second leeway + assert future_task.run_at_in_seconds == pytest.approx(ten_minutes_from_now, abs=2) + + twenty_minutes_from_now = round(20 * 60 + time.time()) + FutureTaskModel.insert_or_update(guid=future_task.guid, run_at_in_seconds=twenty_minutes_from_now) + db.session.commit() + assert future_task.run_at_in_seconds == pytest.approx(twenty_minutes_from_now, abs=2) diff --git a/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_permissions.py b/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_permissions.py index fd49fb86..43f949a7 100644 --- a/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_permissions.py +++ b/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_permissions.py @@ -47,9 +47,7 @@ class TestPermissions(BaseTest): db.session.add(permission_assignment) db.session.commit() - def test_group_a_admin_needs_to_stay_away_from_group_b( - self, app: Flask, with_db_and_bpmn_file_cleanup: None - ) -> None: + def test_group_a_admin_needs_to_stay_away_from_group_b(self, app: Flask, with_db_and_bpmn_file_cleanup: None) -> None: process_group_ids = ["group-a", "group-b"] process_group_a_id = process_group_ids[0] process_group_b_id = process_group_ids[1] @@ -118,9 +116,7 @@ class TestPermissions(BaseTest): self.assert_user_has_permission(user, "update", f"/{process_group_a_id}") - def test_user_can_be_read_models_with_global_permission( - self, app: Flask, with_db_and_bpmn_file_cleanup: None - ) -> None: + def test_user_can_be_read_models_with_global_permission(self, app: Flask, with_db_and_bpmn_file_cleanup: None) -> None: process_group_ids = ["group-a", "group-b"] process_group_a_id = process_group_ids[0] process_group_b_id = process_group_ids[1] diff --git a/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_process_caller_service.py b/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_process_caller_service.py index 3afa6455..a23c2a48 100644 --- a/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_process_caller_service.py +++ b/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_process_caller_service.py @@ -32,9 +32,7 @@ def with_single_process_caller(with_clean_cache: None) -> Generator[None, None, def with_multiple_process_callers(with_clean_cache: None) -> Generator[None, None, None]: db.session.add(ProcessCallerCacheModel(process_identifier="called_many", calling_process_identifier="one_caller")) db.session.add(ProcessCallerCacheModel(process_identifier="called_many", calling_process_identifier="two_caller")) - db.session.add( - ProcessCallerCacheModel(process_identifier="called_many", calling_process_identifier="three_caller") - ) + db.session.add(ProcessCallerCacheModel(process_identifier="called_many", calling_process_identifier="three_caller")) db.session.commit() yield diff --git a/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_process_instance_processor.py b/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_process_instance_processor.py index f5502157..d833c6fc 100644 --- a/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_process_instance_processor.py +++ b/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_process_instance_processor.py @@ -16,6 +16,7 @@ from spiffworkflow_backend.models.process_instance_event import ProcessInstanceE from spiffworkflow_backend.models.process_instance_event import ProcessInstanceEventType from spiffworkflow_backend.models.task import TaskModel # noqa: F401 from spiffworkflow_backend.models.task_definition import TaskDefinitionModel +from spiffworkflow_backend.models.task_instructions_for_end_user import TaskInstructionsForEndUserModel from spiffworkflow_backend.services.authorization_service import AuthorizationService from spiffworkflow_backend.services.process_instance_processor import ProcessInstanceProcessor from spiffworkflow_backend.services.process_instance_service import ProcessInstanceService @@ -75,9 +76,7 @@ class TestProcessInstanceProcessor(BaseTest): bpmn_file_name="lanes.bpmn", process_model_source_directory="model_with_lanes", ) - process_instance = self.create_process_instance_from_process_model( - process_model=process_model, user=initiator_user - ) + process_instance = self.create_process_instance_from_process_model(process_model=process_model, user=initiator_user) processor = ProcessInstanceProcessor(process_instance) processor.do_engine_steps(save=True) @@ -87,9 +86,7 @@ class TestProcessInstanceProcessor(BaseTest): assert len(human_task.potential_owners) == 1 assert human_task.potential_owners[0] == initiator_user - spiff_task = processor.__class__.get_task_by_bpmn_identifier( - human_task.task_name, processor.bpmn_process_instance - ) + spiff_task = processor.__class__.get_task_by_bpmn_identifier(human_task.task_name, processor.bpmn_process_instance) with pytest.raises(UserDoesNotHaveAccessToTaskError): ProcessInstanceService.complete_form_task(processor, spiff_task, {}, finance_user, human_task) ProcessInstanceService.complete_form_task(processor, spiff_task, {}, initiator_user, human_task) @@ -100,9 +97,7 @@ class TestProcessInstanceProcessor(BaseTest): assert len(human_task.potential_owners) == 1 assert human_task.potential_owners[0] == finance_user - spiff_task = processor.__class__.get_task_by_bpmn_identifier( - human_task.task_name, processor.bpmn_process_instance - ) + spiff_task = processor.__class__.get_task_by_bpmn_identifier(human_task.task_name, processor.bpmn_process_instance) with pytest.raises(UserDoesNotHaveAccessToTaskError): ProcessInstanceService.complete_form_task(processor, spiff_task, {}, initiator_user, human_task) @@ -113,9 +108,7 @@ class TestProcessInstanceProcessor(BaseTest): assert len(human_task.potential_owners) == 1 assert human_task.potential_owners[0] == initiator_user - spiff_task = processor.__class__.get_task_by_bpmn_identifier( - human_task.task_name, processor.bpmn_process_instance - ) + spiff_task = processor.__class__.get_task_by_bpmn_identifier(human_task.task_name, processor.bpmn_process_instance) ProcessInstanceService.complete_form_task(processor, spiff_task, {}, initiator_user, human_task) assert process_instance.status == ProcessInstanceStatus.complete.value @@ -143,9 +136,7 @@ class TestProcessInstanceProcessor(BaseTest): bpmn_file_name="lanes_with_owner_dict.bpmn", process_model_source_directory="model_with_lanes", ) - process_instance = self.create_process_instance_from_process_model( - process_model=process_model, user=initiator_user - ) + process_instance = self.create_process_instance_from_process_model(process_model=process_model, user=initiator_user) processor = ProcessInstanceProcessor(process_instance) processor.do_engine_steps(save=True) @@ -155,9 +146,7 @@ class TestProcessInstanceProcessor(BaseTest): assert len(human_task.potential_owners) == 1 assert human_task.potential_owners[0] == initiator_user - spiff_task = processor.__class__.get_task_by_bpmn_identifier( - human_task.task_name, processor.bpmn_process_instance - ) + spiff_task = processor.__class__.get_task_by_bpmn_identifier(human_task.task_name, processor.bpmn_process_instance) with pytest.raises(UserDoesNotHaveAccessToTaskError): ProcessInstanceService.complete_form_task(processor, spiff_task, {}, finance_user_three, human_task) ProcessInstanceService.complete_form_task(processor, spiff_task, {}, initiator_user, human_task) @@ -169,9 +158,7 @@ class TestProcessInstanceProcessor(BaseTest): assert len(human_task.potential_owners) == 2 assert human_task.potential_owners == [finance_user_three, finance_user_four] - spiff_task = processor.__class__.get_task_by_bpmn_identifier( - human_task.task_name, processor.bpmn_process_instance - ) + spiff_task = processor.__class__.get_task_by_bpmn_identifier(human_task.task_name, processor.bpmn_process_instance) with pytest.raises(UserDoesNotHaveAccessToTaskError): ProcessInstanceService.complete_form_task(processor, spiff_task, {}, initiator_user, human_task) @@ -184,9 +171,7 @@ class TestProcessInstanceProcessor(BaseTest): assert len(human_task.potential_owners) == 1 assert human_task.potential_owners[0] == finance_user_four - spiff_task = processor.__class__.get_task_by_bpmn_identifier( - human_task.task_name, processor.bpmn_process_instance - ) + spiff_task = processor.__class__.get_task_by_bpmn_identifier(human_task.task_name, processor.bpmn_process_instance) with pytest.raises(UserDoesNotHaveAccessToTaskError): ProcessInstanceService.complete_form_task(processor, spiff_task, {}, initiator_user, human_task) @@ -198,16 +183,12 @@ class TestProcessInstanceProcessor(BaseTest): assert len(human_task.potential_owners) == 1 assert human_task.potential_owners[0] == initiator_user - spiff_task = processor.__class__.get_task_by_bpmn_identifier( - human_task.task_name, processor.bpmn_process_instance - ) + spiff_task = processor.__class__.get_task_by_bpmn_identifier(human_task.task_name, processor.bpmn_process_instance) ProcessInstanceService.complete_form_task(processor, spiff_task, {}, initiator_user, human_task) assert len(process_instance.active_human_tasks) == 1 human_task = process_instance.active_human_tasks[0] - spiff_task = processor.__class__.get_task_by_bpmn_identifier( - human_task.task_name, processor.bpmn_process_instance - ) + spiff_task = processor.__class__.get_task_by_bpmn_identifier(human_task.task_name, processor.bpmn_process_instance) with pytest.raises(UserDoesNotHaveAccessToTaskError): ProcessInstanceService.complete_form_task(processor, spiff_task, {}, initiator_user, human_task) ProcessInstanceService.complete_form_task(processor, spiff_task, {}, testadmin1, human_task) @@ -226,9 +207,7 @@ class TestProcessInstanceProcessor(BaseTest): process_model_id="test_group/call_activity_nested", process_model_source_directory="call_activity_nested", ) - process_instance = self.create_process_instance_from_process_model( - process_model=process_model, user=initiator_user - ) + process_instance = self.create_process_instance_from_process_model(process_model=process_model, user=initiator_user) processor = ProcessInstanceProcessor(process_instance) processor.do_engine_steps(save=True) @@ -236,9 +215,7 @@ class TestProcessInstanceProcessor(BaseTest): processor = ProcessInstanceProcessor(process_instance) # this task will be found within subprocesses - spiff_task = processor.__class__.get_task_by_bpmn_identifier( - "level_3_script_task", processor.bpmn_process_instance - ) + spiff_task = processor.__class__.get_task_by_bpmn_identifier("level_3_script_task", processor.bpmn_process_instance) assert spiff_task is not None assert spiff_task.state == TaskState.COMPLETED @@ -262,9 +239,7 @@ class TestProcessInstanceProcessor(BaseTest): process_model_id="test_group/manual_task", process_model_source_directory="manual_task", ) - process_instance = self.create_process_instance_from_process_model( - process_model=process_model, user=initiator_user - ) + process_instance = self.create_process_instance_from_process_model(process_model=process_model, user=initiator_user) processor = ProcessInstanceProcessor(process_instance) processor.do_engine_steps(save=True) assert len(process_instance.active_human_tasks) == 1 @@ -316,9 +291,7 @@ class TestProcessInstanceProcessor(BaseTest): process_model_id="test_group/manual_task_with_subprocesses", process_model_source_directory="manual_task_with_subprocesses", ) - process_instance = self.create_process_instance_from_process_model( - process_model=process_model, user=initiator_user - ) + process_instance = self.create_process_instance_from_process_model(process_model=process_model, user=initiator_user) processor = ProcessInstanceProcessor(process_instance) processor.do_engine_steps(save=True) assert len(process_instance.active_human_tasks) == 1 @@ -333,9 +306,7 @@ class TestProcessInstanceProcessor(BaseTest): ProcessInstanceService.complete_form_task(processor, spiff_manual_task, {}, initiator_user, human_task_one) assert len(process_instance.human_tasks) == 2, "expected 2 human tasks after first one is completed" - assert ( - len(process_instance.active_human_tasks) == 1 - ), "expected 1 active human tasks after 1st one is completed" + assert len(process_instance.active_human_tasks) == 1, "expected 1 active human tasks after 1st one is completed" # unnecessary lookup just in case on windows process_instance = ProcessInstanceModel.query.filter_by(id=process_instance.id).first() @@ -366,9 +337,7 @@ class TestProcessInstanceProcessor(BaseTest): # make sure sqlalchemy session matches current db state db.session.expire_all() - assert ( - len(process_instance.human_tasks) == 2 - ), "still expected 3 human tasks after reset and session expire_all" + assert len(process_instance.human_tasks) == 2, "still expected 3 human tasks after reset and session expire_all" process_instance = ProcessInstanceModel.query.filter_by(id=process_instance.id).first() processor = ProcessInstanceProcessor(process_instance) @@ -386,9 +355,7 @@ class TestProcessInstanceProcessor(BaseTest): ready_or_waiting_tasks = processor.get_all_ready_or_waiting_tasks() assert len(ready_or_waiting_tasks) == 2 ready_or_waiting_task_identifiers = [t.task_spec.name for t in ready_or_waiting_tasks] - assert sorted(["top_level_subprocess_script", "top_level_subprocess"]) == sorted( - ready_or_waiting_task_identifiers - ) + assert sorted(["top_level_subprocess_script", "top_level_subprocess"]) == sorted(ready_or_waiting_task_identifiers) processor.do_engine_steps(save=True, execution_strategy_name="greedy") ready_or_waiting_tasks = processor.get_all_ready_or_waiting_tasks() @@ -437,9 +404,7 @@ class TestProcessInstanceProcessor(BaseTest): ProcessInstanceService.complete_form_task( processor, spiff_manual_task, {}, process_instance.process_initiator, human_task_one ) - assert ( - len(process_instance.active_human_tasks) == 1 - ), "expected 1 active human tasks after 2nd one is completed" + assert len(process_instance.active_human_tasks) == 1, "expected 1 active human tasks after 2nd one is completed" assert process_instance.active_human_tasks[0].task_title == "Final" # Reset the process back to the task within the call activity that contains a timer_boundary event. @@ -453,9 +418,7 @@ class TestProcessInstanceProcessor(BaseTest): human_task_one = process_instance.active_human_tasks[0] assert human_task_one.task_title == "Manual Task #1" processor = ProcessInstanceProcessor(process_instance) - processor.manual_complete_task( - str(human_task_one.task_id), execute=True, user=process_instance.process_initiator - ) + processor.manual_complete_task(str(human_task_one.task_id), execute=True, user=process_instance.process_initiator) processor = ProcessInstanceProcessor(process_instance) processor.resume() processor.do_engine_steps(save=True) @@ -489,9 +452,7 @@ class TestProcessInstanceProcessor(BaseTest): assert len(process_instance.active_human_tasks) == 1 human_task_one = process_instance.active_human_tasks[0] processor.bpmn_process_instance.get_task_from_id(UUID(human_task_one.task_id)) - processor.manual_complete_task( - str(human_task_one.task_id), execute=True, user=process_instance.process_initiator - ) + processor.manual_complete_task(str(human_task_one.task_id), execute=True, user=process_instance.process_initiator) processor.save() processor = ProcessInstanceProcessor(process_instance) step1_task = processor.get_task_by_bpmn_identifier("step_1", processor.bpmn_process_instance) @@ -534,9 +495,7 @@ class TestProcessInstanceProcessor(BaseTest): process_model_id="test_group/manual_task_with_subprocesses", process_model_source_directory="manual_task_with_subprocesses", ) - process_instance = self.create_process_instance_from_process_model( - process_model=process_model, user=initiator_user - ) + process_instance = self.create_process_instance_from_process_model(process_model=process_model, user=initiator_user) processor = ProcessInstanceProcessor(process_instance) processor.do_engine_steps(save=True, execution_strategy_name="greedy") assert len(process_instance.active_human_tasks) == 1 @@ -646,8 +605,7 @@ class TestProcessInstanceProcessor(BaseTest): expected_python_env_data = expected_task_data[expected_task_data_key]["data"] base_failure_message = ( - f"Failed on {bpmn_process_identifier} - {spiff_task_identifier} - task data key" - f" {expected_task_data_key}." + f"Failed on {bpmn_process_identifier} - {spiff_task_identifier} - task data key {expected_task_data_key}." ) count_failure_message = ( @@ -672,9 +630,7 @@ class TestProcessInstanceProcessor(BaseTest): task_definition = task_model.task_definition assert task_definition.bpmn_identifier == spiff_task_identifier assert task_definition.bpmn_name == spiff_task_identifier.replace("_", " ").title() - assert ( - task_definition.bpmn_process_definition.bpmn_identifier == bpmn_process_identifier - ), base_failure_message + assert task_definition.bpmn_process_definition.bpmn_identifier == bpmn_process_identifier, base_failure_message message = ( f"{base_failure_message} Expected: {sorted(expected_python_env_data)}. Received:" @@ -716,9 +672,7 @@ class TestProcessInstanceProcessor(BaseTest): assert bpmn_process_definition is not None assert bpmn_process_definition.bpmn_identifier == "test_process_to_call_subprocess" assert bpmn_process.direct_parent_process_id is not None - direct_parent_process = BpmnProcessModel.query.filter_by( - id=bpmn_process.direct_parent_process_id - ).first() + direct_parent_process = BpmnProcessModel.query.filter_by(id=bpmn_process.direct_parent_process_id).first() assert direct_parent_process is not None assert direct_parent_process.bpmn_process_definition.bpmn_identifier == "test_process_to_call" spiff_tasks_checked.append(spiff_task.task_spec.name) @@ -764,9 +718,7 @@ class TestProcessInstanceProcessor(BaseTest): bpmn_file_name="lanes_with_owner_dict.bpmn", process_model_source_directory="model_with_lanes", ) - process_instance = self.create_process_instance_from_process_model( - process_model=process_model, user=initiator_user - ) + process_instance = self.create_process_instance_from_process_model(process_model=process_model, user=initiator_user) processor = ProcessInstanceProcessor(process_instance) processor.do_engine_steps(save=True) assert len(process_instance.active_human_tasks) == 1 @@ -790,9 +742,7 @@ class TestProcessInstanceProcessor(BaseTest): bpmn_file_name="loopback.bpmn", process_model_source_directory="loopback_to_manual_task", ) - process_instance = self.create_process_instance_from_process_model( - process_model=process_model, user=initiator_user - ) + process_instance = self.create_process_instance_from_process_model(process_model=process_model, user=initiator_user) processor = ProcessInstanceProcessor(process_instance) processor.do_engine_steps(save=True) @@ -800,9 +750,7 @@ class TestProcessInstanceProcessor(BaseTest): assert len(process_instance.human_tasks) == 1 human_task_one = process_instance.active_human_tasks[0] - spiff_task = processor.__class__.get_task_by_bpmn_identifier( - human_task_one.task_name, processor.bpmn_process_instance - ) + spiff_task = processor.__class__.get_task_by_bpmn_identifier(human_task_one.task_name, processor.bpmn_process_instance) ProcessInstanceService.complete_form_task(processor, spiff_task, {}, initiator_user, human_task_one) assert len(process_instance.active_human_tasks) == 1 @@ -822,9 +770,7 @@ class TestProcessInstanceProcessor(BaseTest): process_model_id="test_group/loopback_to_subprocess", process_model_source_directory="loopback_to_subprocess", ) - process_instance = self.create_process_instance_from_process_model( - process_model=process_model, user=initiator_user - ) + process_instance = self.create_process_instance_from_process_model(process_model=process_model, user=initiator_user) processor = ProcessInstanceProcessor(process_instance) processor.do_engine_steps(save=True, execution_strategy_name="greedy") @@ -872,9 +818,7 @@ class TestProcessInstanceProcessor(BaseTest): process_instance_final = ProcessInstanceModel.query.filter_by(id=process_instance.id).first() processor_final = ProcessInstanceProcessor(process_instance_final) - spiff_task = processor_final.get_task_by_bpmn_identifier( - "script_task_two", processor_final.bpmn_process_instance - ) + spiff_task = processor_final.get_task_by_bpmn_identifier("script_task_two", processor_final.bpmn_process_instance) assert spiff_task is not None task_model = TaskModel.query.filter_by(guid=str(spiff_task.id)).first() assert task_model is not None @@ -883,9 +827,7 @@ class TestProcessInstanceProcessor(BaseTest): process_instance_events = process_instance.process_instance_events assert len(process_instance_events) == 4 - error_events = [ - e for e in process_instance_events if e.event_type == ProcessInstanceEventType.task_failed.value - ] + error_events = [e for e in process_instance_events if e.event_type == ProcessInstanceEventType.task_failed.value] assert len(error_events) == 1 error_event = error_events[0] assert error_event.task_guid is not None @@ -919,3 +861,45 @@ class TestProcessInstanceProcessor(BaseTest): ProcessInstanceService.complete_form_task( processor, spiff_manual_task, {}, process_instance.process_initiator, human_task_one ) + + def test_can_store_instructions_for_end_user( + self, + app: Flask, + client: FlaskClient, + with_db_and_bpmn_file_cleanup: None, + ) -> None: + process_model = load_test_spec( + process_model_id="test_group/script_task_with_instruction", + bpmn_file_name="script_task_with_instruction.bpmn", + process_model_source_directory="script-task-with-instruction", + ) + process_instance = self.create_process_instance_from_process_model(process_model=process_model) + + processor = ProcessInstanceProcessor(process_instance) + processor.do_engine_steps(save=True, execution_strategy_name="queue_instructions_for_end_user") + user_instructions = TaskInstructionsForEndUserModel.entries_for_process_instance(process_instance.id) + assert len(user_instructions) == 1 + assert user_instructions[0].instruction == "We run script one" + processor.do_engine_steps(execution_strategy_name="run_current_ready_tasks") + + processor.do_engine_steps(save=True, execution_strategy_name="queue_instructions_for_end_user") + user_instructions = TaskInstructionsForEndUserModel.entries_for_process_instance(process_instance.id) + assert len(user_instructions) == 2 + # ensure ordering is correct + assert user_instructions[0].instruction == "We run script two" + + assert process_instance.status == ProcessInstanceStatus.running.value + processor.do_engine_steps(execution_strategy_name="run_current_ready_tasks") + assert process_instance.status == ProcessInstanceStatus.running.value + processor.do_engine_steps(save=True, execution_strategy_name="queue_instructions_for_end_user") + assert process_instance.status == ProcessInstanceStatus.complete.value + + remaining_entries = TaskInstructionsForEndUserModel.query.all() + assert len(remaining_entries) == 2 + user_instruction_list = TaskInstructionsForEndUserModel.retrieve_and_clear(process_instance.id) + user_instruction_strings = [ui.instruction for ui in user_instruction_list] + assert user_instruction_strings == ["We run script two", "We run script one"] + remaining_entries = TaskInstructionsForEndUserModel.query.all() + assert len(remaining_entries) == 2 + for entry in remaining_entries: + assert entry.has_been_retrieved is True diff --git a/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_process_instance_queue_service.py b/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_process_instance_queue_service.py index ed174a31..e844351c 100644 --- a/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_process_instance_queue_service.py +++ b/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_process_instance_queue_service.py @@ -18,9 +18,7 @@ class TestProcessInstanceQueueService(BaseTest): bpmn_file_name="lanes.bpmn", process_model_source_directory="model_with_lanes", ) - process_instance = self.create_process_instance_from_process_model( - process_model=process_model, user=initiator_user - ) + process_instance = self.create_process_instance_from_process_model(process_model=process_model, user=initiator_user) return process_instance def test_newly_created_process_instances_are_not_locked_when_added_to_the_queue( diff --git a/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_process_instance_report_service.py b/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_process_instance_report_service.py index eb44580b..9db787a1 100644 --- a/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_process_instance_report_service.py +++ b/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_process_instance_report_service.py @@ -340,9 +340,7 @@ class TestProcessInstanceReportService(BaseTest): process_instance_report = ProcessInstanceReportService.report_with_identifier(user=user_one) report_metadata = process_instance_report.report_metadata - report_metadata["filter_by"].append( - {"field_name": "with_relation_to_me", "field_value": True, "operator": "equals"} - ) + report_metadata["filter_by"].append({"field_name": "with_relation_to_me", "field_value": True, "operator": "equals"}) response_json = ProcessInstanceReportService.run_process_instance_report( report_metadata=report_metadata, user=user_one, diff --git a/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_script_unit_test_runner.py b/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_script_unit_test_runner.py index 06e0a5a9..b91d5d0f 100644 --- a/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_script_unit_test_runner.py +++ b/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_script_unit_test_runner.py @@ -25,9 +25,7 @@ class TestScriptUnitTestRunner(BaseTest): bpmn_file_name=process_model_id, process_model_source_directory=process_model_id, ) - bpmn_process_instance = ProcessInstanceProcessor.get_bpmn_process_instance_from_process_model( - process_model_identifier - ) + bpmn_process_instance = ProcessInstanceProcessor.get_bpmn_process_instance_from_process_model(process_model_identifier) task = ProcessInstanceProcessor.get_task_by_bpmn_identifier("Activity_CalculateNewData", bpmn_process_instance) assert task is not None @@ -58,9 +56,7 @@ class TestScriptUnitTestRunner(BaseTest): bpmn_file_name=process_model_id, process_model_source_directory=process_model_id, ) - bpmn_process_instance = ProcessInstanceProcessor.get_bpmn_process_instance_from_process_model( - process_model_identifier - ) + bpmn_process_instance = ProcessInstanceProcessor.get_bpmn_process_instance_from_process_model(process_model_identifier) task = ProcessInstanceProcessor.get_task_by_bpmn_identifier("Activity_CalculateNewData", bpmn_process_instance) assert task is not None @@ -91,9 +87,7 @@ class TestScriptUnitTestRunner(BaseTest): bpmn_file_name=process_model_id, process_model_source_directory=process_model_id, ) - bpmn_process_instance = ProcessInstanceProcessor.get_bpmn_process_instance_from_process_model( - process_model_identifier - ) + bpmn_process_instance = ProcessInstanceProcessor.get_bpmn_process_instance_from_process_model(process_model_identifier) task = ProcessInstanceProcessor.get_task_by_bpmn_identifier("script_with_unit_test_id", bpmn_process_instance) assert task is not None @@ -121,9 +115,7 @@ class TestScriptUnitTestRunner(BaseTest): bpmn_file_name=process_model_id, process_model_source_directory=process_model_id, ) - bpmn_process_instance = ProcessInstanceProcessor.get_bpmn_process_instance_from_process_model( - process_model_identifier - ) + bpmn_process_instance = ProcessInstanceProcessor.get_bpmn_process_instance_from_process_model(process_model_identifier) task = ProcessInstanceProcessor.get_task_by_bpmn_identifier("script_with_unit_test_id", bpmn_process_instance) assert task is not None diff --git a/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_service_task_delegate.py b/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_service_task_delegate.py index 8429fa7e..a9bf4d64 100644 --- a/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_service_task_delegate.py +++ b/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_service_task_delegate.py @@ -39,9 +39,7 @@ class TestServiceTaskDelegate(BaseTest): def test_check_prefixes_with_spiff_secret_in_dict(self, app: Flask, with_db_and_bpmn_file_cleanup: None) -> None: user = self.find_or_create_user("test_user") SecretService().add_secret("hot_secret", "my_secret_value", user.id) - result = ServiceTaskDelegate.value_with_secrets_replaced( - {"Authorization": "TOKEN SPIFF_SECRET:hot_secret-haha"} - ) + result = ServiceTaskDelegate.value_with_secrets_replaced({"Authorization": "TOKEN SPIFF_SECRET:hot_secret-haha"}) assert result == {"Authorization": "TOKEN my_secret_value-haha"} def test_invalid_call_returns_good_error_message(self, app: Flask, with_db_and_bpmn_file_cleanup: None) -> None: @@ -171,13 +169,9 @@ class TestServiceTaskDelegate(BaseTest): **{"operator_identifier": "my_operation"}, } - def _assert_error_with_code( - self, response_text: str, error_code: str, contains_message: str, status_code: int - ) -> None: + def _assert_error_with_code(self, response_text: str, error_code: str, contains_message: str, status_code: int) -> None: assert f"'{error_code}'" in response_text assert bool( re.search(rf"\b{contains_message}\b", response_text) ), f"Expected to find '{contains_message}' in: {response_text}" - assert bool( - re.search(rf"\b{status_code}\b", response_text) - ), f"Expected to find '{status_code}' in: {response_text}" + assert bool(re.search(rf"\b{status_code}\b", response_text)), f"Expected to find '{status_code}' in: {response_text}" diff --git a/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_task_service.py b/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_task_service.py index 3c15b69d..d69656c7 100644 --- a/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_task_service.py +++ b/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_task_service.py @@ -153,9 +153,7 @@ class TestTaskService(BaseTest): assert bpmn_process.bpmn_process_definition.bpmn_identifier == "Level2b" task_model_level_3 = ( - TaskModel.query.join(TaskDefinitionModel) - .filter(TaskDefinitionModel.bpmn_identifier == "level_3_script_task") - .first() + TaskModel.query.join(TaskDefinitionModel).filter(TaskDefinitionModel.bpmn_identifier == "level_3_script_task").first() ) assert task_model_level_3 is not None bpmn_process = TaskService.bpmn_process_for_called_activity_or_top_level_process(task_model_level_3) diff --git a/spiffworkflow-frontend/cypress/support/commands.js b/spiffworkflow-frontend/cypress/support/commands.js index 02e21019..6143b9ab 100644 --- a/spiffworkflow-frontend/cypress/support/commands.js +++ b/spiffworkflow-frontend/cypress/support/commands.js @@ -115,11 +115,9 @@ Cypress.Commands.add( cy.url().should('include', `/tasks/`); cy.contains('Task: ', { timeout: 30000 }); } else { - cy.url().should('include', `/interstitial`); - // cy.contains('Status: Completed'); - cy.contains( - 'There are no additional instructions or information for this task.' - ); + cy.url().should('include', `/process-instances`); + cy.contains('Process Instance Id'); + cy.contains('complete'); if (returnToProcessModelShow) { cy.getBySel('process-model-breadcrumb-link').click(); cy.getBySel('process-model-show-permissions-loaded').should('exist'); diff --git a/spiffworkflow-frontend/src/components/ErrorDisplay.tsx b/spiffworkflow-frontend/src/components/ErrorDisplay.tsx index 7a2d3acb..d81099de 100644 --- a/spiffworkflow-frontend/src/components/ErrorDisplay.tsx +++ b/spiffworkflow-frontend/src/components/ErrorDisplay.tsx @@ -129,20 +129,33 @@ export const childrenForErrorObject = (errorObject: ErrorForDisplay) => { ]; }; +export function ErrorDisplayStateless( + errorObject: ErrorForDisplay, + onClose?: Function +) { + const title = 'Error:'; + const hideCloseButton = !onClose; + console.log('hideCloseButton', hideCloseButton); + + return ( + + <>{childrenForErrorObject(errorObject)} + + ); +} + export default function ErrorDisplay() { const errorObject = useAPIError().error; const { removeError } = useAPIError(); let errorTag = null; if (errorObject) { - const title = 'Error:'; - - errorTag = ( - removeError()} type="error"> - <>{childrenForErrorObject(errorObject)} - - ); + errorTag = ErrorDisplayStateless(errorObject, removeError); } - return errorTag; } diff --git a/spiffworkflow-frontend/src/components/InstructionsForEndUser.tsx b/spiffworkflow-frontend/src/components/InstructionsForEndUser.tsx index 644a7cc8..a3c0e462 100644 --- a/spiffworkflow-frontend/src/components/InstructionsForEndUser.tsx +++ b/spiffworkflow-frontend/src/components/InstructionsForEndUser.tsx @@ -3,28 +3,43 @@ import React, { useEffect, useState } from 'react'; import { Toggle } from '@carbon/react'; import FormattingService from '../services/FormattingService'; import MarkdownRenderer from './MarkdownRenderer'; +import { + BasicTask, + ProcessInstanceTask, + TaskInstructionForEndUser, +} from '../interfaces'; type OwnProps = { - task: any; + task?: BasicTask | ProcessInstanceTask | null; + taskInstructionForEndUser?: TaskInstructionForEndUser; defaultMessage?: string; allowCollapse?: boolean; }; export default function InstructionsForEndUser({ task, + taskInstructionForEndUser, defaultMessage = '', allowCollapse = false, }: OwnProps) { const [collapsed, setCollapsed] = useState(false); const [collapsable, setCollapsable] = useState(false); let instructions = defaultMessage; - let { properties } = task; - if (!properties) { - properties = task.extensions; - } - const { instructionsForEndUser } = properties; - if (instructionsForEndUser) { - instructions = instructionsForEndUser; + + if (task) { + let properties = null; + if ('properties' in task) { + properties = task.properties; + } + if (!properties && 'extensions' in task) { + properties = task.extensions; + } + const { instructionsForEndUser } = properties; + if (instructionsForEndUser) { + instructions = instructionsForEndUser; + } + } else if (taskInstructionForEndUser) { + instructions = taskInstructionForEndUser.instruction; } instructions = FormattingService.checkForSpiffFormats(instructions); @@ -39,6 +54,7 @@ export default function InstructionsForEndUser({ return arg.split(' ').length; }; + // this is to allow toggling collapsed instructions useEffect(() => { if ( allowCollapse && @@ -53,10 +69,6 @@ export default function InstructionsForEndUser({ } }, [allowCollapse, instructions]); - if (!task) { - return null; - } - const toggleCollapse = () => { setCollapsed(!collapsed); }; @@ -80,17 +92,21 @@ export default function InstructionsForEndUser({ className += ' markdown-collapsed'; } - return ( -
-
- {/* + if (instructions) { + return ( +
+
+ {/* https://www.npmjs.com/package/@uiw/react-md-editor switches to dark mode by default by respecting @media (prefers-color-scheme: dark) This makes it look like our site is broken, so until the rest of the site supports dark mode, turn off dark mode for this component. */} - + +
+ {showCollapseToggle()}
- {showCollapseToggle()} -
- ); + ); + } + + return null; } diff --git a/spiffworkflow-frontend/src/components/ProcessInstanceCurrentTaskInfo.tsx b/spiffworkflow-frontend/src/components/ProcessInstanceCurrentTaskInfo.tsx new file mode 100644 index 00000000..3df3aa2b --- /dev/null +++ b/spiffworkflow-frontend/src/components/ProcessInstanceCurrentTaskInfo.tsx @@ -0,0 +1,113 @@ +import React, { useEffect, useState } from 'react'; +import { InlineNotification } from '@carbon/react'; + +import InstructionsForEndUser from './InstructionsForEndUser'; +import { ProcessInstance, ProcessInstanceTask } from '../interfaces'; +import { HUMAN_TASK_TYPES } from '../helpers'; +import HttpService from '../services/HttpService'; + +type OwnProps = { + processInstance: ProcessInstance; +}; + +export default function ProcessInstanceCurrentTaskInfo({ + processInstance, +}: OwnProps) { + const [taskResult, setTaskResult] = useState(null); + const [task, setTask] = useState(null); + + useEffect(() => { + const processTaskResult = (result: any) => { + setTaskResult(result); + setTask(result.task); + }; + HttpService.makeCallToBackend({ + path: `/tasks/${processInstance.id}/instruction`, + successCallback: processTaskResult, + }); + }, [processInstance]); + + const inlineMessage = ( + title: string, + subtitle: string, + kind: string = 'info' + ) => { + return ( +
+ +
+ ); + }; + + const taskUserMessage = () => { + if (!task) { + return null; + } + + if (!task.can_complete && HUMAN_TASK_TYPES.includes(task.type)) { + let message = 'This next task is assigned to a different person or team.'; + if (task.assigned_user_group_identifier) { + message = `This next task is assigned to group: ${task.assigned_user_group_identifier}.`; + } else if (task.potential_owner_usernames) { + let potentialOwnerArray = task.potential_owner_usernames.split(','); + if (potentialOwnerArray.length > 2) { + potentialOwnerArray = potentialOwnerArray.slice(0, 2).concat(['...']); + } + message = `This next task is assigned to user(s): ${potentialOwnerArray.join( + ', ' + )}.`; + } + + return inlineMessage( + '', + `${message} There is no action for you to take at this time.` + ); + } + if (task && task.can_complete && HUMAN_TASK_TYPES.includes(task.type)) { + return null; + } + return ( +
+ +
+ ); + }; + + const userMessage = () => { + if (!processInstance) { + return null; + } + if (['terminated', 'suspended'].includes(processInstance.status)) { + return inlineMessage( + `Process ${processInstance.status}`, + `This process instance was ${processInstance.status} by an administrator. Please get in touch with them for more information.`, + 'warning' + ); + } + if (processInstance.status === 'error') { + let errMessage = `This process instance experienced an unexpected error and cannot continue. Please get in touch with an administrator for more information and next steps.`; + if (task && task.error_message) { + errMessage = ` ${errMessage.concat(task.error_message)}`; + } + return inlineMessage(`Process Error`, errMessage, 'error'); + } + + if (task) { + return taskUserMessage(); + } + const defaultMsg = + 'There are no additional instructions or information for this process.'; + return inlineMessage(`Process Error`, defaultMsg, 'info'); + }; + + if (processInstance && taskResult) { + return
{userMessage()}
; + } + + return null; +} diff --git a/spiffworkflow-frontend/src/components/ProcessInstanceListTable.tsx b/spiffworkflow-frontend/src/components/ProcessInstanceListTable.tsx index 208362d7..599eab4e 100644 --- a/spiffworkflow-frontend/src/components/ProcessInstanceListTable.tsx +++ b/spiffworkflow-frontend/src/components/ProcessInstanceListTable.tsx @@ -1783,7 +1783,6 @@ export default function ProcessInstanceListTable({ ) { hasAccessToCompleteTask = true; } - buttonElement = null; if (hasAccessToCompleteTask && processInstance.task_id) { buttonElement = (