From e228ab4b80138b3e943b11774c1fd47957ea7562 Mon Sep 17 00:00:00 2001 From: jasquat Date: Tue, 25 Oct 2022 16:55:11 -0400 Subject: [PATCH] Squashed 'spiffworkflow-backend/' changes from 1092ed111..ab9b5f368 ab9b5f368 Merge pull request #157 from sartography/feature/background_or_not_scheduler ba002f8a4 do not preload gunicorn w/ burnettk 8516154cf pyl now passes w/ burnettk dd0566b1a added script to start scheduler in its own process w/ burnettk danfunk 7d66f922f rename environment variable to something sensible. git-subtree-dir: spiffworkflow-backend git-subtree-split: ab9b5f368a0b954370e43936a88b2ce9b83dc4cc --- bin/boot_server_in_docker | 1 - bin/run_server_locally | 3 --- bin/start_blocking_appscheduler.py | 22 +++++++++++++++++++ bin/wait_for_db_to_be_ready.py | 22 ++++--------------- docker-compose.yml | 2 +- src/spiffworkflow_backend/__init__.py | 11 +++++----- src/spiffworkflow_backend/config/default.py | 4 ++-- src/spiffworkflow_backend/config/demo.py | 4 ++++ .../config/development.py | 4 ++++ .../helpers/db_helper.py | 16 ++++++++++++++ .../helpers/fixture_data.py | 1 - .../routes/process_api_blueprint.py | 2 +- .../services/process_instance_service.py | 6 +++-- .../services/secret_service.py | 8 ++----- .../services/service_task_service.py | 10 +++------ 15 files changed, 69 insertions(+), 47 deletions(-) create mode 100755 bin/start_blocking_appscheduler.py create mode 100644 src/spiffworkflow_backend/helpers/db_helper.py delete mode 100644 src/spiffworkflow_backend/helpers/fixture_data.py diff --git a/bin/boot_server_in_docker b/bin/boot_server_in_docker index cbce26fa1..00c6b5d96 100755 --- a/bin/boot_server_in_docker +++ b/bin/boot_server_in_docker @@ -48,7 +48,6 @@ if [[ "${SPIFFWORKFLOW_BACKEND_RUN_DATA_SETUP:-}" != "false" ]]; then fi export IS_GUNICORN="true" -export PROCESS_WAITING_MESSAGES="true" # THIS MUST BE THE LAST COMMAND! exec poetry run gunicorn ${additional_args} --bind "0.0.0.0:$port" --workers="$workers" --limit-request-line 8192 --timeout 90 --capture-output --access-logfile '-' --log-level debug wsgi:app diff --git a/bin/run_server_locally b/bin/run_server_locally index af51c8153..f06acc9a9 100755 --- a/bin/run_server_locally +++ b/bin/run_server_locally @@ -28,9 +28,6 @@ export APPLICATION_ROOT="/" if [[ -n "${SPIFFWORKFLOW_BACKEND_LOAD_FIXTURE_DATA:-}" ]]; then ./bin/boot_server_in_docker else - if [[ -z "${PROCESS_WAITING_MESSAGES:-}" ]]; then - export PROCESS_WAITING_MESSAGES="true" - fi export FLASK_DEBUG=1 if [[ "${SPIFFWORKFLOW_BACKEND_RUN_DATA_SETUP:-}" != "false" ]]; then diff --git a/bin/start_blocking_appscheduler.py b/bin/start_blocking_appscheduler.py new file mode 100755 index 000000000..4af99e411 --- /dev/null +++ b/bin/start_blocking_appscheduler.py @@ -0,0 +1,22 @@ +"""Start the appscheduler in blocking mode.""" +import time + +from apscheduler.schedulers.background import BlockingScheduler # type: ignore + +from spiffworkflow_backend import create_app +from spiffworkflow_backend import start_scheduler +from spiffworkflow_backend.helpers.db_helper import try_to_connect + + +def main() -> None: + """Main.""" + app = create_app() + start_time = time.time() + with app.app_context(): + try_to_connect(start_time) + + start_scheduler(app, BlockingScheduler) + + +if __name__ == "__main__": + main() diff --git a/bin/wait_for_db_to_be_ready.py b/bin/wait_for_db_to_be_ready.py index 40497ae94..e94532c3b 100644 --- a/bin/wait_for_db_to_be_ready.py +++ b/bin/wait_for_db_to_be_ready.py @@ -1,27 +1,13 @@ -"""Grabs tickets from csv and makes process instances.""" +"""Wait for db to be ready.""" import time -import sqlalchemy -from flask_bpmn.models.db import db - -from spiffworkflow_backend import get_hacked_up_app_for_script - - -def try_to_connect(start_time: float) -> None: - """Try to connect.""" - try: - db.first_or_404("select 1") - except sqlalchemy.exc.DatabaseError as exception: - if time.time() - start_time > 15: - raise exception - else: - time.sleep(1) - try_to_connect(start_time) +from spiffworkflow_backend import create_app +from spiffworkflow_backend.helpers.db_helper import try_to_connect def main() -> None: """Main.""" - app = get_hacked_up_app_for_script() + app = create_app() start_time = time.time() with app.app_context(): try_to_connect(start_time) diff --git a/docker-compose.yml b/docker-compose.yml index c7075b2bd..12a0936fd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -62,7 +62,7 @@ services: - SPIFFWORKFLOW_BACKEND_DATABASE_URI=mysql+mysqlconnector://root:${SPIFFWORKFLOW_BACKEND_MYSQL_ROOT_DATABASE:-my-secret-pw}@localhost:7003/${SPIFFWORKFLOW_BACKEND_DATABASE_NAME:-spiffworkflow_backend_development} - BPMN_SPEC_ABSOLUTE_DIR=/app/process_models - SPIFFWORKFLOW_BACKEND_LOAD_FIXTURE_DATA=${SPIFFWORKFLOW_BACKEND_LOAD_FIXTURE_DATA:-false} - - PROCESS_WAITING_MESSAGES=true + - RUN_BACKGROUND_SCHEDULER=true ports: - "7000:7000" network_mode: host diff --git a/src/spiffworkflow_backend/__init__.py b/src/spiffworkflow_backend/__init__.py index 9a0903305..6a8767234 100644 --- a/src/spiffworkflow_backend/__init__.py +++ b/src/spiffworkflow_backend/__init__.py @@ -7,6 +7,7 @@ import flask.app import flask.json import sqlalchemy from apscheduler.schedulers.background import BackgroundScheduler # type: ignore +from apscheduler.schedulers.base import BaseScheduler # type: ignore from flask.json.provider import DefaultJSONProvider from flask_bpmn.api.api_error import api_error_blueprint from flask_bpmn.models.db import db @@ -52,9 +53,11 @@ class MyJSONEncoder(DefaultJSONProvider): return super().dumps(obj, **kwargs) -def start_scheduler(app: flask.app.Flask) -> None: +def start_scheduler( + app: flask.app.Flask, scheduler_class: BaseScheduler = BackgroundScheduler +) -> None: """Start_scheduler.""" - scheduler = BackgroundScheduler() + scheduler = scheduler_class() scheduler.add_job( BackgroundProcessingService(app).process_message_instances_with_app_context, "interval", @@ -111,7 +114,7 @@ def create_app() -> flask.app.Flask: app.json = MyJSONEncoder(app) - if app.config["PROCESS_WAITING_MESSAGES"]: + if app.config["RUN_BACKGROUND_SCHEDULER"]: start_scheduler(app) configure_sentry(app) @@ -137,8 +140,6 @@ def get_hacked_up_app_for_script() -> flask.app.Flask: else: raise Exception(f"Could not find {full_process_model_path}") app = create_app() - setup_config(app) - configure_sentry(app) return app diff --git a/src/spiffworkflow_backend/config/default.py b/src/spiffworkflow_backend/config/default.py index 917fcdd84..b108eccbb 100644 --- a/src/spiffworkflow_backend/config/default.py +++ b/src/spiffworkflow_backend/config/default.py @@ -13,8 +13,8 @@ CORS_ALLOW_ORIGINS = re.split( r",\s*", environ.get("CORS_ALLOW_ORIGINS", default=CORS_DEFAULT) ) -PROCESS_WAITING_MESSAGES = ( - environ.get("PROCESS_WAITING_MESSAGES", default="false") == "true" +RUN_BACKGROUND_SCHEDULER = ( + environ.get("RUN_BACKGROUND_SCHEDULER", default="false") == "true" ) SPIFFWORKFLOW_FRONTEND_URL = environ.get( "SPIFFWORKFLOW_FRONTEND_URL", default="http://localhost:7001" diff --git a/src/spiffworkflow_backend/config/demo.py b/src/spiffworkflow_backend/config/demo.py index 78e5dd201..a09262a6b 100644 --- a/src/spiffworkflow_backend/config/demo.py +++ b/src/spiffworkflow_backend/config/demo.py @@ -7,3 +7,7 @@ GIT_COMMIT_EMAIL = "demo@example.com" SPIFFWORKFLOW_BACKEND_PERMISSIONS_FILE_NAME = environ.get( "SPIFFWORKFLOW_BACKEND_PERMISSIONS_FILE_NAME", default="demo.yml" ) + +RUN_BACKGROUND_SCHEDULER = ( + environ.get("RUN_BACKGROUND_SCHEDULER", default="false") == "true" +) diff --git a/src/spiffworkflow_backend/config/development.py b/src/spiffworkflow_backend/config/development.py index a7152177d..c3c479460 100644 --- a/src/spiffworkflow_backend/config/development.py +++ b/src/spiffworkflow_backend/config/development.py @@ -8,3 +8,7 @@ SPIFFWORKFLOW_BACKEND_PERMISSIONS_FILE_NAME = environ.get( SPIFFWORKFLOW_BACKEND_LOG_LEVEL = environ.get( "SPIFFWORKFLOW_BACKEND_LOG_LEVEL", default="debug" ) + +RUN_BACKGROUND_SCHEDULER = ( + environ.get("RUN_BACKGROUND_SCHEDULER", default="true") == "true" +) diff --git a/src/spiffworkflow_backend/helpers/db_helper.py b/src/spiffworkflow_backend/helpers/db_helper.py new file mode 100644 index 000000000..647ce4932 --- /dev/null +++ b/src/spiffworkflow_backend/helpers/db_helper.py @@ -0,0 +1,16 @@ +"""Db_helper.""" +import sqlalchemy +from flask_bpmn.models.db import db +import time + + +def try_to_connect(start_time: float) -> None: + """Try to connect.""" + try: + db.first_or_404("select 1") # type: ignore + except sqlalchemy.exc.DatabaseError as exception: + if time.time() - start_time > 15: + raise exception + else: + time.sleep(1) + try_to_connect(start_time) diff --git a/src/spiffworkflow_backend/helpers/fixture_data.py b/src/spiffworkflow_backend/helpers/fixture_data.py deleted file mode 100644 index 3c21b0f93..000000000 --- a/src/spiffworkflow_backend/helpers/fixture_data.py +++ /dev/null @@ -1 +0,0 @@ -"""Fixture_data.""" diff --git a/src/spiffworkflow_backend/routes/process_api_blueprint.py b/src/spiffworkflow_backend/routes/process_api_blueprint.py index b499b7880..3fcd35b0a 100644 --- a/src/spiffworkflow_backend/routes/process_api_blueprint.py +++ b/src/spiffworkflow_backend/routes/process_api_blueprint.py @@ -426,7 +426,7 @@ def process_instance_run( processor.save() ProcessInstanceService.update_task_assignments(processor) - if not current_app.config["PROCESS_WAITING_MESSAGES"]: + if not current_app.config["RUN_BACKGROUND_SCHEDULER"]: MessageService.process_message_instances() process_instance_api = ProcessInstanceService.processor_to_process_instance_api( diff --git a/src/spiffworkflow_backend/services/process_instance_service.py b/src/spiffworkflow_backend/services/process_instance_service.py index 94b2714a8..214dc4f7a 100644 --- a/src/spiffworkflow_backend/services/process_instance_service.py +++ b/src/spiffworkflow_backend/services/process_instance_service.py @@ -73,8 +73,10 @@ class ProcessInstanceService: process_instance.status = ProcessInstanceStatus.erroring.value db.session.add(process_instance) db.session.commit() - error_message = f"Error running waiting task for process_instance {process_instance.id}" + \ - f"({process_instance.process_model_identifier}). {str(e)}" + error_message = ( + f"Error running waiting task for process_instance {process_instance.id}" + + f"({process_instance.process_model_identifier}). {str(e)}" + ) current_app.logger.error(error_message) @staticmethod diff --git a/src/spiffworkflow_backend/services/secret_service.py b/src/spiffworkflow_backend/services/secret_service.py index f18ef7d27..42f401c18 100644 --- a/src/spiffworkflow_backend/services/secret_service.py +++ b/src/spiffworkflow_backend/services/secret_service.py @@ -37,9 +37,7 @@ class SecretService: ) -> SecretModel: """Add_secret.""" # encrypted_key = self.encrypt_key(key) - secret_model = SecretModel( - key=key, value=value, user_id=user_id - ) + secret_model = SecretModel(key=key, value=value, user_id=user_id) db.session.add(secret_model) try: db.session.commit() @@ -81,9 +79,7 @@ class SecretService: db.session.rollback() raise e elif create_if_not_exists: - SecretService.add_secret( - key=key, value=value, user_id=user_id - ) + SecretService.add_secret(key=key, value=value, user_id=user_id) else: raise ApiError( error_code="update_secret_error", diff --git a/src/spiffworkflow_backend/services/service_task_service.py b/src/spiffworkflow_backend/services/service_task_service.py index 0ec8a9cb8..97ce1495a 100644 --- a/src/spiffworkflow_backend/services/service_task_service.py +++ b/src/spiffworkflow_backend/services/service_task_service.py @@ -64,13 +64,9 @@ class ServiceTaskDelegate: return proxied_response.text secret_key = parsed_response["auth"] - refreshed_token_set = json.dumps( - parsed_response["refreshed_token_set"] - ) - SecretService().update_secret( - secret_key, refreshed_token_set, g.user.id - ) - + refreshed_token_set = json.dumps(parsed_response["refreshed_token_set"]) + SecretService().update_secret(secret_key, refreshed_token_set, g.user.id) + return json.dumps(parsed_response["api_response"])