From 893ef56f1ae2cd4af94543547670d5bd9daf15eb Mon Sep 17 00:00:00 2001 From: jasquat <2487833+jasquat@users.noreply.github.com> Date: Thu, 31 Aug 2023 16:09:26 -0400 Subject: [PATCH] feature/fix-run-scheduler-config (#456) * updated scheduler config name to make sense and updated docker-compose file to use the local_docker env * added sample command to boot server in docker * removed terraform configs since they are no longer used * fixed snyk issue --------- Co-authored-by: jasquat --- docker-compose.yml | 5 +- editor.docker-compose.yml | 2 +- .../bin/boot_server_in_docker | 3 ++ spiffworkflow-backend/bin/run_server_locally | 8 ++-- spiffworkflow-backend/docker-compose.yml | 2 +- .../src/spiffworkflow_backend/__init__.py | 4 +- .../spiffworkflow_backend/config/__init__.py | 12 +---- .../spiffworkflow_backend/config/default.py | 4 +- .../config/local_development.py | 4 +- .../config/terraform_deployed_environment.py | 46 ------------------- .../routes/process_instances_controller.py | 2 +- 11 files changed, 21 insertions(+), 71 deletions(-) delete mode 100644 spiffworkflow-backend/src/spiffworkflow_backend/config/terraform_deployed_environment.py diff --git a/docker-compose.yml b/docker-compose.yml index 5e218986..104103ef 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,7 +16,7 @@ services: image: ghcr.io/sartography/spiffworkflow-backend:latest environment: SPIFFWORKFLOW_BACKEND_APPLICATION_ROOT: "/" - SPIFFWORKFLOW_BACKEND_ENV: "local_development" + SPIFFWORKFLOW_BACKEND_ENV: "local_docker" FLASK_DEBUG: "0" FLASK_SESSION_SECRET_KEY: "${FLASK_SESSION_SECRET_KEY:-super_secret_key}" # WARNING: Frontend is a static site which assumes frontend port - 1 on localhost. @@ -27,12 +27,13 @@ services: SPIFFWORKFLOW_BACKEND_DATABASE_TYPE: "sqlite" SPIFFWORKFLOW_BACKEND_DATABASE_URI: "sqlite:////app/db_volume/db.sqlite3" SPIFFWORKFLOW_BACKEND_LOAD_FIXTURE_DATA: "false" + SPIFFWORKFLOW_BACKEND_LOG_LEVEL: "DEBUG" SPIFFWORKFLOW_BACKEND_OPEN_ID_CLIENT_ID: "spiffworkflow-backend" SPIFFWORKFLOW_BACKEND_OPEN_ID_CLIENT_SECRET_KEY: "my_open_id_secret_key" SPIFFWORKFLOW_BACKEND_OPEN_ID_SERVER_URL: "http://localhost:${SPIFF_BACKEND_PORT:-8000}/openid" SPIFFWORKFLOW_BACKEND_PERMISSIONS_FILE_NAME: "example.yml" SPIFFWORKFLOW_BACKEND_PORT: "${SPIFF_BACKEND_PORT:-8000}" - SPIFFWORKFLOW_BACKEND_RUN_BACKGROUND_SCHEDULER: "true" + SPIFFWORKFLOW_BACKEND_RUN_BACKGROUND_SCHEDULER_IN_CREATE_APP: "true" SPIFFWORKFLOW_BACKEND_UPGRADE_DB: "true" SPIFFWORKFLOW_BACKEND_URL_FOR_FRONTEND: "http://localhost:${SPIFFWORKFLOW_FRONTEND_PORT:-8001}" ports: diff --git a/editor.docker-compose.yml b/editor.docker-compose.yml index 2c4232ae..80475b28 100644 --- a/editor.docker-compose.yml +++ b/editor.docker-compose.yml @@ -32,7 +32,7 @@ services: SPIFFWORKFLOW_BACKEND_OPEN_ID_SERVER_URL: "http://localhost:${SPIFF_BACKEND_PORT:-8000}/openid" SPIFFWORKFLOW_BACKEND_PERMISSIONS_FILE_NAME: "example.yml" SPIFFWORKFLOW_BACKEND_PORT: "${SPIFF_BACKEND_PORT:-8000}" - SPIFFWORKFLOW_BACKEND_RUN_BACKGROUND_SCHEDULER: "false" + SPIFFWORKFLOW_BACKEND_RUN_BACKGROUND_SCHEDULER_IN_CREATE_APP: "false" SPIFFWORKFLOW_BACKEND_UPGRADE_DB: "true" SPIFFWORKFLOW_BACKEND_URL_FOR_FRONTEND: "http://localhost:${SPIFFWORKFLOW_FRONTEND_PORT:-8001}" ports: diff --git a/spiffworkflow-backend/bin/boot_server_in_docker b/spiffworkflow-backend/bin/boot_server_in_docker index 2e5c0444..35cd08d6 100755 --- a/spiffworkflow-backend/bin/boot_server_in_docker +++ b/spiffworkflow-backend/bin/boot_server_in_docker @@ -7,6 +7,9 @@ function error_handler() { trap 'error_handler ${LINENO} $?' ERR set -o errtrace -o errexit -o nounset -o pipefail +# example command: +# SPIFFWORKFLOW_BACKEND_PERMISSIONS_FILE_NAME=example.yml SPIFFWORKFLOW_BACKEND_DATABASE_TYPE=sqlite SPIFFWORKFLOW_BACKEND_ENV=local_docker SPIFFWORKFLOW_BACKEND_RUN_BACKGROUND_SCHEDULER_IN_CREATE_APP=true FLASK_DEBUG=0 FLASK_SESSION_SECRET_KEY=HEY SPIFFWORKFLOW_BACKEND_BPMN_SPEC_ABSOLUTE_DIR="${HOME}/projects/github/sartography/sample-process-models/" ./bin/boot_server_in_docker + # run migrations export FLASK_APP=/app/src/spiffworkflow_backend diff --git a/spiffworkflow-backend/bin/run_server_locally b/spiffworkflow-backend/bin/run_server_locally index 6e8482d8..5d5b61e3 100755 --- a/spiffworkflow-backend/bin/run_server_locally +++ b/spiffworkflow-backend/bin/run_server_locally @@ -40,13 +40,13 @@ else export FLASK_DEBUG=1 if [[ "${SPIFFWORKFLOW_BACKEND_RUN_DATA_SETUP:-}" != "false" ]]; then - SPIFFWORKFLOW_BACKEND_RUN_BACKGROUND_SCHEDULER=false SPIFFWORKFLOW_BACKEND_FAIL_ON_INVALID_PROCESS_MODELS=false poetry run python bin/save_all_bpmn.py + SPIFFWORKFLOW_BACKEND_RUN_BACKGROUND_SCHEDULER_IN_CREATE_APP=false SPIFFWORKFLOW_BACKEND_FAIL_ON_INVALID_PROCESS_MODELS=false poetry run python bin/save_all_bpmn.py fi - if [[ -z "${SPIFFWORKFLOW_BACKEND_RUN_BACKGROUND_SCHEDULER:-}" ]]; then - SPIFFWORKFLOW_BACKEND_RUN_BACKGROUND_SCHEDULER=true + if [[ -z "${SPIFFWORKFLOW_BACKEND_RUN_BACKGROUND_SCHEDULER_IN_CREATE_APP:-}" ]]; then + SPIFFWORKFLOW_BACKEND_RUN_BACKGROUND_SCHEDULER_IN_CREATE_APP=true fi # this line blocks - SPIFFWORKFLOW_BACKEND_RUN_BACKGROUND_SCHEDULER="${SPIFFWORKFLOW_BACKEND_RUN_BACKGROUND_SCHEDULER}" FLASK_APP=src/spiffworkflow_backend poetry run flask run -p "$port" --host=0.0.0.0 + SPIFFWORKFLOW_BACKEND_RUN_BACKGROUND_SCHEDULER_IN_CREATE_APP="${SPIFFWORKFLOW_BACKEND_RUN_BACKGROUND_SCHEDULER_IN_CREATE_APP}" FLASK_APP=src/spiffworkflow_backend poetry run flask run -p "$port" --host=0.0.0.0 fi diff --git a/spiffworkflow-backend/docker-compose.yml b/spiffworkflow-backend/docker-compose.yml index 4cecee14..d022a4fd 100644 --- a/spiffworkflow-backend/docker-compose.yml +++ b/spiffworkflow-backend/docker-compose.yml @@ -60,7 +60,7 @@ services: - SPIFFWORKFLOW_BACKEND_OPEN_ID_SERVER_URL=${SPIFFWORKFLOW_BACKEND_OPEN_ID_SERVER_URL:-http://localhost:7002/realms/spiffworkflow} - SPIFFWORKFLOW_BACKEND_PERMISSIONS_FILE_NAME=${SPIFFWORKFLOW_BACKEND_PERMISSIONS_FILE_NAME:-acceptance_tests.yml} - SPIFFWORKFLOW_BACKEND_PORT=7000 - - SPIFFWORKFLOW_BACKEND_RUN_BACKGROUND_SCHEDULER=true + - SPIFFWORKFLOW_BACKEND_RUN_BACKGROUND_SCHEDULER_IN_CREATE_APP=true - SPIFFWORKFLOW_BACKEND_URL_FOR_FRONTEND=${SPIFFWORKFLOW_BACKEND_URL_FOR_FRONTEND:-http://localhost:7001} - SPIFFWORKFLOW_BACKEND_UPGRADE_DB=true - SPIFFWORKFLOW_BACKEND_URL=${SPIFFWORKFLOW_BACKEND_URL:-http://localhost:7000} diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/__init__.py b/spiffworkflow-backend/src/spiffworkflow_backend/__init__.py index dc5b58b2..20d5992f 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/__init__.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/__init__.py @@ -101,11 +101,11 @@ def start_scheduler(app: flask.app.Flask, scheduler_class: BaseScheduler = Backg def should_start_scheduler(app: flask.app.Flask) -> bool: - if not app.config["SPIFFWORKFLOW_BACKEND_RUN_BACKGROUND_SCHEDULER"]: + if not app.config["SPIFFWORKFLOW_BACKEND_RUN_BACKGROUND_SCHEDULER_IN_CREATE_APP"]: return False # do not start the scheduler twice in flask debug mode but support code reloading - if app.config["ENV_IDENTIFIER"] != "local_development" or os.environ.get("WERKZEUG_RUN_MAIN") != "true": + if app.config["ENV_IDENTIFIER"] == "local_development" and os.environ.get("WERKZEUG_RUN_MAIN") == "true": return False return True diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/config/__init__.py b/spiffworkflow-backend/src/spiffworkflow_backend/config/__init__.py index 20fd84bf..a14986bb 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/config/__init__.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/config/__init__.py @@ -146,20 +146,12 @@ def setup_config(app: Flask) -> None: load_config_file(app, "spiffworkflow_backend.config.default") env_config_prefix = "spiffworkflow_backend.config." - if ( - os.environ.get("SPIFFWORKFLOW_BACKEND_TERRAFORM_DEPLOYED_ENVIRONMENT") == "true" - and os.environ.get("SPIFFWORKFLOW_BACKEND_ENV") is not None - ): - load_config_file(app, f"{env_config_prefix}terraform_deployed_environment") - env_config_module = env_config_prefix + app.config["ENV_IDENTIFIER"] load_config_file(app, env_config_module) # This allows config/testing.py or instance/config.py to override the default config - if "ENV_IDENTIFIER" in app.config and app.config["ENV_IDENTIFIER"] == "testing": - app.config.from_pyfile("config/testing.py", silent=True) - elif "ENV_IDENTIFIER" in app.config and app.config["ENV_IDENTIFIER"] == "unit_testing": - app.config.from_pyfile("config/unit_testing.py", silent=True) + if "ENV_IDENTIFIER" in app.config and app.config["ENV_IDENTIFIER"] in ["testing", "unit_testing"]: + app.config.from_pyfile(f"config/{app.config['ENV_IDENTIFIER']}.py", silent=True) else: app.config.from_pyfile(f"{app.instance_path}/config.py", silent=True) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/config/default.py b/spiffworkflow-backend/src/spiffworkflow_backend/config/default.py index 7821bed4..790bd969 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/config/default.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/config/default.py @@ -21,8 +21,8 @@ SPIFFWORKFLOW_BACKEND_CORS_ALLOW_ORIGINS = re.split( environ.get("SPIFFWORKFLOW_BACKEND_CORS_ALLOW_ORIGINS", default=cors_allow_all), ) -SPIFFWORKFLOW_BACKEND_RUN_BACKGROUND_SCHEDULER = ( - environ.get("SPIFFWORKFLOW_BACKEND_RUN_BACKGROUND_SCHEDULER", default="false") == "true" +SPIFFWORKFLOW_BACKEND_RUN_BACKGROUND_SCHEDULER_IN_CREATE_APP = ( + environ.get("SPIFFWORKFLOW_BACKEND_RUN_BACKGROUND_SCHEDULER_IN_CREATE_APP", default="false") == "true" ) SPIFFWORKFLOW_BACKEND_BACKGROUND_SCHEDULER_ALLOW_OPTIMISTIC_CHECKS = ( environ.get("SPIFFWORKFLOW_BACKEND_BACKGROUND_SCHEDULER_ALLOW_OPTIMISTIC_CHECKS", default="true") == "true" diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/config/local_development.py b/spiffworkflow-backend/src/spiffworkflow_backend/config/local_development.py index d1017315..313f67ac 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/config/local_development.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/config/local_development.py @@ -6,8 +6,8 @@ SPIFFWORKFLOW_BACKEND_PERMISSIONS_FILE_NAME = environ.get( SPIFFWORKFLOW_BACKEND_LOG_LEVEL = environ.get("SPIFFWORKFLOW_BACKEND_LOG_LEVEL", default="debug") -SPIFFWORKFLOW_BACKEND_RUN_BACKGROUND_SCHEDULER = ( - environ.get("SPIFFWORKFLOW_BACKEND_RUN_BACKGROUND_SCHEDULER", default="false") == "true" +SPIFFWORKFLOW_BACKEND_RUN_BACKGROUND_SCHEDULER_IN_CREATE_APP = ( + environ.get("SPIFFWORKFLOW_BACKEND_RUN_BACKGROUND_SCHEDULER_IN_CREATE_APP", default="false") == "true" ) SPIFFWORKFLOW_BACKEND_GIT_PUBLISH_CLONE_URL = environ.get( "SPIFFWORKFLOW_BACKEND_GIT_PUBLISH_CLONE_URL", diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/config/terraform_deployed_environment.py b/spiffworkflow-backend/src/spiffworkflow_backend/config/terraform_deployed_environment.py deleted file mode 100644 index e1b85751..00000000 --- a/spiffworkflow-backend/src/spiffworkflow_backend/config/terraform_deployed_environment.py +++ /dev/null @@ -1,46 +0,0 @@ -"""Terraform-deployed environment.""" -from os import environ - -# default.py already ensured that this key existed as was not None -environment_identifier_for_this_config_file_only = environ["SPIFFWORKFLOW_BACKEND_ENV"] - -SPIFFWORKFLOW_BACKEND_GIT_COMMIT_ON_SAVE = True -SPIFFWORKFLOW_BACKEND_GIT_USERNAME = environ.get( - "SPIFFWORKFLOW_BACKEND_GIT_USERNAME", default="sartography-automated-committer" -) -SPIFFWORKFLOW_BACKEND_GIT_USER_EMAIL = environ.get( - "SPIFFWORKFLOW_BACKEND_GIT_USER_EMAIL", - default=f"{SPIFFWORKFLOW_BACKEND_GIT_USERNAME}@users.noreply.github.com", -) -SPIFFWORKFLOW_BACKEND_PERMISSIONS_FILE_NAME = environ.get( - "SPIFFWORKFLOW_BACKEND_PERMISSIONS_FILE_NAME", - default="terraform_deployed_environment.yml", -) - -SPIFFWORKFLOW_BACKEND_RUN_BACKGROUND_SCHEDULER = ( - environ.get("SPIFFWORKFLOW_BACKEND_RUN_BACKGROUND_SCHEDULER", default="false") == "true" -) - -SPIFFWORKFLOW_BACKEND_OPEN_ID_SERVER_URL = environ.get( - "SPIFFWORKFLOW_BACKEND_OPEN_ID_SERVER_URL", - default=( - f"https://keycloak.{environment_identifier_for_this_config_file_only}.spiffworkflow.org/realms/spiffworkflow" - ), -) - -SPIFFWORKFLOW_BACKEND_URL_FOR_FRONTEND = environ.get( - "SPIFFWORKFLOW_BACKEND_URL_FOR_FRONTEND", - default=f"https://{environment_identifier_for_this_config_file_only}.spiffworkflow.org", -) -SPIFFWORKFLOW_BACKEND_URL = environ.get( - "SPIFFWORKFLOW_BACKEND_URL", - default=f"https://api.{environment_identifier_for_this_config_file_only}.spiffworkflow.org", -) -SPIFFWORKFLOW_BACKEND_CONNECTOR_PROXY_URL = environ.get( - "SPIFFWORKFLOW_BACKEND_CONNECTOR_PROXY_URL", - default=f"https://connector-proxy.{environment_identifier_for_this_config_file_only}.spiffworkflow.org", -) -SPIFFWORKFLOW_BACKEND_GIT_PUBLISH_CLONE_URL = environ.get( - "SPIFFWORKFLOW_BACKEND_GIT_PUBLISH_CLONE_URL", - default="https://github.com/sartography/sample-process-models.git", -) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/routes/process_instances_controller.py b/spiffworkflow-backend/src/spiffworkflow_backend/routes/process_instances_controller.py index f3dd8e3a..0ca0d96b 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/routes/process_instances_controller.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/routes/process_instances_controller.py @@ -123,7 +123,7 @@ def _process_instance_run( ) from e raise e - if not current_app.config["SPIFFWORKFLOW_BACKEND_RUN_BACKGROUND_SCHEDULER"]: + if not current_app.config["SPIFFWORKFLOW_BACKEND_RUN_BACKGROUND_SCHEDULER_IN_CREATE_APP"]: MessageService.correlate_all_message_instances() return processor