Merge pull request #130 from sartography/feature/data

data setup on app boot

bin/boot_server_in_docker

@@ -38,7 +38,9 @@ if [[ "${SPIFFWORKFLOW_BACKEND_LOAD_FIXTURE_DATA:-}" == "true" ]]; then
   workers=1
 fi
 
-SPIFFWORKFLOW_BACKEND_FAIL_ON_INVALID_PROCESS_MODELS=false poetry run python bin/save_all_bpmn.py
+if [[ "${SPIFFWORKFLOW_BACKEND_RUN_DATA_SETUP:-}" != "false" ]]; then
+  SPIFFWORKFLOW_BACKEND_FAIL_ON_INVALID_PROCESS_MODELS=false poetry run python bin/save_all_bpmn.py
+fi
 
 export IS_GUNICORN="true"
 export PROCESS_WAITING_MESSAGES="true"

bin/run_server_locally

@@ -32,6 +32,9 @@ else
     export PROCESS_WAITING_MESSAGES="true"
   fi
   export FLASK_DEBUG=1
-  SPIFFWORKFLOW_BACKEND_FAIL_ON_INVALID_PROCESS_MODELS=false poetry run python bin/save_all_bpmn.py
+
+  if [[ "${SPIFFWORKFLOW_BACKEND_RUN_DATA_SETUP:-}" != "false" ]]; then
+    SPIFFWORKFLOW_BACKEND_FAIL_ON_INVALID_PROCESS_MODELS=false poetry run python bin/save_all_bpmn.py
+  fi
   FLASK_APP=src/spiffworkflow_backend poetry run flask run -p 7000
 fi

bin/save_all_bpmn.py

@@ -1,5 +1,6 @@
 """Grabs tickets from csv and makes process instances."""
 import os
+
 from spiffworkflow_backend import get_hacked_up_app_for_script
 from spiffworkflow_backend.services.data_setup_service import DataSetupService
 
@@ -8,10 +9,14 @@ def main() -> None:
     """Main."""
     app = get_hacked_up_app_for_script()
     with app.app_context():
-        failing_process_models = DataSetupService.save_all()
+        failing_process_models = DataSetupService.save_all_process_models()
         for bpmn_errors in failing_process_models:
             print(bpmn_errors)
-        if os.environ.get('SPIFFWORKFLOW_BACKEND_FAIL_ON_INVALID_PROCESS_MODELS') != "false" and len(failing_process_models) > 0:
+        if (
+            os.environ.get("SPIFFWORKFLOW_BACKEND_FAIL_ON_INVALID_PROCESS_MODELS")
+            != "false"
+            and len(failing_process_models) > 0
+        ):
             exit(1)
 
 

src/spiffworkflow_backend/config/default.py

@@ -49,3 +49,7 @@ SPIFFWORKFLOW_BACKEND_PERMISSIONS_FILE_NAME = environ.get(
 # Sentry Configuration
 SENTRY_DSN = environ.get("SENTRY_DSN", default="")
 SENTRY_SAMPLE_RATE = environ.get("SENTRY_SAMPLE_RATE", default="1.0")
+
+SPIFFWORKFLOW_BACKEND_LOG_LEVEL = environ.get(
+    "SPIFFWORKFLOW_BACKEND_LOG_LEVEL", default="info"
+)

src/spiffworkflow_backend/config/development.py

@@ -4,3 +4,7 @@ from os import environ
 SPIFFWORKFLOW_BACKEND_PERMISSIONS_FILE_NAME = environ.get(
     "SPIFFWORKFLOW_BACKEND_PERMISSIONS_FILE_NAME", default="staging.yml"
 )
+
+SPIFFWORKFLOW_BACKEND_LOG_LEVEL = environ.get(
+    "SPIFFWORKFLOW_BACKEND_LOG_LEVEL", default="debug"
+)

src/spiffworkflow_backend/services/acceptance_test_fixtures.py

@@ -2,6 +2,7 @@
 import json
 import time
 
+from flask import current_app
 from flask_bpmn.models.db import db
 from tests.spiffworkflow_backend.helpers.base_test import BaseTest
 
@@ -9,8 +10,9 @@ from spiffworkflow_backend.models.process_instance import ProcessInstanceModel
 from spiffworkflow_backend.models.process_instance import ProcessInstanceStatus
 
 
-def load_fixtures() -> list[ProcessInstanceModel]:
+def load_acceptance_test_fixtures() -> list[ProcessInstanceModel]:
     """Load_fixtures."""
+    current_app.logger.debug("load_acceptance_test_fixtures() start")
     test_process_group_id = "acceptance-tests-group-one"
     test_process_model_id = "acceptance-tests-model-1"
     user = BaseTest.find_or_create_user()
@@ -40,4 +42,5 @@ def load_fixtures() -> list[ProcessInstanceModel]:
         process_instances.append(process_instance)
 
     db.session.commit()
+    current_app.logger.debug("load_acceptance_test_fixtures() end")
     return process_instances

src/spiffworkflow_backend/services/data_setup_service.py

@@ -1,4 +1,6 @@
 """Data_setup_service."""
+from flask import current_app
+
 from spiffworkflow_backend.services.process_model_service import ProcessModelService
 from spiffworkflow_backend.services.spec_file_service import SpecFileService
 
@@ -6,9 +8,15 @@ from spiffworkflow_backend.services.spec_file_service import SpecFileService
 class DataSetupService:
     """DataSetupService."""
 
-    @staticmethod
-    def save_all() -> list:
+    @classmethod
+    def run_setup(cls) -> list:
+        """Run_setup."""
+        return cls.save_all_process_models()
+
+    @classmethod
+    def save_all_process_models(cls) -> list:
         """Save_all."""
+        current_app.logger.debug("DataSetupService.save_all_process_models() start")
         failing_process_models = []
         process_models = ProcessModelService().get_process_models()
         for process_model in process_models:
@@ -26,7 +34,9 @@ class DataSetupService:
                 ]
                 if process_model.primary_file_name in bad_files:
                     continue
-                print(f"primary_file_name: {process_model.primary_file_name}")
+                current_app.logger.debug(
+                    f"primary_file_name: {process_model.primary_file_name}"
+                )
                 try:
                     SpecFileService.update_file(
                         process_model,
@@ -79,4 +89,5 @@ class DataSetupService:
                         "primary_file_name not set",
                     )
                 )
+        current_app.logger.debug("DataSetupService.save_all_process_models() end")
         return failing_process_models

src/spiffworkflow_backend/services/logging_service.py

@@ -23,7 +23,14 @@ from spiffworkflow_backend.models.spiff_logging import SpiffLoggingModel
 # full message list:
 # {'name': 'gunicorn.error', 'msg': 'GET /admin/token', 'args': (), 'levelname': 'DEBUG', 'levelno': 10, 'pathname': '~/.cache/pypoetry/virtualenvs/spiffworkflow-backend-R_hdWfN1-py3.10/lib/python3.10/site-packages/gunicorn/glogging.py', 'filename': 'glogging.py', 'module': 'glogging', 'exc_info': None, 'exc_text': None, 'stack_info': None, 'lineno': 267, 'funcName': 'debug', 'created': 1657307111.4513023, 'msecs': 451.30228996276855, 'relativeCreated': 1730.785846710205, 'thread': 139945864087360, 'threadName': 'MainThread', 'processName': 'MainProcess', 'process': 2109561, 'message': 'GET /admin/token', 'asctime': '2022-07-08T15:05:11.451Z'}
 
+
+class InvalidLogLevelError(Exception):
+    """InvalidLogLevelError."""
+
+
 # originally from https://stackoverflow.com/a/70223539/6090676
+
+
 class JsonFormatter(logging.Formatter):
     """Formatter that outputs JSON strings after parsing the LogRecord.
 
@@ -108,8 +115,16 @@ class SpiffFilter(logging.Filter):
 
 def setup_logger(app: Flask) -> None:
     """Setup_logger."""
-    log_level = logging.DEBUG
-    spiff_log_level = logging.DEBUG
+    upper_log_level_string = app.config["SPIFFWORKFLOW_BACKEND_LOG_LEVEL"].upper()
+    log_levels = ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]
+
+    if upper_log_level_string not in log_levels:
+        raise InvalidLogLevelError(
+            f"Log level given is invalid: '{upper_log_level_string}'. Valid options are {log_levels}"
+        )
+
+    log_level = getattr(logging, upper_log_level_string)
+    spiff_log_level = getattr(logging, upper_log_level_string)
     log_formatter = logging.Formatter(
         "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
     )

tests/spiffworkflow_backend/unit/test_acceptance_test_fixtures.py

@@ -1,12 +1,14 @@
 """Test_acceptance_test_fixtures."""
 from flask.app import Flask
 
-from spiffworkflow_backend.services.acceptance_test_fixtures import load_fixtures
+from spiffworkflow_backend.services.acceptance_test_fixtures import (
+    load_acceptance_test_fixtures,
+)
 
 
 def test_start_dates_are_one_hour_apart(app: Flask) -> None:
     """Test_start_dates_are_one_hour_apart."""
-    process_instances = load_fixtures()
+    process_instances = load_acceptance_test_fixtures()
 
     assert len(process_instances) > 2
     assert process_instances[0].start_in_seconds is not None

wsgi.py

@@ -2,7 +2,10 @@
 import os
 
 from spiffworkflow_backend import create_app
-from spiffworkflow_backend.services.acceptance_test_fixtures import load_fixtures
+from spiffworkflow_backend.services.acceptance_test_fixtures import (
+    load_acceptance_test_fixtures,
+)
+from spiffworkflow_backend.services.data_setup_service import DataSetupService
 
 app = create_app()
 
@@ -10,4 +13,4 @@ app = create_app()
 # it also loaded when we were running migrations, which resulted in a chicken/egg thing.
 if os.environ.get("SPIFFWORKFLOW_BACKEND_LOAD_FIXTURE_DATA") == "true":
     with app.app_context():
-        load_fixtures()
+        load_acceptance_test_fixtures()