Merge branch 'main' into cullerton

.github/workflows/tests.yml

@@ -129,15 +129,6 @@ jobs:
           key: ${{ steps.pre-commit-cache.outputs.result }}-${{ hashFiles('.pre-commit-config.yaml') }}
           restore-keys: |
             ${{ steps.pre-commit-cache.outputs.result }}-
-
-      # TODO: remove keycloak server when calls to keycloak are being mocked
-      - name: start_keycloak
-        run: ./bin/start_keycloak
-        if: matrix.session == 'tests'
-      - name: wait_for_keycloak
-        run: ./bin/wait_for_keycloak 5
-        if: matrix.session == 'tests'
-
       - name: Setup Mysql
         uses: mirromutth/mysql-action@v1.1
         with:

bin/boot_in_docker_debug_mode

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+function error_handler() {
+  >&2 echo "Exited with BAD EXIT CODE '${2}' in ${0} script at line: ${1}."
+  exit "$2"
+}
+trap 'error_handler ${LINENO} $?' ERR
+set -o errtrace -o errexit -o nounset -o pipefail
+
+if [[ ! -f /app/log/db_development.log ]]; then
+  touch /app/log/db_development.log
+fi
+
+tail -f /app/log/db_development.log

bin/build_and_run_with_docker_compose

@@ -12,13 +12,17 @@ if [[ -z "${BPMN_SPEC_ABSOLUTE_DIR:-}" ]]; then
   export BPMN_SPEC_ABSOLUTE_DIR="$script_dir/../../sample-process-models"
 fi
 
+if [[ -z "${SPIFFWORKFLOW_BACKEND_DOCKER_COMPOSE_PROFILE:-}" ]]; then
+  export SPIFFWORKFLOW_BACKEND_DOCKER_COMPOSE_PROFILE=run
+fi
+
 additional_args=""
 if [[ "${RUN_WITH_DAEMON:-}" != "false" ]]; then
   additional_args="${additional_args} -d"
 fi
 
-docker compose build
+docker compose --profile "$SPIFFWORKFLOW_BACKEND_DOCKER_COMPOSE_PROFILE" build
-docker compose stop
+docker compose --profile "$SPIFFWORKFLOW_BACKEND_DOCKER_COMPOSE_PROFILE" stop
 
 # i observed a case locally where the db had a stale sqlalchemy revision which
 # caused the backend to exit and when docker compose up was running with
@@ -29,4 +33,4 @@ docker compose stop
 # bin/wait_for_server_to_be_up script.
 docker volume rm spiffworkflow-backend_spiffworkflow_backend || echo 'docker volume not found'
 
-docker compose up --wait $additional_args
+docker compose --profile "$SPIFFWORKFLOW_BACKEND_DOCKER_COMPOSE_PROFILE" up --wait $additional_args

bin/deploy

@@ -27,6 +27,10 @@ if [[ -z "${SPIFFWORKFLOW_BACKEND_DATABASE_DOCKER_RESTART_POLICY:-}" ]]; then
   export SPIFFWORKFLOW_BACKEND_DATABASE_DOCKER_RESTART_POLICY=always
 fi
 
+if [[ -z "${SPIFFWORKFLOW_BACKEND_DOCKER_COMPOSE_PROFILE:-}" ]]; then
+  export SPIFFWORKFLOW_BACKEND_DOCKER_COMPOSE_PROFILE=run
+fi
+
 git pull
 ./bin/build_and_run_with_docker_compose
 ./bin/wait_for_server_to_be_up

bin/start_keycloak

@@ -6,10 +6,22 @@ function error_handler() {
 }
 trap 'error_handler ${LINENO} $?' ERR
 set -o errtrace -o errexit -o nounset -o pipefail
-set -x
 
+if ! grep -qE '\<spiffworkflow\>' <<<"$(docker network ls)" ; then
+  docker network create spiffworkflow
+fi
 docker rm keycloak 2>/dev/null || echo 'no keycloak container found'
-docker run -p 7002:8080 -d  --name keycloak -e KEYCLOAK_LOGLEVEL=ALL -e ROOT_LOGLEVEL=ALL -e KEYCLOAK_ADMIN=admin -e KEYCLOAK_ADMIN_PASSWORD=admin quay.io/keycloak/keycloak:18.0.2 start-dev -Dkeycloak.profile.feature.token_exchange=enabled -Dkeycloak.profile.feature.admin_fine_grained_authz=enabled
+docker run \
+  -p 7002:8080 \
+  -d \
+  --network=spiffworkflow \
+  --name keycloak \
+  -e KEYCLOAK_LOGLEVEL=ALL \
+  -e ROOT_LOGLEVEL=ALL \
+  -e KEYCLOAK_ADMIN=admin \
+  -e KEYCLOAK_ADMIN_PASSWORD=admin quay.io/keycloak/keycloak:18.0.2 start-dev \
+  -Dkeycloak.profile.feature.token_exchange=enabled \
+  -Dkeycloak.profile.feature.admin_fine_grained_authz=enabled
 
 docker cp  bin/finance-realm.json keycloak:/tmp
 docker cp  bin/spiffworkflow-realm.json keycloak:/tmp

docker-compose.yml

@@ -1,3 +1,21 @@
+# Why we are running with network_mode: host
+# Wow this has been awful. We run three things in docker: mysql, keycloak, and the backend server.
+# The backend-server needs to talk to the other two.
+#
+# In order to talk to keycloak, it needs to go through localhost so that it can communicate with
+# keycloak using the same url as the frontend so that tokens can be properly validated.
+# If the domains are different, keycloak invalidates the token. There may be a way to change
+# this but I didn't find it.
+#
+# In order for the backend server to talk to the mysql server, they need to be on the same network.
+# I tried splitting it out where the mysql runs on a custom network and the backend runs on both
+# the custom network AND with localhost. Nothing I tried worked and googling didn't help. They
+# only ever mentioned one thing or using host.docker.internal which would cause the domains to
+# be different.
+#
+# So instead we are running with both the mysql server and the backend server in host netowrk mode.
+# There may be a better way to do this but if it works, then it works.
+
 version: "3.8"
 services:
   db:
@@ -10,8 +28,10 @@ services:
     environment:
       - MYSQL_DATABASE=${SPIFFWORKFLOW_BACKEND_DATABASE_NAME:-spiffworkflow_backend_development}
       - MYSQL_ROOT_PASSWORD=${SPIFFWORKFLOW_BACKEND_MYSQL_ROOT_DATABASE:-my-secret-pw}
+      - MYSQL_TCP_PORT=7003
+    network_mode: host
     ports:
-      - "3306"
+      - "7003"
     volumes:
       - spiffworkflow_backend:/var/lib/mysql
     healthcheck:
@@ -20,13 +40,13 @@ services:
       timeout: 5s
       retries: 10
 
-  spiffworkflow-backend:
+  spiffworkflow-backend: &spiffworkflow-backend
     container_name: spiffworkflow-backend
-    # command: tail -f /etc/hostname
+    profiles:
+      - run
     depends_on:
       db:
         condition: service_healthy
-    # image: sartography/cr-connect-workflow:dev
     build:
       context: .
     environment:
@@ -36,11 +56,12 @@ services:
       - FLASK_SESSION_SECRET_KEY=${FLASK_SESSION_SECRET_KEY:-super_secret_key}
       - SPIFFWORKFLOW_BACKEND_PORT=7000
       - SPIFFWORKFLOW_BACKEND_UPGRADE_DB=true
-      - SPIFFWORKFLOW_BACKEND_DATABASE_URI=mysql+mysqlconnector://root:${SPIFFWORKFLOW_BACKEND_MYSQL_ROOT_DATABASE:-my-secret-pw}@db/${SPIFFWORKFLOW_BACKEND_DATABASE_NAME:-spiffworkflow_backend_development}
+      - SPIFFWORKFLOW_BACKEND_DATABASE_URI=mysql+mysqlconnector://root:${SPIFFWORKFLOW_BACKEND_MYSQL_ROOT_DATABASE:-my-secret-pw}@localhost:7003/${SPIFFWORKFLOW_BACKEND_DATABASE_NAME:-spiffworkflow_backend_development}
       - BPMN_SPEC_ABSOLUTE_DIR=/app/process_models
       - SPIFFWORKFLOW_BACKEND_LOAD_FIXTURE_DATA=${SPIFFWORKFLOW_BACKEND_LOAD_FIXTURE_DATA:-false}
     ports:
       - "7000:7000"
+    network_mode: host
     volumes:
       - ${BPMN_SPEC_ABSOLUTE_DIR:-./../sample-process-models}:/app/process_models
       - ./log:/app/log
@@ -50,6 +71,26 @@ services:
       timeout: 5s
       retries: 20
 
+  spiffworkflow-backend-local-debug:
+    <<: *spiffworkflow-backend
+    container_name: spiffworkflow-backend-local-debug
+    profiles:
+      - debug
+    volumes:
+      - ${BPMN_SPEC_ABSOLUTE_DIR:-./../sample-process-models}:/app/process_models
+      - ./:/app
+    command: /app/bin/boot_in_docker_debug_mode
+
+    # the docs say we can disable healthchecks with disable: true
+    # but it returns a bad exit code so setup one that doesn't matter
+    # since there is nothing to healthcheck in this case
+    # https://docs.docker.com/compose/compose-file/compose-file-v3/#healthcheck
+    healthcheck:
+      test: cat /etc/hosts
+      interval: 10s
+      timeout: 5s
+      retries: 20
+
 volumes:
   spiffworkflow_backend:
     driver: local