parallel tests with xdist (#335)

* parallel tests with xdist

* add pytest-xdist as dev dep

* put back spiff

* update messaging

* get more in line with main

---------

Co-authored-by: burnettk <burnettk@users.noreply.github.com>
Co-authored-by: jasquat <jasquat@users.noreply.github.com>

spiffworkflow-backend/bin/recreate_db

@@ -94,3 +94,10 @@ if [[ -n "${SPIFFWORKFLOW_BACKEND_ENV:-}" ]] && ! grep -Eq '^(local_development|
   fi
   FLASK_APP=src/spiffworkflow_backend poetry run flask db upgrade
 fi
+
+# for ./bin/tests-par (parallel tests with xdist)
+if [[ -f "./src/instance/db_unit_testing.sqlite3" ]] ; then
+  for i in $(seq 0 16); do
+    cp ./src/instance/db_unit_testing.sqlite3 ./src/instance/db_unit_testing_gw$i.sqlite3
+  done
+fi

spiffworkflow-backend/bin/tests-par

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+function error_handler() {
+  >&2 echo "Exited with BAD EXIT CODE '${2}' in ${0} script at line: ${1}."
+  exit "$2"
+}
+trap 'error_handler ${LINENO} $?' ERR
+set -o errtrace -o errexit -o nounset -o pipefail
+
+if [[ ! -f ./src/instance/db_unit_testing_gw0.sqlite3 ]] ; then
+  >&2 echo -e "ERROR: please run the following command first in order to set up and migrate the sqlite unit_testing database:\n\n\tSPIFFWORKFLOW_BACKEND_DATABASE_TYPE=sqlite ./bin/recreate_db clean"
+  exit 1
+fi
+
+# check if python package pytest-xdist is installed
+if ! python -c "import xdist" &>/dev/null; then
+  >&2 echo -e "ERROR: please install the python package pytest-xdist by running poetry install"
+  exit 1
+fi
+
+SPIFFWORKFLOW_BACKEND_DATABASE_TYPE=sqlite poet test -n auto -x --ff

spiffworkflow-backend/poetry.lock

@@ -643,6 +643,21 @@ files = [
 [package.extras]
 test = ["pytest (>=6)"]
 
+[[package]]
+name = "execnet"
+version = "1.9.0"
+description = "execnet: rapid multi-Python deployment"
+category = "main"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
+files = [
+    {file = "execnet-1.9.0-py2.py3-none-any.whl", hash = "sha256:a295f7cc774947aac58dde7fdc85f4aa00c42adf5d8f5468fc630c1acf30a142"},
+    {file = "execnet-1.9.0.tar.gz", hash = "sha256:8f694f3ba9cc92cab508b152dcfe322153975c29bda272e2fd7f3f00f36e47c5"},
+]
+
+[package.extras]
+testing = ["pre-commit"]
+
 [[package]]
 name = "filelock"
 version = "3.11.0"
@@ -1852,6 +1867,27 @@ pytest = ">=5.0"
 [package.extras]
 dev = ["pre-commit", "pytest-asyncio", "tox"]
 
+[[package]]
+name = "pytest-xdist"
+version = "3.3.1"
+description = "pytest xdist plugin for distributed testing, most importantly across multiple CPUs"
+category = "main"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "pytest-xdist-3.3.1.tar.gz", hash = "sha256:d5ee0520eb1b7bcca50a60a518ab7a7707992812c578198f8b44fdfac78e8c93"},
+    {file = "pytest_xdist-3.3.1-py3-none-any.whl", hash = "sha256:ff9daa7793569e6a68544850fd3927cd257cc03a7ef76c95e86915355e82b5f2"},
+]
+
+[package.dependencies]
+execnet = ">=1.1"
+pytest = ">=6.2.0"
+
+[package.extras]
+psutil = ["psutil (>=3.0)"]
+setproctitle = ["setproctitle"]
+testing = ["filelock"]
+
 [[package]]
 name = "python-dateutil"
 version = "2.8.2"
@@ -2839,4 +2875,4 @@ tests-strict = ["codecov (==2.0.15)", "pytest (==4.6.0)", "pytest (==4.6.0)", "p
 [metadata]
 lock-version = "2.0"
 python-versions = ">=3.10,<3.12"
-content-hash = "67863394f8de94eaddd20964ae383c6dc3416bbdec623e399b5a8a0d163e5178"
+content-hash = "de301503903ea357212400d7cf27feffe5a73b2733c0b4f2c39cabf3de3b9bc7"

spiffworkflow-backend/pyproject.toml

@@ -82,6 +82,7 @@ prometheus-flask-exporter = "^0.22.3"
 sqlalchemy = "^2.0.7"
 marshmallow-sqlalchemy = "^0.29.0"
 spiff-element-units = "^0.3.0"
+pytest-xdist = "^3.3.1"
 
 [tool.poetry.dev-dependencies]
 pytest = "^7.1.2"

spiffworkflow-backend/src/spiffworkflow_backend/config/__init__.py

@@ -18,11 +18,16 @@ class ConfigurationError(Exception):
 
 
 def setup_database_configs(app: Flask) -> None:
+    worker_id = os.environ.get("PYTEST_XDIST_WORKER")
+    parallel_test_suffix = ""
+    if worker_id is not None:
+        parallel_test_suffix = f"_{worker_id}"
+
     if app.config.get("SPIFFWORKFLOW_BACKEND_DATABASE_URI") is None:
         database_name = f"spiffworkflow_backend_{app.config['ENV_IDENTIFIER']}"
         if app.config.get("SPIFFWORKFLOW_BACKEND_DATABASE_TYPE") == "sqlite":
             app.config["SQLALCHEMY_DATABASE_URI"] = (
-                f"sqlite:///{app.instance_path}/db_{app.config['ENV_IDENTIFIER']}.sqlite3"
+                f"sqlite:///{app.instance_path}/db_{app.config['ENV_IDENTIFIER']}{parallel_test_suffix}.sqlite3"
             )
         elif app.config.get("SPIFFWORKFLOW_BACKEND_DATABASE_TYPE") == "postgres":
             app.config["SQLALCHEMY_DATABASE_URI"] = (

spiffworkflow-backend/src/spiffworkflow_backend/config/unit_testing.py

@@ -15,6 +15,10 @@ SPIFFWORKFLOW_BACKEND_GIT_COMMIT_ON_SAVE = False
 
 # NOTE: set this here since nox shoves tests and src code to
 # different places and this allows us to know exactly where we are at the start
+worker_id = environ.get("PYTEST_XDIST_WORKER")
+parallel_test_suffix = ""
+if worker_id is not None:
+    parallel_test_suffix = f"_{worker_id}"
 SPIFFWORKFLOW_BACKEND_BPMN_SPEC_ABSOLUTE_DIR = os.path.join(
     os.path.dirname(__file__),
     "..",
@@ -23,5 +27,5 @@ SPIFFWORKFLOW_BACKEND_BPMN_SPEC_ABSOLUTE_DIR = os.path.join(
     "tests",
     "spiffworkflow_backend",
     "files",
-    "bpmn_specs",
+    f"bpmn_specs{parallel_test_suffix}",
 )