Merge remote-tracking branch 'origin/main' into feature/permissions3

bin/boot_server_in_docker

@@ -10,6 +10,11 @@ set -o errtrace -o errexit -o nounset -o pipefail
 # run migrations
 export FLASK_APP=/app/src/spiffworkflow_backend
 
+if [[ "${WAIT_FOR_DB_TO_BE_READY:-}" == "true" ]]; then
+  echo 'Waiting for db to be ready...'
+  poetry run python ./bin/wait_for_db_to_be_ready.py
+fi
+
 if [[ "${DOWNGRADE_DB:-}" == "true" ]]; then
   echo 'Downgrading database...'
   poetry run flask db downgrade

bin/smash_all_containers

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+function error_handler() {
+  >&2 echo "Exited with BAD EXIT CODE '${2}' in ${0} script at line: ${1}."
+  exit "$2"
+}
+trap 'error_handler ${LINENO} $?' ERR
+set -o errtrace -o errexit -o nounset -o pipefail
+
+docker ps | grep -Ev '(static|CONTAINER)' | awk '{print $1}' | xargs docker stop
+docker ps -a | grep -Ev '(static|CONTAINER)' | awk '{print $1}' | xargs docker rm

bin/spiffworkflow-realm.json

@@ -1251,6 +1251,8 @@
       "secret": "JXeQExm0JhQPLumgHtIIqf52bDalHz0q",
       "redirectUris": [
         "http://localhost:7000/*",
+        "http://67.205.133.116:7000/*",
+        "https://api.demo.spiffworkflow.org/*",
         "http://167.172.242.138:7000/*"
       ],
       "webOrigins": [],
@@ -1520,6 +1522,8 @@
       "clientAuthenticatorType": "client-secret",
       "redirectUris": [
         "http://localhost:7001/*",
+        "http://67.205.133.116:7000/*",
+        "https://api.demo.spiffworkflow.org/*",
         "http://167.172.242.138:7001/*"
       ],
       "webOrigins": ["*"],
@@ -1588,6 +1592,8 @@
       "secret": "6o8kIKQznQtejHOdRhWeKorBJclMGcgA",
       "redirectUris": [
         "http://localhost:7001/*",
+        "http://67.205.133.116:7000/*",
+        "https://api.demo.spiffworkflow.org/*",
         "http://167.172.242.138:7001/*"
       ],
       "webOrigins": [],

bin/start_keycloak

@@ -35,7 +35,7 @@ docker exec keycloak /opt/keycloak/bin/kc.sh import --file /tmp/quarkus-realm.js
 echo 'imported realms'
 
 if [ "${TURN_OFF_SSL:-}" == "true" ]; then
-  docker exec -it keycloak /opt/keycloak/bin/kcadm.sh config credentials --server http://localhost:8080 --realm master --user admin
+  docker exec -it keycloak /opt/keycloak/bin/kcadm.sh config credentials --server http://localhost:8080 --realm master --user admin --password admin
   docker exec -it keycloak /opt/keycloak/bin/kcadm.sh update realms/master -s sslRequired=NONE
   docker exec -it keycloak /opt/keycloak/bin/kcadm.sh update realms/spiffworkflow -s sslRequired=NONE
   echo 'turned off SSL requirement'

bin/wait_for_db_to_be_ready.py

@@ -0,0 +1,31 @@
+"""Grabs tickets from csv and makes process instances."""
+import time
+
+import sqlalchemy
+from flask_bpmn.models.db import db
+
+from spiffworkflow_backend import get_hacked_up_app_for_script
+
+
+def try_to_connect(start_time: float) -> None:
+    """Try to connect."""
+    try:
+        db.first_or_404("select 1")
+    except sqlalchemy.exc.DatabaseError as exception:
+        if time.time() - start_time > 15:
+            raise exception
+        else:
+            time.sleep(1)
+            try_to_connect(start_time)
+
+
+def main() -> None:
+    """Main."""
+    app = get_hacked_up_app_for_script()
+    start_time = time.time()
+    with app.app_context():
+        try_to_connect(start_time)
+
+
+if __name__ == "__main__":
+    main()

keycloak/Dockerfile

@@ -0,0 +1,26 @@
+FROM quay.io/keycloak/keycloak:18.0.2 as builder
+
+ENV KEYCLOAK_LOGLEVEL="ALL"
+ENV ROOT_LOGLEVEL="ALL"
+ENV KC_HEALTH_ENABLED="true"
+# ENV KC_METRICS_ENABLED=true
+ENV PROXY_ADDRESS_FORWARDING="true"
+ENV KC_HOSTNAME="keycloak.demo.spiffworkflow.org"
+ENV KC_HOSTNAME_URL="https://keycloak.demo.spiffworkflow.org"
+ENV KC_FEATURES="token-exchange,admin-fine-grained-authz"
+# ENV KC_DB=postgres
+# Install custom providers
+# RUN curl -sL https://github.com/aerogear/keycloak-metrics-spi/releases/download/2.5.3/keycloak-metrics-spi-2.5.3.jar -o /opt/keycloak/providers/keycloak-metrics-spi-2.5.3.jar
+RUN /opt/keycloak/bin/kc.sh build
+
+FROM quay.io/keycloak/keycloak:18.0.2
+COPY --from=builder /opt/keycloak/ /opt/keycloak/
+WORKDIR /opt/keycloak
+# for demonstration purposes only, please make sure to use proper certificates in production instead
+# RUN keytool -genkeypair -storepass password -storetype PKCS12 -keyalg RSA -keysize 2048 -dname "CN=server" -alias server -ext "SAN:c=DNS:localhost,IP:127.0.0.1" -keystore conf/server.keystore
+# # change these values to point to a running postgres instance
+# ENV KC_DB_URL=<DBURL>
+# ENV KC_DB_USERNAME=<DBUSERNAME>
+# ENV KC_DB_PASSWORD=<DBPASSWORD>
+# ENV KC_HOSTNAME=localhost
+ENTRYPOINT ["/opt/keycloak/bin/kc.sh"]

poetry.lock

@@ -95,7 +95,7 @@ python-versions = ">=3.5"
 dev = ["cloudpickle", "coverage[toml] (>=5.0.2)", "furo", "hypothesis", "mypy (>=0.900,!=0.940)", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "sphinx", "sphinx-notfound-page", "zope.interface"]
 docs = ["furo", "sphinx", "sphinx-notfound-page", "zope.interface"]
 tests = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "zope.interface"]
-tests-no-zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins"]
+tests_no_zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins"]
 
 [[package]]
 name = "Babel"
@@ -268,7 +268,7 @@ optional = false
 python-versions = ">=3.6.0"
 
 [package.extras]
-unicode-backport = ["unicodedata2"]
+unicode_backport = ["unicodedata2"]
 
 [[package]]
 name = "classify-imports"
@@ -1512,7 +1512,7 @@ urllib3 = ">=1.21.1,<1.27"
 
 [package.extras]
 socks = ["PySocks (>=1.5.6,!=1.5.7)"]
-use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"]
+use_chardet_on_py3 = ["chardet (>=3.0.2,<6)"]
 
 [[package]]
 name = "requests-toolbelt"
@@ -1625,7 +1625,7 @@ falcon = ["falcon (>=1.4)"]
 fastapi = ["fastapi (>=0.79.0)"]
 flask = ["blinker (>=1.1)", "flask (>=0.11)"]
 httpx = ["httpx (>=0.16.0)"]
-pure-eval = ["asttokens", "executing", "pure-eval"]
+pure_eval = ["asttokens", "executing", "pure-eval"]
 pyspark = ["pyspark (>=2.4.4)"]
 quart = ["blinker (>=1.1)", "quart (>=0.16.1)"]
 rq = ["rq (>=0.6)"]
@@ -1847,7 +1847,7 @@ test = ["pytest"]
 
 [[package]]
 name = "SpiffWorkflow"
-version = "1.1.7"
+version = "1.2.0"
 description = "A workflow framework and BPMN/DMN Processor"
 category = "main"
 optional = false
@@ -1866,7 +1866,7 @@ pytz = "*"
 type = "git"
 url = "https://github.com/sartography/SpiffWorkflow"
 reference = "main"
-resolved_reference = "63db3e45947ec66b8d0efc2c74064004f8ff482c"
+resolved_reference = "d9fcd45a384f8376a669cf58677564289d2c661c"
 
 [[package]]
 name = "SQLAlchemy"
@@ -1884,19 +1884,19 @@ aiomysql = ["aiomysql", "greenlet (!=0.4.17)"]
 aiosqlite = ["aiosqlite", "greenlet (!=0.4.17)", "typing_extensions (!=3.10.0.1)"]
 asyncio = ["greenlet (!=0.4.17)"]
 asyncmy = ["asyncmy (>=0.2.3,!=0.2.4)", "greenlet (!=0.4.17)"]
-mariadb-connector = ["mariadb (>=1.0.1,!=1.1.2)"]
+mariadb_connector = ["mariadb (>=1.0.1,!=1.1.2)"]
 mssql = ["pyodbc"]
-mssql-pymssql = ["pymssql"]
+mssql_pymssql = ["pymssql"]
-mssql-pyodbc = ["pyodbc"]
+mssql_pyodbc = ["pyodbc"]
 mypy = ["mypy (>=0.910)", "sqlalchemy2-stubs"]
 mysql = ["mysqlclient (>=1.4.0)", "mysqlclient (>=1.4.0,<2)"]
-mysql-connector = ["mysql-connector-python"]
+mysql_connector = ["mysql-connector-python"]
 oracle = ["cx_oracle (>=7)", "cx_oracle (>=7,<8)"]
 postgresql = ["psycopg2 (>=2.7)"]
-postgresql-asyncpg = ["asyncpg", "greenlet (!=0.4.17)"]
+postgresql_asyncpg = ["asyncpg", "greenlet (!=0.4.17)"]
-postgresql-pg8000 = ["pg8000 (>=1.16.6,!=1.29.0)"]
+postgresql_pg8000 = ["pg8000 (>=1.16.6,!=1.29.0)"]
-postgresql-psycopg2binary = ["psycopg2-binary"]
+postgresql_psycopg2binary = ["psycopg2-binary"]
-postgresql-psycopg2cffi = ["psycopg2cffi"]
+postgresql_psycopg2cffi = ["psycopg2cffi"]
 pymysql = ["pymysql", "pymysql (<1)"]
 sqlcipher = ["sqlcipher3_binary"]
 
@@ -1978,6 +1978,46 @@ python-versions = ">=3.5.3"
 doc = ["sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme"]
 test = ["mypy", "pytest", "typing-extensions"]
 
+[[package]]
+name = "types-click"
+version = "7.1.8"
+description = "Typing stubs for click"
+category = "main"
+optional = false
+python-versions = "*"
+
+[[package]]
+name = "types-Flask"
+version = "1.1.6"
+description = "Typing stubs for Flask"
+category = "main"
+optional = false
+python-versions = "*"
+
+[package.dependencies]
+types-click = "*"
+types-Jinja2 = "*"
+types-Werkzeug = "*"
+
+[[package]]
+name = "types-Jinja2"
+version = "2.11.9"
+description = "Typing stubs for Jinja2"
+category = "main"
+optional = false
+python-versions = "*"
+
+[package.dependencies]
+types-MarkupSafe = "*"
+
+[[package]]
+name = "types-MarkupSafe"
+version = "1.1.10"
+description = "Typing stubs for MarkupSafe"
+category = "main"
+optional = false
+python-versions = "*"
+
 [[package]]
 name = "types-pytz"
 version = "2022.4.0.0"
@@ -2013,6 +2053,14 @@ category = "main"
 optional = false
 python-versions = "*"
 
+[[package]]
+name = "types-Werkzeug"
+version = "1.0.9"
+description = "Typing stubs for Werkzeug"
+category = "main"
+optional = false
+python-versions = "*"
+
 [[package]]
 name = "typing-extensions"
 version = "4.4.0"
@@ -2186,7 +2234,7 @@ testing = ["func-timeout", "jaraco.itertools", "pytest (>=6)", "pytest-black (>=
 [metadata]
 lock-version = "1.1"
 python-versions = ">=3.9,<3.11"
-content-hash = "7b4eb35239359ebff4c5597052aedc14b47cc7d1880b5617632edbb957511908"
+content-hash = "80df3106c84ea30cfbad3e6d708e136e07b6390c4d0c622aced0abf052d11300"
 
 [metadata.files]
 alabaster = [
@@ -3400,6 +3448,22 @@ typeguard = [
     {file = "typeguard-2.13.3-py3-none-any.whl", hash = "sha256:5e3e3be01e887e7eafae5af63d1f36c849aaa94e3a0112097312aabfa16284f1"},
     {file = "typeguard-2.13.3.tar.gz", hash = "sha256:00edaa8da3a133674796cf5ea87d9f4b4c367d77476e185e80251cc13dfbb8c4"},
 ]
+types-click = [
+    {file = "types-click-7.1.8.tar.gz", hash = "sha256:b6604968be6401dc516311ca50708a0a28baa7a0cb840efd7412f0dbbff4e092"},
+    {file = "types_click-7.1.8-py3-none-any.whl", hash = "sha256:8cb030a669e2e927461be9827375f83c16b8178c365852c060a34e24871e7e81"},
+]
+types-Flask = [
+    {file = "types-Flask-1.1.6.tar.gz", hash = "sha256:aac777b3abfff9436e6b01f6d08171cf23ea6e5be71cbf773aaabb1c5763e9cf"},
+    {file = "types_Flask-1.1.6-py3-none-any.whl", hash = "sha256:6ab8a9a5e258b76539d652f6341408867298550b19b81f0e41e916825fc39087"},
+]
+types-Jinja2 = [
+    {file = "types-Jinja2-2.11.9.tar.gz", hash = "sha256:dbdc74a40aba7aed520b7e4d89e8f0fe4286518494208b35123bcf084d4b8c81"},
+    {file = "types_Jinja2-2.11.9-py3-none-any.whl", hash = "sha256:60a1e21e8296979db32f9374d8a239af4cb541ff66447bb915d8ad398f9c63b2"},
+]
+types-MarkupSafe = [
+    {file = "types-MarkupSafe-1.1.10.tar.gz", hash = "sha256:85b3a872683d02aea3a5ac2a8ef590193c344092032f58457287fbf8e06711b1"},
+    {file = "types_MarkupSafe-1.1.10-py3-none-any.whl", hash = "sha256:ca2bee0f4faafc45250602567ef38d533e877d2ddca13003b319c551ff5b3cc5"},
+]
 types-pytz = [
     {file = "types-pytz-2022.4.0.0.tar.gz", hash = "sha256:17d66e4b16e80ceae0787726f3a22288df7d3f9fdebeb091dc64b92c0e4ea09d"},
     {file = "types_pytz-2022.4.0.0-py3-none-any.whl", hash = "sha256:950b0f3d64ed5b03a3e29c1e38fe2be8371c933c8e97922d0352345336eb8af4"},
@@ -3416,6 +3480,10 @@ types-urllib3 = [
     {file = "types-urllib3-1.26.25.tar.gz", hash = "sha256:5aef0e663724eef924afa8b320b62ffef2c1736c1fa6caecfc9bc6c8ae2c3def"},
     {file = "types_urllib3-1.26.25-py3-none-any.whl", hash = "sha256:c1d78cef7bd581e162e46c20a57b2e1aa6ebecdcf01fd0713bb90978ff3e3427"},
 ]
+types-Werkzeug = [
+    {file = "types-Werkzeug-1.0.9.tar.gz", hash = "sha256:5cc269604c400133d452a40cee6397655f878fc460e03fde291b9e3a5eaa518c"},
+    {file = "types_Werkzeug-1.0.9-py3-none-any.whl", hash = "sha256:194bd5715a13c598f05c63e8a739328657590943bce941e8a3619a6b5d4a54ec"},
+]
 typing-extensions = [
     {file = "typing_extensions-4.4.0-py3-none-any.whl", hash = "sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e"},
     {file = "typing_extensions-4.4.0.tar.gz", hash = "sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa"},

pyproject.toml

@@ -47,15 +47,12 @@ marshmallow-enum = "^1.5.1"
 marshmallow-sqlalchemy = "^0.28.0"
 PyJWT = "^2.4.0"
 gunicorn = "^20.1.0"
-types-pytz = "^2022.1.1"
 python-keycloak = "^2.5.0"
 APScheduler = "^3.9.1"
-types-requests = "^2.28.6"
 Jinja2 = "^3.1.2"
 RestrictedPython = "^5.2"
 Flask-SQLAlchemy = "^3"
 orjson = "^3.8.0"
-types-PyYAML = "^6.0.12"
 
 
 [tool.poetry.dev-dependencies]
@@ -71,6 +68,11 @@ pre-commit = "^2.20.0"
 flake8 = "^4.0.1"
 black = ">=21.10b0"
 flake8-bandit = "^2.1.2"
+types-Werkzeug = "^1.0.9"
+types-PyYAML = "^6.0.12"
+types-Flask = "^1.1.6"
+types-requests = "^2.28.6"
+types-pytz = "^2022.1.1"
 
 # 1.7.3 broke us. https://github.com/PyCQA/bandit/issues/841
 bandit = "1.7.2"

src/spiffworkflow_backend/__init__.py

@@ -13,6 +13,7 @@ from flask_bpmn.models.db import db
 from flask_bpmn.models.db import migrate
 from flask_cors import CORS  # type: ignore
 from flask_mail import Mail  # type: ignore
+from werkzeug.exceptions import NotFound
 
 import spiffworkflow_backend.load_database_models  # noqa: F401
 from spiffworkflow_backend.config import setup_config
@@ -145,6 +146,15 @@ def configure_sentry(app: flask.app.Flask) -> None:
     from flask import Flask
     from sentry_sdk.integrations.flask import FlaskIntegration
 
+    def before_send(event: Any, hint: Any) -> Any:
+        """Before_send."""
+        if "exc_info" in hint:
+            _exc_type, exc_value, _tb = hint["exc_info"]
+            # NotFound is mostly from web crawlers
+            if isinstance(exc_value, NotFound):
+                return None
+        return event
+
     sentry_sample_rate = app.config.get("SENTRY_SAMPLE_RATE")
     if sentry_sample_rate is None:
         return
@@ -153,10 +163,12 @@ def configure_sentry(app: flask.app.Flask) -> None:
         integrations=[
             FlaskIntegration(),
         ],
+        environment=app.config["ENV_IDENTIFIER"],
         # Set traces_sample_rate to 1.0 to capture 100%
         # of transactions for performance monitoring.
         # We recommend adjusting this value in production.
         traces_sample_rate=float(sentry_sample_rate),
+        before_send=before_send,
     )
 
     app = Flask(__name__)

src/spiffworkflow_backend/config/demo.py

@@ -0,0 +1,9 @@
+"""Staging."""
+from os import environ
+
+GIT_COMMIT_ON_SAVE = True
+GIT_COMMIT_USERNAME = "demo"
+GIT_COMMIT_EMAIL = "demo@example.com"
+SPIFFWORKFLOW_BACKEND_PERMISSIONS_FILE_NAME = environ.get(
+    "SPIFFWORKFLOW_BACKEND_PERMISSIONS_FILE_NAME", default="demo.yml"
+)

src/spiffworkflow_backend/config/permissions/demo.yml

@@ -0,0 +1,28 @@
+groups:
+  admin:
+    users: [jakub, kb, alex, dan, mike, jason, amir, jarrad, elizabeth, jon]
+
+  finance:
+    users: [harmeet, sasha]
+
+  hr:
+    users: [manuchehr]
+
+permissions:
+  admin:
+    groups: [admin]
+    users: []
+    allowed_permissions: [create, read, update, delete, list, instantiate]
+    uri: /*
+
+  finance-admin:
+    groups: [finance]
+    users: []
+    allowed_permissions: [create, read, update, delete]
+    uri: /v1.0/process-groups/finance/*
+
+  read-all:
+    groups: [finance, hr, admin]
+    users: []
+    allowed_permissions: [read]
+    uri: /*

src/spiffworkflow_backend/routes/admin_blueprint/admin_blueprint.py

@@ -7,7 +7,7 @@ from flask import redirect
 from flask import render_template
 from flask import request
 from flask import url_for
-from werkzeug.wrappers.response import Response
+from werkzeug.wrappers import Response
 
 from spiffworkflow_backend.services.process_instance_processor import (
     ProcessInstanceProcessor,

src/spiffworkflow_backend/routes/user.py

@@ -12,7 +12,7 @@ from flask import g
 from flask import redirect
 from flask import request
 from flask_bpmn.api.api_error import ApiError
-from werkzeug.wrappers.response import Response
+from werkzeug.wrappers import Response
 
 from spiffworkflow_backend.models.user import UserModel
 from spiffworkflow_backend.services.authentication_service import (

src/spiffworkflow_backend/services/authentication_service.py

@@ -10,7 +10,7 @@ import requests
 from flask import current_app
 from flask import redirect
 from flask_bpmn.api.api_error import ApiError
-from werkzeug.wrappers.response import Response
+from werkzeug.wrappers import Response
 
 
 class AuthenticationProviderTypes(enum.Enum):

tests/spiffworkflow_backend/helpers/base_test.py

@@ -13,7 +13,7 @@ from flask.testing import FlaskClient
 from flask_bpmn.api.api_error import ApiError
 from flask_bpmn.models.db import db
 from tests.spiffworkflow_backend.helpers.test_data import load_test_spec
-from werkzeug.test import TestResponse
+from werkzeug.test import TestResponse  # type: ignore
 
 from spiffworkflow_backend.models.permission_assignment import Permission
 from spiffworkflow_backend.models.permission_target import PermissionTargetModel

tests/spiffworkflow_backend/integration/test_secret_service.py

@@ -7,7 +7,7 @@ from flask.app import Flask
 from flask.testing import FlaskClient
 from flask_bpmn.api.api_error import ApiError
 from tests.spiffworkflow_backend.helpers.base_test import BaseTest
-from werkzeug.test import TestResponse
+from werkzeug.test import TestResponse  # type: ignore
 
 from spiffworkflow_backend.models.process_model import ProcessModelInfo
 from spiffworkflow_backend.models.secret_model import SecretModel