diff --git a/bin/boot_server_in_docker b/bin/boot_server_in_docker index a6f9bf0c..0a7fc6fe 100755 --- a/bin/boot_server_in_docker +++ b/bin/boot_server_in_docker @@ -10,6 +10,11 @@ set -o errtrace -o errexit -o nounset -o pipefail # run migrations export FLASK_APP=/app/src/spiffworkflow_backend +if [[ "${WAIT_FOR_DB_TO_BE_READY:-}" == "true" ]]; then + echo 'Waiting for db to be ready...' + poetry run python ./bin/wait_for_db_to_be_ready.py +fi + if [[ "${DOWNGRADE_DB:-}" == "true" ]]; then echo 'Downgrading database...' poetry run flask db downgrade diff --git a/bin/smash_all_containers b/bin/smash_all_containers new file mode 100755 index 00000000..eb26841a --- /dev/null +++ b/bin/smash_all_containers @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +function error_handler() { + >&2 echo "Exited with BAD EXIT CODE '${2}' in ${0} script at line: ${1}." + exit "$2" +} +trap 'error_handler ${LINENO} $?' ERR +set -o errtrace -o errexit -o nounset -o pipefail + +docker ps | grep -Ev '(static|CONTAINER)' | awk '{print $1}' | xargs docker stop +docker ps -a | grep -Ev '(static|CONTAINER)' | awk '{print $1}' | xargs docker rm diff --git a/bin/spiffworkflow-realm.json b/bin/spiffworkflow-realm.json index 15737d7a..1e659e63 100644 --- a/bin/spiffworkflow-realm.json +++ b/bin/spiffworkflow-realm.json @@ -1251,6 +1251,8 @@ "secret": "JXeQExm0JhQPLumgHtIIqf52bDalHz0q", "redirectUris": [ "http://localhost:7000/*", + "http://67.205.133.116:7000/*", + "https://api.demo.spiffworkflow.org/*", "http://167.172.242.138:7000/*" ], "webOrigins": [], @@ -1520,6 +1522,8 @@ "clientAuthenticatorType": "client-secret", "redirectUris": [ "http://localhost:7001/*", + "http://67.205.133.116:7000/*", + "https://api.demo.spiffworkflow.org/*", "http://167.172.242.138:7001/*" ], "webOrigins": ["*"], @@ -1588,6 +1592,8 @@ "secret": "6o8kIKQznQtejHOdRhWeKorBJclMGcgA", "redirectUris": [ "http://localhost:7001/*", + "http://67.205.133.116:7000/*", + "https://api.demo.spiffworkflow.org/*", "http://167.172.242.138:7001/*" ], "webOrigins": [], diff --git a/bin/start_keycloak b/bin/start_keycloak index 932e14ba..3b5ddd1e 100755 --- a/bin/start_keycloak +++ b/bin/start_keycloak @@ -35,7 +35,7 @@ docker exec keycloak /opt/keycloak/bin/kc.sh import --file /tmp/quarkus-realm.js echo 'imported realms' if [ "${TURN_OFF_SSL:-}" == "true" ]; then - docker exec -it keycloak /opt/keycloak/bin/kcadm.sh config credentials --server http://localhost:8080 --realm master --user admin + docker exec -it keycloak /opt/keycloak/bin/kcadm.sh config credentials --server http://localhost:8080 --realm master --user admin --password admin docker exec -it keycloak /opt/keycloak/bin/kcadm.sh update realms/master -s sslRequired=NONE docker exec -it keycloak /opt/keycloak/bin/kcadm.sh update realms/spiffworkflow -s sslRequired=NONE echo 'turned off SSL requirement' diff --git a/bin/wait_for_db_to_be_ready.py b/bin/wait_for_db_to_be_ready.py new file mode 100644 index 00000000..40497ae9 --- /dev/null +++ b/bin/wait_for_db_to_be_ready.py @@ -0,0 +1,31 @@ +"""Grabs tickets from csv and makes process instances.""" +import time + +import sqlalchemy +from flask_bpmn.models.db import db + +from spiffworkflow_backend import get_hacked_up_app_for_script + + +def try_to_connect(start_time: float) -> None: + """Try to connect.""" + try: + db.first_or_404("select 1") + except sqlalchemy.exc.DatabaseError as exception: + if time.time() - start_time > 15: + raise exception + else: + time.sleep(1) + try_to_connect(start_time) + + +def main() -> None: + """Main.""" + app = get_hacked_up_app_for_script() + start_time = time.time() + with app.app_context(): + try_to_connect(start_time) + + +if __name__ == "__main__": + main() diff --git a/keycloak/Dockerfile b/keycloak/Dockerfile new file mode 100644 index 00000000..af750bde --- /dev/null +++ b/keycloak/Dockerfile @@ -0,0 +1,26 @@ +FROM quay.io/keycloak/keycloak:18.0.2 as builder + +ENV KEYCLOAK_LOGLEVEL="ALL" +ENV ROOT_LOGLEVEL="ALL" +ENV KC_HEALTH_ENABLED="true" +# ENV KC_METRICS_ENABLED=true +ENV PROXY_ADDRESS_FORWARDING="true" +ENV KC_HOSTNAME="keycloak.demo.spiffworkflow.org" +ENV KC_HOSTNAME_URL="https://keycloak.demo.spiffworkflow.org" +ENV KC_FEATURES="token-exchange,admin-fine-grained-authz" +# ENV KC_DB=postgres +# Install custom providers +# RUN curl -sL https://github.com/aerogear/keycloak-metrics-spi/releases/download/2.5.3/keycloak-metrics-spi-2.5.3.jar -o /opt/keycloak/providers/keycloak-metrics-spi-2.5.3.jar +RUN /opt/keycloak/bin/kc.sh build + +FROM quay.io/keycloak/keycloak:18.0.2 +COPY --from=builder /opt/keycloak/ /opt/keycloak/ +WORKDIR /opt/keycloak +# for demonstration purposes only, please make sure to use proper certificates in production instead +# RUN keytool -genkeypair -storepass password -storetype PKCS12 -keyalg RSA -keysize 2048 -dname "CN=server" -alias server -ext "SAN:c=DNS:localhost,IP:127.0.0.1" -keystore conf/server.keystore +# # change these values to point to a running postgres instance +# ENV KC_DB_URL= +# ENV KC_DB_USERNAME= +# ENV KC_DB_PASSWORD= +# ENV KC_HOSTNAME=localhost +ENTRYPOINT ["/opt/keycloak/bin/kc.sh"] diff --git a/poetry.lock b/poetry.lock index b3e02eab..48574ef5 100644 --- a/poetry.lock +++ b/poetry.lock @@ -95,7 +95,7 @@ python-versions = ">=3.5" dev = ["cloudpickle", "coverage[toml] (>=5.0.2)", "furo", "hypothesis", "mypy (>=0.900,!=0.940)", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "sphinx", "sphinx-notfound-page", "zope.interface"] docs = ["furo", "sphinx", "sphinx-notfound-page", "zope.interface"] tests = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "zope.interface"] -tests-no-zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins"] +tests_no_zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins"] [[package]] name = "Babel" @@ -268,7 +268,7 @@ optional = false python-versions = ">=3.6.0" [package.extras] -unicode-backport = ["unicodedata2"] +unicode_backport = ["unicodedata2"] [[package]] name = "classify-imports" @@ -1512,7 +1512,7 @@ urllib3 = ">=1.21.1,<1.27" [package.extras] socks = ["PySocks (>=1.5.6,!=1.5.7)"] -use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] +use_chardet_on_py3 = ["chardet (>=3.0.2,<6)"] [[package]] name = "requests-toolbelt" @@ -1625,7 +1625,7 @@ falcon = ["falcon (>=1.4)"] fastapi = ["fastapi (>=0.79.0)"] flask = ["blinker (>=1.1)", "flask (>=0.11)"] httpx = ["httpx (>=0.16.0)"] -pure-eval = ["asttokens", "executing", "pure-eval"] +pure_eval = ["asttokens", "executing", "pure-eval"] pyspark = ["pyspark (>=2.4.4)"] quart = ["blinker (>=1.1)", "quart (>=0.16.1)"] rq = ["rq (>=0.6)"] @@ -1847,7 +1847,7 @@ test = ["pytest"] [[package]] name = "SpiffWorkflow" -version = "1.1.7" +version = "1.2.0" description = "A workflow framework and BPMN/DMN Processor" category = "main" optional = false @@ -1866,7 +1866,7 @@ pytz = "*" type = "git" url = "https://github.com/sartography/SpiffWorkflow" reference = "main" -resolved_reference = "63db3e45947ec66b8d0efc2c74064004f8ff482c" +resolved_reference = "d9fcd45a384f8376a669cf58677564289d2c661c" [[package]] name = "SQLAlchemy" @@ -1884,19 +1884,19 @@ aiomysql = ["aiomysql", "greenlet (!=0.4.17)"] aiosqlite = ["aiosqlite", "greenlet (!=0.4.17)", "typing_extensions (!=3.10.0.1)"] asyncio = ["greenlet (!=0.4.17)"] asyncmy = ["asyncmy (>=0.2.3,!=0.2.4)", "greenlet (!=0.4.17)"] -mariadb-connector = ["mariadb (>=1.0.1,!=1.1.2)"] +mariadb_connector = ["mariadb (>=1.0.1,!=1.1.2)"] mssql = ["pyodbc"] -mssql-pymssql = ["pymssql"] -mssql-pyodbc = ["pyodbc"] +mssql_pymssql = ["pymssql"] +mssql_pyodbc = ["pyodbc"] mypy = ["mypy (>=0.910)", "sqlalchemy2-stubs"] mysql = ["mysqlclient (>=1.4.0)", "mysqlclient (>=1.4.0,<2)"] -mysql-connector = ["mysql-connector-python"] +mysql_connector = ["mysql-connector-python"] oracle = ["cx_oracle (>=7)", "cx_oracle (>=7,<8)"] postgresql = ["psycopg2 (>=2.7)"] -postgresql-asyncpg = ["asyncpg", "greenlet (!=0.4.17)"] -postgresql-pg8000 = ["pg8000 (>=1.16.6,!=1.29.0)"] -postgresql-psycopg2binary = ["psycopg2-binary"] -postgresql-psycopg2cffi = ["psycopg2cffi"] +postgresql_asyncpg = ["asyncpg", "greenlet (!=0.4.17)"] +postgresql_pg8000 = ["pg8000 (>=1.16.6,!=1.29.0)"] +postgresql_psycopg2binary = ["psycopg2-binary"] +postgresql_psycopg2cffi = ["psycopg2cffi"] pymysql = ["pymysql", "pymysql (<1)"] sqlcipher = ["sqlcipher3_binary"] @@ -1978,6 +1978,46 @@ python-versions = ">=3.5.3" doc = ["sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme"] test = ["mypy", "pytest", "typing-extensions"] +[[package]] +name = "types-click" +version = "7.1.8" +description = "Typing stubs for click" +category = "main" +optional = false +python-versions = "*" + +[[package]] +name = "types-Flask" +version = "1.1.6" +description = "Typing stubs for Flask" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +types-click = "*" +types-Jinja2 = "*" +types-Werkzeug = "*" + +[[package]] +name = "types-Jinja2" +version = "2.11.9" +description = "Typing stubs for Jinja2" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +types-MarkupSafe = "*" + +[[package]] +name = "types-MarkupSafe" +version = "1.1.10" +description = "Typing stubs for MarkupSafe" +category = "main" +optional = false +python-versions = "*" + [[package]] name = "types-pytz" version = "2022.4.0.0" @@ -2013,6 +2053,14 @@ category = "main" optional = false python-versions = "*" +[[package]] +name = "types-Werkzeug" +version = "1.0.9" +description = "Typing stubs for Werkzeug" +category = "main" +optional = false +python-versions = "*" + [[package]] name = "typing-extensions" version = "4.4.0" @@ -2186,7 +2234,7 @@ testing = ["func-timeout", "jaraco.itertools", "pytest (>=6)", "pytest-black (>= [metadata] lock-version = "1.1" python-versions = ">=3.9,<3.11" -content-hash = "7b4eb35239359ebff4c5597052aedc14b47cc7d1880b5617632edbb957511908" +content-hash = "80df3106c84ea30cfbad3e6d708e136e07b6390c4d0c622aced0abf052d11300" [metadata.files] alabaster = [ @@ -3400,6 +3448,22 @@ typeguard = [ {file = "typeguard-2.13.3-py3-none-any.whl", hash = "sha256:5e3e3be01e887e7eafae5af63d1f36c849aaa94e3a0112097312aabfa16284f1"}, {file = "typeguard-2.13.3.tar.gz", hash = "sha256:00edaa8da3a133674796cf5ea87d9f4b4c367d77476e185e80251cc13dfbb8c4"}, ] +types-click = [ + {file = "types-click-7.1.8.tar.gz", hash = "sha256:b6604968be6401dc516311ca50708a0a28baa7a0cb840efd7412f0dbbff4e092"}, + {file = "types_click-7.1.8-py3-none-any.whl", hash = "sha256:8cb030a669e2e927461be9827375f83c16b8178c365852c060a34e24871e7e81"}, +] +types-Flask = [ + {file = "types-Flask-1.1.6.tar.gz", hash = "sha256:aac777b3abfff9436e6b01f6d08171cf23ea6e5be71cbf773aaabb1c5763e9cf"}, + {file = "types_Flask-1.1.6-py3-none-any.whl", hash = "sha256:6ab8a9a5e258b76539d652f6341408867298550b19b81f0e41e916825fc39087"}, +] +types-Jinja2 = [ + {file = "types-Jinja2-2.11.9.tar.gz", hash = "sha256:dbdc74a40aba7aed520b7e4d89e8f0fe4286518494208b35123bcf084d4b8c81"}, + {file = "types_Jinja2-2.11.9-py3-none-any.whl", hash = "sha256:60a1e21e8296979db32f9374d8a239af4cb541ff66447bb915d8ad398f9c63b2"}, +] +types-MarkupSafe = [ + {file = "types-MarkupSafe-1.1.10.tar.gz", hash = "sha256:85b3a872683d02aea3a5ac2a8ef590193c344092032f58457287fbf8e06711b1"}, + {file = "types_MarkupSafe-1.1.10-py3-none-any.whl", hash = "sha256:ca2bee0f4faafc45250602567ef38d533e877d2ddca13003b319c551ff5b3cc5"}, +] types-pytz = [ {file = "types-pytz-2022.4.0.0.tar.gz", hash = "sha256:17d66e4b16e80ceae0787726f3a22288df7d3f9fdebeb091dc64b92c0e4ea09d"}, {file = "types_pytz-2022.4.0.0-py3-none-any.whl", hash = "sha256:950b0f3d64ed5b03a3e29c1e38fe2be8371c933c8e97922d0352345336eb8af4"}, @@ -3416,6 +3480,10 @@ types-urllib3 = [ {file = "types-urllib3-1.26.25.tar.gz", hash = "sha256:5aef0e663724eef924afa8b320b62ffef2c1736c1fa6caecfc9bc6c8ae2c3def"}, {file = "types_urllib3-1.26.25-py3-none-any.whl", hash = "sha256:c1d78cef7bd581e162e46c20a57b2e1aa6ebecdcf01fd0713bb90978ff3e3427"}, ] +types-Werkzeug = [ + {file = "types-Werkzeug-1.0.9.tar.gz", hash = "sha256:5cc269604c400133d452a40cee6397655f878fc460e03fde291b9e3a5eaa518c"}, + {file = "types_Werkzeug-1.0.9-py3-none-any.whl", hash = "sha256:194bd5715a13c598f05c63e8a739328657590943bce941e8a3619a6b5d4a54ec"}, +] typing-extensions = [ {file = "typing_extensions-4.4.0-py3-none-any.whl", hash = "sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e"}, {file = "typing_extensions-4.4.0.tar.gz", hash = "sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa"}, diff --git a/pyproject.toml b/pyproject.toml index 76edc9a9..f1037540 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,15 +47,12 @@ marshmallow-enum = "^1.5.1" marshmallow-sqlalchemy = "^0.28.0" PyJWT = "^2.4.0" gunicorn = "^20.1.0" -types-pytz = "^2022.1.1" python-keycloak = "^2.5.0" APScheduler = "^3.9.1" -types-requests = "^2.28.6" Jinja2 = "^3.1.2" RestrictedPython = "^5.2" Flask-SQLAlchemy = "^3" orjson = "^3.8.0" -types-PyYAML = "^6.0.12" [tool.poetry.dev-dependencies] @@ -71,6 +68,11 @@ pre-commit = "^2.20.0" flake8 = "^4.0.1" black = ">=21.10b0" flake8-bandit = "^2.1.2" +types-Werkzeug = "^1.0.9" +types-PyYAML = "^6.0.12" +types-Flask = "^1.1.6" +types-requests = "^2.28.6" +types-pytz = "^2022.1.1" # 1.7.3 broke us. https://github.com/PyCQA/bandit/issues/841 bandit = "1.7.2" diff --git a/src/spiffworkflow_backend/__init__.py b/src/spiffworkflow_backend/__init__.py index f2e14ad5..0b2fd320 100644 --- a/src/spiffworkflow_backend/__init__.py +++ b/src/spiffworkflow_backend/__init__.py @@ -13,6 +13,7 @@ from flask_bpmn.models.db import db from flask_bpmn.models.db import migrate from flask_cors import CORS # type: ignore from flask_mail import Mail # type: ignore +from werkzeug.exceptions import NotFound import spiffworkflow_backend.load_database_models # noqa: F401 from spiffworkflow_backend.config import setup_config @@ -145,6 +146,15 @@ def configure_sentry(app: flask.app.Flask) -> None: from flask import Flask from sentry_sdk.integrations.flask import FlaskIntegration + def before_send(event: Any, hint: Any) -> Any: + """Before_send.""" + if "exc_info" in hint: + _exc_type, exc_value, _tb = hint["exc_info"] + # NotFound is mostly from web crawlers + if isinstance(exc_value, NotFound): + return None + return event + sentry_sample_rate = app.config.get("SENTRY_SAMPLE_RATE") if sentry_sample_rate is None: return @@ -153,10 +163,12 @@ def configure_sentry(app: flask.app.Flask) -> None: integrations=[ FlaskIntegration(), ], + environment=app.config["ENV_IDENTIFIER"], # Set traces_sample_rate to 1.0 to capture 100% # of transactions for performance monitoring. # We recommend adjusting this value in production. traces_sample_rate=float(sentry_sample_rate), + before_send=before_send, ) app = Flask(__name__) diff --git a/src/spiffworkflow_backend/config/demo.py b/src/spiffworkflow_backend/config/demo.py new file mode 100644 index 00000000..78e5dd20 --- /dev/null +++ b/src/spiffworkflow_backend/config/demo.py @@ -0,0 +1,9 @@ +"""Staging.""" +from os import environ + +GIT_COMMIT_ON_SAVE = True +GIT_COMMIT_USERNAME = "demo" +GIT_COMMIT_EMAIL = "demo@example.com" +SPIFFWORKFLOW_BACKEND_PERMISSIONS_FILE_NAME = environ.get( + "SPIFFWORKFLOW_BACKEND_PERMISSIONS_FILE_NAME", default="demo.yml" +) diff --git a/src/spiffworkflow_backend/config/permissions/demo.yml b/src/spiffworkflow_backend/config/permissions/demo.yml new file mode 100644 index 00000000..9fe49997 --- /dev/null +++ b/src/spiffworkflow_backend/config/permissions/demo.yml @@ -0,0 +1,28 @@ +groups: + admin: + users: [jakub, kb, alex, dan, mike, jason, amir, jarrad, elizabeth, jon] + + finance: + users: [harmeet, sasha] + + hr: + users: [manuchehr] + +permissions: + admin: + groups: [admin] + users: [] + allowed_permissions: [create, read, update, delete, list, instantiate] + uri: /* + + finance-admin: + groups: [finance] + users: [] + allowed_permissions: [create, read, update, delete] + uri: /v1.0/process-groups/finance/* + + read-all: + groups: [finance, hr, admin] + users: [] + allowed_permissions: [read] + uri: /* diff --git a/src/spiffworkflow_backend/routes/admin_blueprint/admin_blueprint.py b/src/spiffworkflow_backend/routes/admin_blueprint/admin_blueprint.py index 8a74c547..2e480f2a 100644 --- a/src/spiffworkflow_backend/routes/admin_blueprint/admin_blueprint.py +++ b/src/spiffworkflow_backend/routes/admin_blueprint/admin_blueprint.py @@ -7,7 +7,7 @@ from flask import redirect from flask import render_template from flask import request from flask import url_for -from werkzeug.wrappers.response import Response +from werkzeug.wrappers import Response from spiffworkflow_backend.services.process_instance_processor import ( ProcessInstanceProcessor, diff --git a/src/spiffworkflow_backend/routes/user.py b/src/spiffworkflow_backend/routes/user.py index 43806728..3099cb5c 100644 --- a/src/spiffworkflow_backend/routes/user.py +++ b/src/spiffworkflow_backend/routes/user.py @@ -12,7 +12,7 @@ from flask import g from flask import redirect from flask import request from flask_bpmn.api.api_error import ApiError -from werkzeug.wrappers.response import Response +from werkzeug.wrappers import Response from spiffworkflow_backend.models.user import UserModel from spiffworkflow_backend.services.authentication_service import ( diff --git a/src/spiffworkflow_backend/services/authentication_service.py b/src/spiffworkflow_backend/services/authentication_service.py index 73559590..666aad27 100644 --- a/src/spiffworkflow_backend/services/authentication_service.py +++ b/src/spiffworkflow_backend/services/authentication_service.py @@ -10,7 +10,7 @@ import requests from flask import current_app from flask import redirect from flask_bpmn.api.api_error import ApiError -from werkzeug.wrappers.response import Response +from werkzeug.wrappers import Response class AuthenticationProviderTypes(enum.Enum): diff --git a/tests/spiffworkflow_backend/helpers/base_test.py b/tests/spiffworkflow_backend/helpers/base_test.py index 9fca3d27..531e7110 100644 --- a/tests/spiffworkflow_backend/helpers/base_test.py +++ b/tests/spiffworkflow_backend/helpers/base_test.py @@ -13,7 +13,7 @@ from flask.testing import FlaskClient from flask_bpmn.api.api_error import ApiError from flask_bpmn.models.db import db from tests.spiffworkflow_backend.helpers.test_data import load_test_spec -from werkzeug.test import TestResponse +from werkzeug.test import TestResponse # type: ignore from spiffworkflow_backend.models.permission_assignment import Permission from spiffworkflow_backend.models.permission_target import PermissionTargetModel diff --git a/tests/spiffworkflow_backend/integration/test_secret_service.py b/tests/spiffworkflow_backend/integration/test_secret_service.py index 2261cc49..3cfc83a7 100644 --- a/tests/spiffworkflow_backend/integration/test_secret_service.py +++ b/tests/spiffworkflow_backend/integration/test_secret_service.py @@ -7,7 +7,7 @@ from flask.app import Flask from flask.testing import FlaskClient from flask_bpmn.api.api_error import ApiError from tests.spiffworkflow_backend.helpers.base_test import BaseTest -from werkzeug.test import TestResponse +from werkzeug.test import TestResponse # type: ignore from spiffworkflow_backend.models.process_model import ProcessModelInfo from spiffworkflow_backend.models.secret_model import SecretModel