From 553c93be271a3eef385fb49ba96b97844f128382 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Soko=C5=82owski?= Date: Mon, 30 Jan 2023 16:18:17 +0100 Subject: [PATCH 01/24] backend: avoid redundant steps in Dockerfile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use separate `base`, `setup` and `final` to avoid redundat steps. Avoid runnig `poetry` twice, and add `source` and `description`. Signed-off-by: Jakub SokoĊ‚owski --- Dockerfile | 43 +++++++++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/Dockerfile b/Dockerfile index e2d89beb..f05f1973 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,28 +1,35 @@ -FROM ghcr.io/sartography/python:3.11 +# Base image to share ENV vars that activate VENV. +FROM ghcr.io/sartography/python:3.11 AS base + +ENV VIRTUAL_ENV=/app/venv +RUN python3 -m venv $VIRTUAL_ENV +ENV PATH="$VIRTUAL_ENV/bin:$PATH" + +WORKDIR /app + +# Setup image for installing Python dependencies. +FROM base AS setup RUN pip install poetry RUN useradd _gunicorn --no-create-home --user-group -RUN apt-get update && \ - apt-get install -y -q \ - gcc libssl-dev \ - curl git-core libpq-dev \ - gunicorn3 default-mysql-client +RUN apt-get update \ + && apt-get install -y -q gcc libssl-dev libpq-dev -WORKDIR /app -COPY pyproject.toml poetry.lock /app/ +COPY . /app RUN poetry install --without dev -RUN set -xe \ - && apt-get remove -y gcc python3-dev libssl-dev \ - && apt-get autoremove -y \ - && apt-get clean -y \ - && rm -rf /var/lib/apt/lists/* +# Final image without setup dependencies. +FROM base AS final -COPY . /app/ +LABEL source="https://github.com/sartography/spiff-arena" +LABEL description="Software development platform for building, running, and monitoring executable diagrams" -# run poetry install again AFTER copying the app into the image -# otherwise it does not know what the main app module is -RUN poetry install --without dev +RUN apt-get update \ + && apt-get clean -y \ + && apt-get install -y -q curl git-core gunicorn3 default-mysql-client \ + && rm -rf /var/lib/apt/lists/* -CMD ./bin/boot_server_in_docker +COPY --from=setup /app /app + +ENTRYPOINT ["./bin/boot_server_in_docker"] From b0a05adccc0bb54d26a7af057cb69f6229bb32a4 Mon Sep 17 00:00:00 2001 From: Dan Date: Mon, 30 Jan 2023 13:09:23 -0500 Subject: [PATCH 02/24] Use the id_token, not the auth_token from the open id server for authentication with the front end. The auth_token should be kept safe, and not guranteeded to be a json token. --- src/spiffworkflow_backend/routes/user.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/spiffworkflow_backend/routes/user.py b/src/spiffworkflow_backend/routes/user.py index 976f7883..50c6f88a 100644 --- a/src/spiffworkflow_backend/routes/user.py +++ b/src/spiffworkflow_backend/routes/user.py @@ -96,7 +96,7 @@ def verify_token( ) if auth_token and "error" not in auth_token: tld = current_app.config["THREAD_LOCAL_DATA"] - tld.new_access_token = auth_token["access_token"] + tld.new_access_token = auth_token["id_token"] tld.new_id_token = auth_token["id_token"] # We have the user, but this code is a bit convoluted, and will later demand # a user_info object so it can look up the user. Sorry to leave this crap here. @@ -186,6 +186,7 @@ def set_new_access_token_in_cookie( ): domain_for_frontend_cookie = None + # fixme - we should not be passing the access token back to the client if hasattr(tld, "new_access_token") and tld.new_access_token: response.set_cookie( "access_token", tld.new_access_token, domain=domain_for_frontend_cookie @@ -254,7 +255,7 @@ def parse_id_token(token: str) -> Any: return json.loads(decoded) -def login_return(code: str, state: str, session_state: str) -> Optional[Response]: +def login_return(code: str, state: str, session_state: str = None) -> Optional[Response]: """Login_return.""" state_dict = ast.literal_eval(base64.b64decode(state).decode("utf-8")) state_redirect_url = state_dict["redirect_url"] @@ -269,12 +270,13 @@ def login_return(code: str, state: str, session_state: str) -> Optional[Response user_model = AuthorizationService.create_user_from_sign_in(user_info) g.user = user_model.id g.token = auth_token_object["id_token"] - AuthenticationService.store_refresh_token( - user_model.id, auth_token_object["refresh_token"] - ) + if "refresh_token" in auth_token_object: + AuthenticationService.store_refresh_token( + user_model.id, auth_token_object["refresh_token"] + ) redirect_url = state_redirect_url tld = current_app.config["THREAD_LOCAL_DATA"] - tld.new_access_token = auth_token_object["access_token"] + tld.new_access_token = auth_token_object["id_token"] tld.new_id_token = auth_token_object["id_token"] return redirect(redirect_url) From ba6d4c5f2d3f8808748a56fd8fff2afbdf260c61 Mon Sep 17 00:00:00 2001 From: Dan Date: Mon, 30 Jan 2023 16:50:43 -0500 Subject: [PATCH 03/24] Fix typing issue. --- src/spiffworkflow_backend/routes/user.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/spiffworkflow_backend/routes/user.py b/src/spiffworkflow_backend/routes/user.py index 50c6f88a..6873198a 100644 --- a/src/spiffworkflow_backend/routes/user.py +++ b/src/spiffworkflow_backend/routes/user.py @@ -255,7 +255,7 @@ def parse_id_token(token: str) -> Any: return json.loads(decoded) -def login_return(code: str, state: str, session_state: str = None) -> Optional[Response]: +def login_return(code: str, state: str, session_state: str = "") -> Optional[Response]: """Login_return.""" state_dict = ast.literal_eval(base64.b64decode(state).decode("utf-8")) state_redirect_url = state_dict["redirect_url"] From c586e0ea83412527be1695087d65a8a478b99c08 Mon Sep 17 00:00:00 2001 From: burnettk Date: Mon, 30 Jan 2023 18:40:03 -0500 Subject: [PATCH 04/24] allow overriding git related configs w/ env var and log permissions stuff on boot --- src/spiffworkflow_backend/config/__init__.py | 10 ++++++++-- src/spiffworkflow_backend/config/default.py | 2 ++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/spiffworkflow_backend/config/__init__.py b/src/spiffworkflow_backend/config/__init__.py index f9f19571..d7afbeb9 100644 --- a/src/spiffworkflow_backend/config/__init__.py +++ b/src/spiffworkflow_backend/config/__init__.py @@ -82,13 +82,19 @@ def setup_config(app: Flask) -> None: app.config.from_pyfile(f"{app.instance_path}/config.py", silent=True) app.config["PERMISSIONS_FILE_FULLPATH"] = None - if app.config["SPIFFWORKFLOW_BACKEND_PERMISSIONS_FILE_NAME"]: + permissions_file_name = app.config["SPIFFWORKFLOW_BACKEND_PERMISSIONS_FILE_NAME"] + if permissions_file_name is not None: app.config["PERMISSIONS_FILE_FULLPATH"] = os.path.join( app.root_path, "config", "permissions", - app.config["SPIFFWORKFLOW_BACKEND_PERMISSIONS_FILE_NAME"], + permissions_file_name, ) + print(f"base_permissions: loaded permissions file: {permissions_file_name}") + else: + print("base_permissions: no permissions file loaded") + + # unversioned (see .gitignore) config that can override everything and include secrets. # src/spiffworkflow_backend/config/secrets.py diff --git a/src/spiffworkflow_backend/config/default.py b/src/spiffworkflow_backend/config/default.py index 252b2b89..4f0a8296 100644 --- a/src/spiffworkflow_backend/config/default.py +++ b/src/spiffworkflow_backend/config/default.py @@ -69,6 +69,8 @@ GIT_BRANCH = environ.get("GIT_BRANCH") GIT_CLONE_URL_FOR_PUBLISHING = environ.get("GIT_CLONE_URL") GIT_COMMIT_ON_SAVE = environ.get("GIT_COMMIT_ON_SAVE", default="false") == "true" GIT_SSH_PRIVATE_KEY = environ.get("GIT_SSH_PRIVATE_KEY") +GIT_USERNAME = environ.get("GIT_USERNAME") +GIT_USER_EMAIL = environ.get("GIT_USER_EMAIL") # Datbase Configuration SPIFF_DATABASE_TYPE = environ.get( From 17fb81bcbda6540252e42d3f5e32d15dc97d8be8 Mon Sep 17 00:00:00 2001 From: burnettk Date: Tue, 31 Jan 2023 16:14:22 -0500 Subject: [PATCH 05/24] shuffle around Dockerfile to allow to work for background container --- Dockerfile | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index f05f1973..f4a8f8ec 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,6 +7,14 @@ ENV PATH="$VIRTUAL_ENV/bin:$PATH" WORKDIR /app +# base plus packages needed for deployment. Could just install these in final, but then we can't cache as much. +FROM base AS deployment + +RUN apt-get update \ + && apt-get clean -y \ + && apt-get install -y -q curl git-core gunicorn3 default-mysql-client \ + && rm -rf /var/lib/apt/lists/* + # Setup image for installing Python dependencies. FROM base AS setup @@ -20,16 +28,11 @@ COPY . /app RUN poetry install --without dev # Final image without setup dependencies. -FROM base AS final +FROM deployment AS final LABEL source="https://github.com/sartography/spiff-arena" LABEL description="Software development platform for building, running, and monitoring executable diagrams" -RUN apt-get update \ - && apt-get clean -y \ - && apt-get install -y -q curl git-core gunicorn3 default-mysql-client \ - && rm -rf /var/lib/apt/lists/* - COPY --from=setup /app /app -ENTRYPOINT ["./bin/boot_server_in_docker"] +CMD ["./bin/boot_server_in_docker"] From 8a98e8d9be7bf7c863b8d440d948707dcdfe540a Mon Sep 17 00:00:00 2001 From: burnettk Date: Tue, 31 Jan 2023 17:11:11 -0500 Subject: [PATCH 06/24] folks who can start instances can also view their logs --- .../services/authorization_service.py | 8 +++++++- .../scripts/test_get_all_permissions.py | 5 +++++ .../unit/test_authorization_service.py | 8 ++++++++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/spiffworkflow_backend/services/authorization_service.py b/src/spiffworkflow_backend/services/authorization_service.py index 793a3f9b..19f9f418 100644 --- a/src/spiffworkflow_backend/services/authorization_service.py +++ b/src/spiffworkflow_backend/services/authorization_service.py @@ -551,7 +551,9 @@ class AuthorizationService: permissions_to_assign: list[PermissionToAssign] = [] - # we were thinking that if you can start an instance, you ought to be able to view your own instances. + # we were thinking that if you can start an instance, you ought to be able to: + # 1. view your own instances. + # 2. view the logs for these instances. if permission_set == "start": target_uri = f"/process-instances/{process_related_path_segment}" permissions_to_assign.append( @@ -561,6 +563,10 @@ class AuthorizationService: permissions_to_assign.append( PermissionToAssign(permission="read", target_uri=target_uri) ) + target_uri = f"/logs/{process_related_path_segment}" + permissions_to_assign.append( + PermissionToAssign(permission="read", target_uri=target_uri) + ) else: if permission_set == "all": diff --git a/tests/spiffworkflow_backend/scripts/test_get_all_permissions.py b/tests/spiffworkflow_backend/scripts/test_get_all_permissions.py index b31c7228..95d15fbf 100644 --- a/tests/spiffworkflow_backend/scripts/test_get_all_permissions.py +++ b/tests/spiffworkflow_backend/scripts/test_get_all_permissions.py @@ -41,6 +41,11 @@ class TestGetAllPermissions(BaseTest): ) expected_permissions = [ + { + "group_identifier": "my_test_group", + "uri": "/logs/hey:group:*", + "permissions": ["read"], + }, { "group_identifier": "my_test_group", "uri": "/process-instances/hey:group:*", diff --git a/tests/spiffworkflow_backend/unit/test_authorization_service.py b/tests/spiffworkflow_backend/unit/test_authorization_service.py index 2736693e..d414616c 100644 --- a/tests/spiffworkflow_backend/unit/test_authorization_service.py +++ b/tests/spiffworkflow_backend/unit/test_authorization_service.py @@ -197,6 +197,10 @@ class TestAuthorizationService(BaseTest): ) -> None: """Test_explode_permissions_start_on_process_group.""" expected_permissions = [ + ( + "/logs/some-process-group:some-process-model:*", + "read", + ), ( "/process-instances/for-me/some-process-group:some-process-model:*", "read", @@ -255,6 +259,10 @@ class TestAuthorizationService(BaseTest): ) -> None: """Test_explode_permissions_start_on_process_model.""" expected_permissions = [ + ( + "/logs/some-process-group:some-process-model/*", + "read", + ), ( "/process-instances/for-me/some-process-group:some-process-model/*", "read", From a23a0700bbdeed0083e00b7485b001cf27e5d0ba Mon Sep 17 00:00:00 2001 From: burnettk Date: Tue, 31 Jan 2023 22:30:15 -0500 Subject: [PATCH 07/24] more sentry performance tracing --- src/spiffworkflow_backend/__init__.py | 28 +++++++++++++++++ src/spiffworkflow_backend/config/__init__.py | 2 -- .../routes/tasks_controller.py | 30 +++++++++++++++++-- 3 files changed, 56 insertions(+), 4 deletions(-) diff --git a/src/spiffworkflow_backend/__init__.py b/src/spiffworkflow_backend/__init__.py index 46f82581..341cfac8 100644 --- a/src/spiffworkflow_backend/__init__.py +++ b/src/spiffworkflow_backend/__init__.py @@ -157,6 +157,29 @@ def get_hacked_up_app_for_script() -> flask.app.Flask: return app +def traces_sampler(sampling_context: Any) -> Any: + # always inherit + if sampling_context["parent_sampled"] is not None: + return sampling_context["parent_sampled"] + + if "wsgi_environ" in sampling_context: + wsgi_environ = sampling_context["wsgi_environ"] + path_info = wsgi_environ.get("PATH_INFO") + request_method = wsgi_environ.get("REQUEST_METHOD") + + # tasks_controller.task_submit + # this is the current pain point as of 31 jan 2023. + if ( + path_info + and path_info.startswith("/v1.0/tasks/") + and request_method == "PUT" + ): + return 1 + + # Default sample rate for all others (replaces traces_sample_rate) + return 0.01 + + def configure_sentry(app: flask.app.Flask) -> None: """Configure_sentry.""" import sentry_sdk @@ -193,5 +216,10 @@ def configure_sentry(app: flask.app.Flask) -> None: # of transactions for performance monitoring. # We recommend adjusting this value to less than 1(00%) in production. traces_sample_rate=float(sentry_traces_sample_rate), + traces_sampler=traces_sampler, + # The profiles_sample_rate setting is relative to the traces_sample_rate setting. + _experiments={ + "profiles_sample_rate": 1, + }, before_send=before_send, ) diff --git a/src/spiffworkflow_backend/config/__init__.py b/src/spiffworkflow_backend/config/__init__.py index d7afbeb9..64c7e2c1 100644 --- a/src/spiffworkflow_backend/config/__init__.py +++ b/src/spiffworkflow_backend/config/__init__.py @@ -94,8 +94,6 @@ def setup_config(app: Flask) -> None: else: print("base_permissions: no permissions file loaded") - - # unversioned (see .gitignore) config that can override everything and include secrets. # src/spiffworkflow_backend/config/secrets.py app.config.from_pyfile(os.path.join("config", "secrets.py"), silent=True) diff --git a/src/spiffworkflow_backend/routes/tasks_controller.py b/src/spiffworkflow_backend/routes/tasks_controller.py index 8ee9f53d..fcc0dba0 100644 --- a/src/spiffworkflow_backend/routes/tasks_controller.py +++ b/src/spiffworkflow_backend/routes/tasks_controller.py @@ -10,6 +10,7 @@ from typing import Union import flask.wrappers import jinja2 +import sentry_sdk from flask import current_app from flask import g from flask import jsonify @@ -326,13 +327,12 @@ def process_data_show( ) -def task_submit( +def task_submit_shared( process_instance_id: int, task_id: str, body: Dict[str, Any], terminate_loop: bool = False, ) -> flask.wrappers.Response: - """Task_submit_user_data.""" principal = _find_principal_or_raise() process_instance = _find_process_instance_by_id_or_raise(process_instance_id) if not process_instance.can_submit_task(): @@ -417,6 +417,32 @@ def task_submit( return Response(json.dumps({"ok": True}), status=202, mimetype="application/json") +def task_submit( + process_instance_id: int, + task_id: str, + body: Dict[str, Any], + terminate_loop: bool = False, +) -> flask.wrappers.Response: + """Task_submit_user_data.""" + sentry_op = "controller_action" + sentry_transaction_name = "tasks_controller.task_submit" + transaction = sentry_sdk.Hub.current.scope.transaction + if transaction is None: + current_app.logger.debug( + "transaction was None. pretty sure this never happens." + ) + with sentry_sdk.start_transaction(op=sentry_op, name=sentry_transaction_name): + return task_submit_shared( + process_instance_id, task_id, body, terminate_loop + ) + else: + current_app.logger.debug("transaction existed.") + with transaction.start_child(op=sentry_op, description=sentry_transaction_name): + return task_submit_shared( + process_instance_id, task_id, body, terminate_loop + ) + + def _get_tasks( processes_started_by_user: bool = True, has_lane_assignment_id: bool = True, From 5d50ee8ea6278a45c1fe2521e4cd8b93feb836a9 Mon Sep 17 00:00:00 2001 From: burnettk Date: Wed, 1 Feb 2023 07:45:48 -0500 Subject: [PATCH 08/24] more spans to track performance --- .../routes/tasks_controller.py | 25 +++++++++---------- .../services/process_instance_service.py | 6 +++-- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/src/spiffworkflow_backend/routes/tasks_controller.py b/src/spiffworkflow_backend/routes/tasks_controller.py index fcc0dba0..feb9218e 100644 --- a/src/spiffworkflow_backend/routes/tasks_controller.py +++ b/src/spiffworkflow_backend/routes/tasks_controller.py @@ -380,15 +380,16 @@ def task_submit_shared( ) ) - processor.lock_process_instance("Web") - ProcessInstanceService.complete_form_task( - processor=processor, - spiff_task=spiff_task, - data=body, - user=g.user, - human_task=human_task, - ) - processor.unlock_process_instance("Web") + with sentry_sdk.start_span(op="task", description="complete_form_task"): + processor.lock_process_instance("Web") + ProcessInstanceService.complete_form_task( + processor=processor, + spiff_task=spiff_task, + data=body, + user=g.user, + human_task=human_task, + ) + processor.unlock_process_instance("Web") # If we need to update all tasks, then get the next ready task and if it a multi-instance with the same # task spec, complete that form as well. @@ -428,15 +429,13 @@ def task_submit( sentry_transaction_name = "tasks_controller.task_submit" transaction = sentry_sdk.Hub.current.scope.transaction if transaction is None: - current_app.logger.debug( - "transaction was None. pretty sure this never happens." - ) + current_app.logger.info("transaction was None. pretty sure this never happens.") with sentry_sdk.start_transaction(op=sentry_op, name=sentry_transaction_name): return task_submit_shared( process_instance_id, task_id, body, terminate_loop ) else: - current_app.logger.debug("transaction existed.") + current_app.logger.info("transaction existed.") with transaction.start_child(op=sentry_op, description=sentry_transaction_name): return task_submit_shared( process_instance_id, task_id, body, terminate_loop diff --git a/src/spiffworkflow_backend/services/process_instance_service.py b/src/spiffworkflow_backend/services/process_instance_service.py index 9b07ce1f..63c53a21 100644 --- a/src/spiffworkflow_backend/services/process_instance_service.py +++ b/src/spiffworkflow_backend/services/process_instance_service.py @@ -4,6 +4,7 @@ from typing import Any from typing import List from typing import Optional +import sentry_sdk from flask import current_app from SpiffWorkflow.task import Task as SpiffTask # type: ignore @@ -234,8 +235,9 @@ class ProcessInstanceService: # ProcessInstanceService.post_process_form(spiff_task) # some properties may update the data store. processor.complete_task(spiff_task, human_task, user=user) - # maybe move this out once we have the interstitial page since this is here just so we can get the next human task - processor.do_engine_steps(save=True) + with sentry_sdk.start_span(op="task", description="backend_do_engine_steps"): + # maybe move this out once we have the interstitial page since this is here just so we can get the next human task + processor.do_engine_steps(save=True) @staticmethod def extract_form_data(latest_data: dict, task: SpiffTask) -> dict: From 7ffcded9dbb9991a35f2e28c42c540d47152f041 Mon Sep 17 00:00:00 2001 From: burnettk Date: Wed, 1 Feb 2023 07:53:35 -0500 Subject: [PATCH 09/24] avoid poetry installing deps when we have them cached if they do not change --- Dockerfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Dockerfile b/Dockerfile index f4a8f8ec..d7a4b034 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,6 +24,11 @@ RUN useradd _gunicorn --no-create-home --user-group RUN apt-get update \ && apt-get install -y -q gcc libssl-dev libpq-dev +# poetry install takes a long time and can be cached if dependencies don't change, +# so that's why we tolerate running it twice. +COPY pyproject.toml poetry.lock /app/ +RUN poetry install --without dev + COPY . /app RUN poetry install --without dev From 1368d71cc969ac60eb6b3384549ad9b46200ebba Mon Sep 17 00:00:00 2001 From: burnettk Date: Wed, 1 Feb 2023 13:30:45 -0500 Subject: [PATCH 10/24] get some more insight into connector proxy timings --- .../services/service_task_service.py | 33 ++++++++++--------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/src/spiffworkflow_backend/services/service_task_service.py b/src/spiffworkflow_backend/services/service_task_service.py index 674ad54d..c5401104 100644 --- a/src/spiffworkflow_backend/services/service_task_service.py +++ b/src/spiffworkflow_backend/services/service_task_service.py @@ -3,6 +3,7 @@ import json from typing import Any import requests +import sentry_sdk from flask import current_app from flask import g @@ -45,27 +46,27 @@ class ServiceTaskDelegate: @staticmethod def call_connector(name: str, bpmn_params: Any, task_data: Any) -> str: """Calls a connector via the configured proxy.""" - params = { - k: ServiceTaskDelegate.check_prefixes(v["value"]) - for k, v in bpmn_params.items() - } - params["spiff__task_data"] = task_data + call_url = f"{connector_proxy_url()}/v1/do/{name}" + with sentry_sdk.start_transaction(op="call-connector", name=call_url): + params = { + k: ServiceTaskDelegate.check_prefixes(v["value"]) + for k, v in bpmn_params.items() + } + params["spiff__task_data"] = task_data - proxied_response = requests.post( - f"{connector_proxy_url()}/v1/do/{name}", json=params - ) + proxied_response = requests.post(call_url, json=params) - parsed_response = json.loads(proxied_response.text) + parsed_response = json.loads(proxied_response.text) - if "refreshed_token_set" not in parsed_response: - return proxied_response.text + if "refreshed_token_set" not in parsed_response: + return proxied_response.text - secret_key = parsed_response["auth"] - refreshed_token_set = json.dumps(parsed_response["refreshed_token_set"]) - user_id = g.user.id if UserService.has_user() else None - SecretService().update_secret(secret_key, refreshed_token_set, user_id) + secret_key = parsed_response["auth"] + refreshed_token_set = json.dumps(parsed_response["refreshed_token_set"]) + user_id = g.user.id if UserService.has_user() else None + SecretService().update_secret(secret_key, refreshed_token_set, user_id) - return json.dumps(parsed_response["api_response"]) + return json.dumps(parsed_response["api_response"]) class ServiceTaskService: From c50744a189757fba8e7c2178c623895251260fd6 Mon Sep 17 00:00:00 2001 From: burnettk Date: Wed, 1 Feb 2023 13:44:12 -0500 Subject: [PATCH 11/24] there is no need to ever sentry_sdk.start_transaction because the flask integration does that --- .../routes/tasks_controller.py | 19 ++++--------------- .../services/service_task_service.py | 2 +- 2 files changed, 5 insertions(+), 16 deletions(-) diff --git a/src/spiffworkflow_backend/routes/tasks_controller.py b/src/spiffworkflow_backend/routes/tasks_controller.py index feb9218e..2879c120 100644 --- a/src/spiffworkflow_backend/routes/tasks_controller.py +++ b/src/spiffworkflow_backend/routes/tasks_controller.py @@ -425,21 +425,10 @@ def task_submit( terminate_loop: bool = False, ) -> flask.wrappers.Response: """Task_submit_user_data.""" - sentry_op = "controller_action" - sentry_transaction_name = "tasks_controller.task_submit" - transaction = sentry_sdk.Hub.current.scope.transaction - if transaction is None: - current_app.logger.info("transaction was None. pretty sure this never happens.") - with sentry_sdk.start_transaction(op=sentry_op, name=sentry_transaction_name): - return task_submit_shared( - process_instance_id, task_id, body, terminate_loop - ) - else: - current_app.logger.info("transaction existed.") - with transaction.start_child(op=sentry_op, description=sentry_transaction_name): - return task_submit_shared( - process_instance_id, task_id, body, terminate_loop - ) + with sentry_sdk.start_span( + op="controller_action", description="tasks_controller.task_submit" + ): + return task_submit_shared(process_instance_id, task_id, body, terminate_loop) def _get_tasks( diff --git a/src/spiffworkflow_backend/services/service_task_service.py b/src/spiffworkflow_backend/services/service_task_service.py index c5401104..37af3956 100644 --- a/src/spiffworkflow_backend/services/service_task_service.py +++ b/src/spiffworkflow_backend/services/service_task_service.py @@ -47,7 +47,7 @@ class ServiceTaskDelegate: def call_connector(name: str, bpmn_params: Any, task_data: Any) -> str: """Calls a connector via the configured proxy.""" call_url = f"{connector_proxy_url()}/v1/do/{name}" - with sentry_sdk.start_transaction(op="call-connector", name=call_url): + with sentry_sdk.start_span(op="call-connector", description=call_url): params = { k: ServiceTaskDelegate.check_prefixes(v["value"]) for k, v in bpmn_params.items() From 6d18bd234ef0e17ca25d5e97ca40f254d2935f89 Mon Sep 17 00:00:00 2001 From: burnettk Date: Wed, 1 Feb 2023 17:06:34 -0500 Subject: [PATCH 12/24] bulk insert logs for performance improvement --- src/spiffworkflow_backend/services/logging_service.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/spiffworkflow_backend/services/logging_service.py b/src/spiffworkflow_backend/services/logging_service.py index 6a60944e..9981e1eb 100644 --- a/src/spiffworkflow_backend/services/logging_service.py +++ b/src/spiffworkflow_backend/services/logging_service.py @@ -240,5 +240,8 @@ class DBHandler(logging.Handler): "spiff_step": spiff_step, } ) - if len(self.logs) % 1 == 0: + # so at some point we are going to insert logs. + # we don't want to insert on every log, so we will insert every 100 logs, which is just about as fast as inserting + # on every 1,000 logs. if we get deadlocks in the database, this can be changed to 1 in order to insert on every log. + if len(self.logs) % 100 == 0: self.bulk_insert_logs() From 650b91ed5210760e32b606bbd9e0080f6b759518 Mon Sep 17 00:00:00 2001 From: burnettk Date: Thu, 2 Feb 2023 09:54:19 -0500 Subject: [PATCH 13/24] add keycloak users --- .../realm_exports/spiffworkflow-realm.json | 208 ++++++++++++++++-- keycloak/test_user_lists/status | 8 + 2 files changed, 192 insertions(+), 24 deletions(-) diff --git a/keycloak/realm_exports/spiffworkflow-realm.json b/keycloak/realm_exports/spiffworkflow-realm.json index a32acf00..634caef7 100644 --- a/keycloak/realm_exports/spiffworkflow-realm.json +++ b/keycloak/realm_exports/spiffworkflow-realm.json @@ -854,6 +854,46 @@ "realmRoles" : [ "default-roles-spiffworkflow" ], "notBefore" : 0, "groups" : [ ] + }, { + "id" : "672167fd-ae79-47a7-8429-f3bb1bd4ee55", + "createdTimestamp" : 1675349217829, + "username" : "infra1.sme", + "enabled" : true, + "totp" : false, + "emailVerified" : false, + "email" : "infra1.sme@status.im", + "credentials" : [ { + "id" : "bd5843bf-98cc-4891-ab03-693a5d69078b", + "type" : "password", + "createdDate" : 1675349217863, + "secretData" : "{\"value\":\"A78sm/+e2x/N/3A7Pk05eKhfANp+ZO9BQA3LYMwpzQ5KK2D/Ot8d1plOnqMT61rTnnCgxP8dtlA6/Ws61CMTYg==\",\"salt\":\"XOOknamJPwXD1LDj6LEodA==\",\"additionalParameters\":{}}", + "credentialData" : "{\"hashIterations\":27500,\"algorithm\":\"pbkdf2-sha256\",\"additionalParameters\":{}}" + } ], + "disableableCredentialTypes" : [ ], + "requiredActions" : [ ], + "realmRoles" : [ "default-roles-spiffworkflow" ], + "notBefore" : 0, + "groups" : [ ] + }, { + "id" : "40891b68-121f-4fdb-86c0-0f52836d7e65", + "createdTimestamp" : 1675349217890, + "username" : "infra2.sme", + "enabled" : true, + "totp" : false, + "emailVerified" : false, + "email" : "infra2.sme@status.im", + "credentials" : [ { + "id" : "7e9927e2-ef7f-4247-b663-1f59147a9066", + "type" : "password", + "createdDate" : 1675349217926, + "secretData" : "{\"value\":\"j4M9u8p9FDCitGpb7JXM9JWFVGvBu7R2TOYG79c+Witl7gfWppues9fFzhlFyXgC78v6diHoQ4LwCwJGJS3loQ==\",\"salt\":\"H+i8qv6ulrBEZla/v8gDDw==\",\"additionalParameters\":{}}", + "credentialData" : "{\"hashIterations\":27500,\"algorithm\":\"pbkdf2-sha256\",\"additionalParameters\":{}}" + } ], + "disableableCredentialTypes" : [ ], + "requiredActions" : [ ], + "realmRoles" : [ "default-roles-spiffworkflow" ], + "notBefore" : 0, + "groups" : [ ] }, { "id" : "1561518b-c327-491e-9db3-23c2b5394104", "createdTimestamp" : 1669303773974, @@ -1043,6 +1083,46 @@ "realmRoles" : [ "default-roles-spiffworkflow" ], "notBefore" : 0, "groups" : [ ] + }, { + "id" : "eff82d12-9a67-4002-b3c5-37811bd45199", + "createdTimestamp" : 1675349217585, + "username" : "legal.program-lead.sme", + "enabled" : true, + "totp" : false, + "emailVerified" : false, + "email" : "legal.program-lead.sme@status.im", + "credentials" : [ { + "id" : "933e3fc4-398a-46c3-bc4d-783ab29a0a5b", + "type" : "password", + "createdDate" : 1675349217655, + "secretData" : "{\"value\":\"x2M9khnGK+VCykoWbZKEcHNv5QMAcumqLa7+o+STJV8UYt7BobSBn7w1r3cbyYlvkgoWIglG8S2nLDFFb6hAQg==\",\"salt\":\"/lQYRrsUY1BxNUOZSKaZwA==\",\"additionalParameters\":{}}", + "credentialData" : "{\"hashIterations\":27500,\"algorithm\":\"pbkdf2-sha256\",\"additionalParameters\":{}}" + } ], + "disableableCredentialTypes" : [ ], + "requiredActions" : [ ], + "realmRoles" : [ "default-roles-spiffworkflow" ], + "notBefore" : 0, + "groups" : [ ] + }, { + "id" : "8cd6feba-5ca6-4cfb-bc1a-a52c80595783", + "createdTimestamp" : 1675349217698, + "username" : "legal.project-lead.sme", + "enabled" : true, + "totp" : false, + "emailVerified" : false, + "email" : "legal.project-lead.sme@status.im", + "credentials" : [ { + "id" : "908f858c-d3cd-47a9-b611-a1d48f0247e5", + "type" : "password", + "createdDate" : 1675349217733, + "secretData" : "{\"value\":\"r53SXu0dp6FrSJAVLHYrfwSKPZY9OKHfHBuJDEE2DCbZiQRH77C4sZWfUwbu/6OOhTtiBEe7gz2DQpimIDY4RQ==\",\"salt\":\"+g/OXXJEMkQiahmjSylAkw==\",\"additionalParameters\":{}}", + "credentialData" : "{\"hashIterations\":27500,\"algorithm\":\"pbkdf2-sha256\",\"additionalParameters\":{}}" + } ], + "disableableCredentialTypes" : [ ], + "requiredActions" : [ ], + "realmRoles" : [ "default-roles-spiffworkflow" ], + "notBefore" : 0, + "groups" : [ ] }, { "id" : "2a3176a0-8dd5-4223-a3e1-3cac4134e474", "createdTimestamp" : 1674148695030, @@ -1063,6 +1143,26 @@ "realmRoles" : [ "default-roles-spiffworkflow" ], "notBefore" : 0, "groups" : [ ] + }, { + "id" : "3d62ca4e-88bc-4302-89c1-8741c771147e", + "createdTimestamp" : 1675349217762, + "username" : "legal1.sme", + "enabled" : true, + "totp" : false, + "emailVerified" : false, + "email" : "legal1.sme@status.im", + "credentials" : [ { + "id" : "b774d46d-a3e8-417f-97c6-2d2102a54b0b", + "type" : "password", + "createdDate" : 1675349217799, + "secretData" : "{\"value\":\"PF21YsnIoYZLJFT/y1i2FV4OmaQj8dRsalZ9R2PK6t/jKze3ds4k+I7WVe4h2H0hMB9fo9cSQ7kt2ygxfEBheg==\",\"salt\":\"5sOkSXzRSgNz7lHfUbKzdQ==\",\"additionalParameters\":{}}", + "credentialData" : "{\"hashIterations\":27500,\"algorithm\":\"pbkdf2-sha256\",\"additionalParameters\":{}}" + } ], + "disableableCredentialTypes" : [ ], + "requiredActions" : [ ], + "realmRoles" : [ "default-roles-spiffworkflow" ], + "notBefore" : 0, + "groups" : [ ] }, { "id" : "6f5bfa09-7494-4a2f-b871-cf327048cac7", "createdTimestamp" : 1665517010600, @@ -1225,6 +1325,46 @@ "realmRoles" : [ "default-roles-spiffworkflow" ], "notBefore" : 0, "groups" : [ ] + }, { + "id" : "ace0432f-1818-4210-8bcf-15533abfb3ce", + "createdTimestamp" : 1675349217958, + "username" : "security.program-lead.sme", + "enabled" : true, + "totp" : false, + "emailVerified" : false, + "email" : "security.program-lead.sme@status.im", + "credentials" : [ { + "id" : "602512dd-b24f-458c-9cef-7271bd8177bc", + "type" : "password", + "createdDate" : 1675349217993, + "secretData" : "{\"value\":\"vUb+t9ukHz3oHGUxaYUP34riZrshZU4c3iWpHB0OzI3y0ggCeT9xFEcmrwdkfilkKvCBJxLswlirWmgnmxZH0w==\",\"salt\":\"0hzZkDK4hPH5xgR1TpyG1Q==\",\"additionalParameters\":{}}", + "credentialData" : "{\"hashIterations\":27500,\"algorithm\":\"pbkdf2-sha256\",\"additionalParameters\":{}}" + } ], + "disableableCredentialTypes" : [ ], + "requiredActions" : [ ], + "realmRoles" : [ "default-roles-spiffworkflow" ], + "notBefore" : 0, + "groups" : [ ] + }, { + "id" : "6272ac80-1d79-4e3c-a5c1-b31660560318", + "createdTimestamp" : 1675349218020, + "username" : "security.project-lead.sme", + "enabled" : true, + "totp" : false, + "emailVerified" : false, + "email" : "security.project-lead.sme@status.im", + "credentials" : [ { + "id" : "eb7673bf-50f1-40af-927b-162f536f6187", + "type" : "password", + "createdDate" : 1675349218054, + "secretData" : "{\"value\":\"E1eLmC7hCcv7I5X30TfMvpZv3MtHH+rVhgLrZnBJSUvsrXmRkHWScJ/POHQLwUgCLJeU/lKDP/f0TdO2PvHiow==\",\"salt\":\"dWM5XJIR7m/eZ0YlHmuC3A==\",\"additionalParameters\":{}}", + "credentialData" : "{\"hashIterations\":27500,\"algorithm\":\"pbkdf2-sha256\",\"additionalParameters\":{}}" + } ], + "disableableCredentialTypes" : [ ], + "requiredActions" : [ ], + "realmRoles" : [ "default-roles-spiffworkflow" ], + "notBefore" : 0, + "groups" : [ ] }, { "id" : "74374cda-1516-48e5-9ef2-1fd7bcee84d3", "createdTimestamp" : 1674148695088, @@ -1245,6 +1385,26 @@ "realmRoles" : [ "default-roles-spiffworkflow" ], "notBefore" : 0, "groups" : [ ] + }, { + "id" : "98faab0c-d2af-4794-8491-03dad5f30c63", + "createdTimestamp" : 1675349218087, + "username" : "security1.sme", + "enabled" : true, + "totp" : false, + "emailVerified" : false, + "email" : "security1.sme@status.im", + "credentials" : [ { + "id" : "37bd6b9b-015b-4790-8a4f-883c47035bc4", + "type" : "password", + "createdDate" : 1675349218122, + "secretData" : "{\"value\":\"BJP9K4qIdnaDnE3meM2GLWMFdSJryxcZovtKDlZNaQXfSUH3X1mOJfaLXQsuTWJzSMIow8XZ5+ye47ZNabLCaQ==\",\"salt\":\"BqD7jPpdB7PzU6QTN5dpMA==\",\"additionalParameters\":{}}", + "credentialData" : "{\"hashIterations\":27500,\"algorithm\":\"pbkdf2-sha256\",\"additionalParameters\":{}}" + } ], + "disableableCredentialTypes" : [ ], + "requiredActions" : [ ], + "realmRoles" : [ "default-roles-spiffworkflow" ], + "notBefore" : 0, + "groups" : [ ] }, { "id" : "487d3a85-89dd-4839-957a-c3f6d70551f6", "createdTimestamp" : 1657115173081, @@ -2514,7 +2674,7 @@ "subType" : "authenticated", "subComponents" : { }, "config" : { - "allowed-protocol-mapper-types" : [ "oidc-usermodel-attribute-mapper", "saml-user-attribute-mapper", "oidc-usermodel-property-mapper", "oidc-sha256-pairwise-sub-mapper", "saml-role-list-mapper", "saml-user-property-mapper", "oidc-address-mapper", "oidc-full-name-mapper" ] + "allowed-protocol-mapper-types" : [ "oidc-address-mapper", "oidc-sha256-pairwise-sub-mapper", "saml-role-list-mapper", "oidc-usermodel-attribute-mapper", "saml-user-property-mapper", "saml-user-attribute-mapper", "oidc-usermodel-property-mapper", "oidc-full-name-mapper" ] } }, { "id" : "d68e938d-dde6-47d9-bdc8-8e8523eb08cd", @@ -2532,7 +2692,7 @@ "subType" : "anonymous", "subComponents" : { }, "config" : { - "allowed-protocol-mapper-types" : [ "oidc-full-name-mapper", "saml-role-list-mapper", "oidc-usermodel-property-mapper", "saml-user-attribute-mapper", "oidc-sha256-pairwise-sub-mapper", "saml-user-property-mapper", "oidc-usermodel-attribute-mapper", "oidc-address-mapper" ] + "allowed-protocol-mapper-types" : [ "saml-role-list-mapper", "oidc-sha256-pairwise-sub-mapper", "oidc-usermodel-attribute-mapper", "saml-user-attribute-mapper", "oidc-full-name-mapper", "saml-user-property-mapper", "oidc-address-mapper", "oidc-usermodel-property-mapper" ] } }, { "id" : "3854361d-3fe5-47fb-9417-a99592e3dc5c", @@ -2622,7 +2782,7 @@ "internationalizationEnabled" : false, "supportedLocales" : [ ], "authenticationFlows" : [ { - "id" : "a91920d9-792e-486f-9a02-49fe00857ce5", + "id" : "feafc299-fede-4880-9e23-eb81aca22808", "alias" : "Account verification options", "description" : "Method with which to verity the existing account", "providerId" : "basic-flow", @@ -2644,7 +2804,7 @@ "userSetupAllowed" : false } ] }, { - "id" : "6b8f504c-39fb-4608-9223-52deb5ae0dfe", + "id" : "ce7904d0-9182-49a2-aa71-a7b43e21f3ac", "alias" : "Authentication Options", "description" : "Authentication options.", "providerId" : "basic-flow", @@ -2673,7 +2833,7 @@ "userSetupAllowed" : false } ] }, { - "id" : "ac4dd6f3-43b2-4212-90eb-4df7c9a6a0bc", + "id" : "d9c6909a-5cc1-4ddf-b297-dbfcf6e609a6", "alias" : "Browser - Conditional OTP", "description" : "Flow to determine if the OTP is required for the authentication", "providerId" : "basic-flow", @@ -2695,7 +2855,7 @@ "userSetupAllowed" : false } ] }, { - "id" : "726b4a58-cb78-4105-a34c-3e4404c74362", + "id" : "083a589e-a486-42b6-ae73-1ec983967ff5", "alias" : "Direct Grant - Conditional OTP", "description" : "Flow to determine if the OTP is required for the authentication", "providerId" : "basic-flow", @@ -2717,7 +2877,7 @@ "userSetupAllowed" : false } ] }, { - "id" : "be1b5f5d-b80f-46a6-804b-bce20e2de246", + "id" : "7f0248b0-2d51-4175-9fd2-52b606a39e26", "alias" : "First broker login - Conditional OTP", "description" : "Flow to determine if the OTP is required for the authentication", "providerId" : "basic-flow", @@ -2739,7 +2899,7 @@ "userSetupAllowed" : false } ] }, { - "id" : "ff5097d8-818a-4176-8512-caf9d81eb6db", + "id" : "44465f1f-c700-4ec0-a234-d95c994c9e25", "alias" : "Handle Existing Account", "description" : "Handle what to do if there is existing account with same email/username like authenticated identity provider", "providerId" : "basic-flow", @@ -2761,7 +2921,7 @@ "userSetupAllowed" : false } ] }, { - "id" : "b9ecf989-e87b-45c0-a440-bce46b473dec", + "id" : "8cf09055-5b98-4fc8-b867-3dffacdec21b", "alias" : "Reset - Conditional OTP", "description" : "Flow to determine if the OTP should be reset or not. Set to REQUIRED to force.", "providerId" : "basic-flow", @@ -2783,7 +2943,7 @@ "userSetupAllowed" : false } ] }, { - "id" : "4554310c-e125-4834-a84e-53bbec7a79d6", + "id" : "16b50b3e-4240-4f49-a85e-1bfd40def300", "alias" : "User creation or linking", "description" : "Flow for the existing/non-existing user alternatives", "providerId" : "basic-flow", @@ -2806,7 +2966,7 @@ "userSetupAllowed" : false } ] }, { - "id" : "204549aa-c931-45a2-b2f0-1a5a0c724935", + "id" : "2aa981ae-d67e-49fb-95a4-91de1e5ab724", "alias" : "Verify Existing Account by Re-authentication", "description" : "Reauthentication of existing account", "providerId" : "basic-flow", @@ -2828,7 +2988,7 @@ "userSetupAllowed" : false } ] }, { - "id" : "d02f58b1-6469-46ea-a348-d923b5aa9727", + "id" : "cf8406f7-09c3-4614-a898-99c9d66746f6", "alias" : "browser", "description" : "browser based authentication", "providerId" : "basic-flow", @@ -2864,7 +3024,7 @@ "userSetupAllowed" : false } ] }, { - "id" : "7ef6a658-be09-4b81-91ac-f21dc80b0841", + "id" : "e1ec7d6e-7612-4c5b-afce-c7f4fddbf6ec", "alias" : "clients", "description" : "Base authentication for clients", "providerId" : "client-flow", @@ -2900,7 +3060,7 @@ "userSetupAllowed" : false } ] }, { - "id" : "f7f2eeab-6455-4a18-a98d-b1a5f04e35fb", + "id" : "f5862b09-6e01-4c88-b44e-26dc59d71b80", "alias" : "direct grant", "description" : "OpenID Connect Resource Owner Grant", "providerId" : "basic-flow", @@ -2929,7 +3089,7 @@ "userSetupAllowed" : false } ] }, { - "id" : "c44389c2-08b2-4adb-a6e9-e41006cb20c7", + "id" : "7caa8611-8b13-437e-83b2-556899b5444f", "alias" : "docker auth", "description" : "Used by Docker clients to authenticate against the IDP", "providerId" : "basic-flow", @@ -2944,7 +3104,7 @@ "userSetupAllowed" : false } ] }, { - "id" : "edf00de8-8f19-4a32-98c4-15e719c1fadd", + "id" : "91d40deb-344f-4e0b-a845-98b2fc4a633a", "alias" : "first broker login", "description" : "Actions taken after first broker login with identity provider account, which is not yet linked to any Keycloak account", "providerId" : "basic-flow", @@ -2967,7 +3127,7 @@ "userSetupAllowed" : false } ] }, { - "id" : "58415605-eb47-41b3-a07f-90bbbbcb9963", + "id" : "f221b5e6-1bcc-4b37-ba61-4d3bc6a30a8b", "alias" : "forms", "description" : "Username, password, otp and other auth forms.", "providerId" : "basic-flow", @@ -2989,7 +3149,7 @@ "userSetupAllowed" : false } ] }, { - "id" : "1eae6099-3e1e-484b-ad94-b09339affb68", + "id" : "3ed8e597-19af-4ec8-b532-a97311f52de3", "alias" : "http challenge", "description" : "An authentication flow based on challenge-response HTTP Authentication Schemes", "providerId" : "basic-flow", @@ -3011,7 +3171,7 @@ "userSetupAllowed" : false } ] }, { - "id" : "8af03739-b77a-4582-ab63-a1855ca4f637", + "id" : "3970fd16-3786-4eb3-9efe-453d0984b18b", "alias" : "registration", "description" : "registration flow", "providerId" : "basic-flow", @@ -3027,7 +3187,7 @@ "userSetupAllowed" : false } ] }, { - "id" : "0c308998-c5ad-4cf8-ab5c-15be89cbe4d7", + "id" : "e26b27b4-c957-491c-bb6d-9d226b22399c", "alias" : "registration form", "description" : "registration form", "providerId" : "form-flow", @@ -3063,7 +3223,7 @@ "userSetupAllowed" : false } ] }, { - "id" : "5510aa65-e78d-4d08-a3ca-31e277bc3cd0", + "id" : "3ae37429-a623-42e3-a4a1-f9586b96b730", "alias" : "reset credentials", "description" : "Reset credentials for a user if they forgot their password or something", "providerId" : "basic-flow", @@ -3099,7 +3259,7 @@ "userSetupAllowed" : false } ] }, { - "id" : "b6b3e35d-8df3-487e-b2d2-9fdf524a4181", + "id" : "7606ecd5-eb13-4aee-bd9f-3ec4ce77c59c", "alias" : "saml ecp", "description" : "SAML ECP Profile Authentication Flow", "providerId" : "basic-flow", @@ -3115,13 +3275,13 @@ } ] } ], "authenticatorConfig" : [ { - "id" : "a2e9294b-74ce-4ea6-8372-9d9fb3d60a06", + "id" : "058b3c89-4ea4-43fa-b337-e523b1d93ec3", "alias" : "create unique user config", "config" : { "require.password.update.after.registration" : "false" } }, { - "id" : "de65a90c-cc4b-4bf0-8e84-756e23a504f0", + "id" : "21410ac7-4b82-4f19-aae2-43ac33ba3f8f", "alias" : "review profile config", "config" : { "update.profile.on.first.login" : "missing" diff --git a/keycloak/test_user_lists/status b/keycloak/test_user_lists/status index 651e76da..667c4f03 100644 --- a/keycloak/test_user_lists/status +++ b/keycloak/test_user_lists/status @@ -15,3 +15,11 @@ dao.project.lead@status.im desktop.project.lead@status.im app.program.lead@status.im desktop.program.lead@status.im +legal.program-lead.sme@status.im +legal.project-lead.sme@status.im +legal1.sme@status.im +infra1.sme@status.im +infra2.sme@status.im +security.program-lead.sme@status.im +security.project-lead.sme@status.im +security1.sme@status.im From 30a73e2a0abf52472ad3917bccd4b0fb83c7440b Mon Sep 17 00:00:00 2001 From: jbirddog <100367399+jbirddog@users.noreply.github.com> Date: Thu, 2 Feb 2023 10:24:55 -0500 Subject: [PATCH 14/24] Allow for different Python Environments when executing scripts within SpiffWorkflow (#121) --- poetry.lock | 5 +- .../services/process_instance_processor.py | 184 ++++++++++++++++-- .../services/script_unit_test_runner.py | 2 + .../scripts/test_get_localtime.py | 3 +- 4 files changed, 168 insertions(+), 26 deletions(-) diff --git a/poetry.lock b/poetry.lock index 312890e5..733c84ac 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1825,7 +1825,7 @@ lxml = "*" type = "git" url = "https://github.com/sartography/SpiffWorkflow" reference = "main" -resolved_reference = "98c6294f1240aee599cd98bcee58d121cb57b331" +resolved_reference = "64737498caa36c25b12f5216bdc9c30338b2a1fa" [[package]] name = "SQLAlchemy" @@ -2863,10 +2863,7 @@ orjson = [ {file = "orjson-3.8.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b68a42a31f8429728183c21fb440c21de1b62e5378d0d73f280e2d894ef8942e"}, {file = "orjson-3.8.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:ff13410ddbdda5d4197a4a4c09969cb78c722a67550f0a63c02c07aadc624833"}, {file = "orjson-3.8.0-cp310-none-win_amd64.whl", hash = "sha256:2d81e6e56bbea44be0222fb53f7b255b4e7426290516771592738ca01dbd053b"}, - {file = "orjson-3.8.0-cp311-cp311-macosx_10_7_x86_64.whl", hash = "sha256:200eae21c33f1f8b02a11f5d88d76950cd6fd986d88f1afe497a8ae2627c49aa"}, - {file = "orjson-3.8.0-cp311-cp311-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:9529990f3eab54b976d327360aa1ff244a4b12cb5e4c5b3712fcdd96e8fe56d4"}, {file = "orjson-3.8.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:e2defd9527651ad39ec20ae03c812adf47ef7662bdd6bc07dabb10888d70dc62"}, - {file = "orjson-3.8.0-cp311-none-win_amd64.whl", hash = "sha256:b21c7af0ff6228ca7105f54f0800636eb49201133e15ddb80ac20c1ce973ef07"}, {file = "orjson-3.8.0-cp37-cp37m-macosx_10_7_x86_64.whl", hash = "sha256:9e6ac22cec72d5b39035b566e4b86c74b84866f12b5b0b6541506a080fb67d6d"}, {file = "orjson-3.8.0-cp37-cp37m-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:e2f4a5542f50e3d336a18cb224fc757245ca66b1fd0b70b5dd4471b8ff5f2b0e"}, {file = "orjson-3.8.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e1418feeb8b698b9224b1f024555895169d481604d5d884498c1838d7412794c"}, diff --git a/src/spiffworkflow_backend/services/process_instance_processor.py b/src/spiffworkflow_backend/services/process_instance_processor.py index 7cec48a1..40458838 100644 --- a/src/spiffworkflow_backend/services/process_instance_processor.py +++ b/src/spiffworkflow_backend/services/process_instance_processor.py @@ -26,8 +26,10 @@ from lxml import etree # type: ignore from lxml.etree import XMLSyntaxError # type: ignore from RestrictedPython import safe_globals # type: ignore from SpiffWorkflow.bpmn.parser.ValidationException import ValidationException # type: ignore -from SpiffWorkflow.bpmn.PythonScriptEngine import Box # type: ignore -from SpiffWorkflow.bpmn.PythonScriptEngine import PythonScriptEngine +from SpiffWorkflow.bpmn.PythonScriptEngine import PythonScriptEngine # type: ignore +from SpiffWorkflow.bpmn.PythonScriptEngineEnvironment import BasePythonScriptEngineEnvironment # type: ignore +from SpiffWorkflow.bpmn.PythonScriptEngineEnvironment import Box +from SpiffWorkflow.bpmn.PythonScriptEngineEnvironment import BoxedTaskDataEnvironment from SpiffWorkflow.bpmn.serializer.workflow import BpmnWorkflowSerializer # type: ignore from SpiffWorkflow.bpmn.specs.BpmnProcessSpec import BpmnProcessSpec # type: ignore from SpiffWorkflow.bpmn.specs.events.EndEvent import EndEvent # type: ignore @@ -150,6 +152,132 @@ class ProcessInstanceLockedBySomethingElseError(Exception): pass +class BoxedTaskDataBasedScriptEngineEnvironment(BoxedTaskDataEnvironment): # type: ignore + def __init__(self, environment_globals: Dict[str, Any]): + """BoxedTaskDataBasedScriptEngineEnvironment.""" + self._last_result: Dict[str, Any] = {} + super().__init__(environment_globals) + + def execute( + self, + script: str, + context: Dict[str, Any], + external_methods: Optional[Dict[str, Any]] = None, + ) -> None: + super().execute(script, context, external_methods) + self._last_result = context + + def last_result(self) -> Dict[str, Any]: + return self._last_result + + def clear_state(self) -> None: + pass + + def preserve_state(self, bpmn_process_instance: BpmnWorkflow) -> None: + pass + + def restore_state(self, bpmn_process_instance: BpmnWorkflow) -> None: + pass + + def finalize_result(self, bpmn_process_instance: BpmnWorkflow) -> None: + pass + + def revise_state_with_task_data(self, task: SpiffTask) -> None: + pass + + +class NonTaskDataBasedScriptEngineEnvironment(BasePythonScriptEngineEnvironment): # type: ignore + PYTHON_ENVIRONMENT_STATE_KEY = "spiff__python_env_state" + + def __init__(self, environment_globals: Dict[str, Any]): + """NonTaskDataBasedScriptEngineEnvironment.""" + self.state: Dict[str, Any] = {} + self.non_user_defined_keys = set( + [*environment_globals.keys()] + ["__builtins__", "current_user"] + ) + super().__init__(environment_globals) + + def evaluate( + self, + expression: str, + context: Dict[str, Any], + external_methods: Optional[dict[str, Any]] = None, + ) -> Any: + # TODO: once integrated look at the tests that fail without Box + Box.convert_to_box(context) + state = {} + state.update(self.globals) + state.update(external_methods or {}) + state.update(self.state) + state.update(context) + return eval(expression, state) # noqa + + def execute( + self, + script: str, + context: Dict[str, Any], + external_methods: Optional[Dict[str, Any]] = None, + ) -> None: + # TODO: once integrated look at the tests that fail without Box + Box.convert_to_box(context) + self.state.update(self.globals) + self.state.update(external_methods or {}) + self.state.update(context) + exec(script, self.state) # noqa + + self.state = self._user_defined_state(external_methods) + + # the task data needs to be updated with the current state so data references can be resolved properly. + # the state will be removed later once the task is completed. + context.update(self.state) + + def _user_defined_state( + self, external_methods: Optional[Dict[str, Any]] = None + ) -> Dict[str, Any]: + keys_to_filter = self.non_user_defined_keys + if external_methods is not None: + keys_to_filter |= set(external_methods.keys()) + + return { + k: v + for k, v in self.state.items() + if k not in keys_to_filter and not callable(v) + } + + def last_result(self) -> Dict[str, Any]: + return self.state + + def clear_state(self) -> None: + self.state = {} + + def preserve_state(self, bpmn_process_instance: BpmnWorkflow) -> None: + key = self.PYTHON_ENVIRONMENT_STATE_KEY + state = self._user_defined_state() + bpmn_process_instance.data[key] = state + + def restore_state(self, bpmn_process_instance: BpmnWorkflow) -> None: + key = self.PYTHON_ENVIRONMENT_STATE_KEY + self.state = bpmn_process_instance.data.get(key, {}) + + def finalize_result(self, bpmn_process_instance: BpmnWorkflow) -> None: + bpmn_process_instance.data.update(self._user_defined_state()) + + def revise_state_with_task_data(self, task: SpiffTask) -> None: + state_keys = set(self.state.keys()) + task_data_keys = set(task.data.keys()) + state_keys_to_remove = state_keys - task_data_keys + task_data_keys_to_keep = task_data_keys - state_keys + + self.state = { + k: v for k, v in self.state.items() if k not in state_keys_to_remove + } + task.data = {k: v for k, v in task.data.items() if k in task_data_keys_to_keep} + + +class CustomScriptEngineEnvironment(BoxedTaskDataBasedScriptEngineEnvironment): + pass + + class CustomBpmnScriptEngine(PythonScriptEngine): # type: ignore """This is a custom script processor that can be easily injected into Spiff Workflow. @@ -179,7 +307,9 @@ class CustomBpmnScriptEngine(PythonScriptEngine): # type: ignore default_globals.update(safe_globals) default_globals["__builtins__"]["__import__"] = _import - super().__init__(default_globals=default_globals) + environment = CustomScriptEngineEnvironment(default_globals) + + super().__init__(environment=environment) def __get_augment_methods(self, task: SpiffTask) -> Dict[str, Callable]: """__get_augment_methods.""" @@ -392,7 +522,7 @@ class ProcessInstanceProcessor: validate_only, subprocesses=subprocesses, ) - self.bpmn_process_instance.script_engine = self._script_engine + self.set_script_engine(self.bpmn_process_instance) self.add_user_info_to_process_instance(self.bpmn_process_instance) except MissingSpecError as ke: @@ -438,6 +568,18 @@ class ProcessInstanceProcessor: bpmn_process_spec, subprocesses ) + @staticmethod + def set_script_engine(bpmn_process_instance: BpmnWorkflow) -> None: + ProcessInstanceProcessor._script_engine.environment.restore_state( + bpmn_process_instance + ) + bpmn_process_instance.script_engine = ProcessInstanceProcessor._script_engine + + def preserve_script_engine_state(self) -> None: + ProcessInstanceProcessor._script_engine.environment.preserve_state( + self.bpmn_process_instance + ) + def current_user(self) -> Any: """Current_user.""" current_user = None @@ -470,11 +612,12 @@ class ProcessInstanceProcessor: subprocesses: Optional[IdToBpmnProcessSpecMapping] = None, ) -> BpmnWorkflow: """Get_bpmn_process_instance_from_workflow_spec.""" - return BpmnWorkflow( + bpmn_process_instance = BpmnWorkflow( spec, - script_engine=ProcessInstanceProcessor._script_engine, subprocess_specs=subprocesses, ) + ProcessInstanceProcessor.set_script_engine(bpmn_process_instance) + return bpmn_process_instance @staticmethod def __get_bpmn_process_instance( @@ -501,9 +644,7 @@ class ProcessInstanceProcessor: finally: spiff_logger.setLevel(original_spiff_logger_log_level) - bpmn_process_instance.script_engine = ( - ProcessInstanceProcessor._script_engine - ) + ProcessInstanceProcessor.set_script_engine(bpmn_process_instance) else: bpmn_process_instance = ( ProcessInstanceProcessor.get_bpmn_process_instance_from_workflow_spec( @@ -1384,25 +1525,25 @@ class ProcessInstanceProcessor: def do_engine_steps(self, exit_at: None = None, save: bool = False) -> None: """Do_engine_steps.""" step_details = [] + + def did_complete_task(task: SpiffTask) -> None: + self._script_engine.environment.revise_state_with_task_data(task) + step_details.append(self.spiff_step_details_mapping()) + try: - self.bpmn_process_instance.refresh_waiting_tasks( - # - # commenting out to see if this helps with the growing spiff steps/db issue - # - # will_refresh_task=lambda t: self.increment_spiff_step(), - # did_refresh_task=lambda t: step_details.append( - # self.spiff_step_details_mapping() - # ), - ) + self.bpmn_process_instance.refresh_waiting_tasks() self.bpmn_process_instance.do_engine_steps( exit_at=exit_at, will_complete_task=lambda t: self.increment_spiff_step(), - did_complete_task=lambda t: step_details.append( - self.spiff_step_details_mapping() - ), + did_complete_task=did_complete_task, ) + if self.bpmn_process_instance.is_completed(): + self._script_engine.environment.finalize_result( + self.bpmn_process_instance + ) + self.process_bpmn_messages() self.queue_waiting_receive_messages() @@ -1466,6 +1607,7 @@ class ProcessInstanceProcessor: def serialize(self) -> str: """Serialize.""" self.check_task_data_size() + self.preserve_script_engine_state() return self._serializer.serialize_json(self.bpmn_process_instance) # type: ignore def next_user_tasks(self) -> list[SpiffTask]: diff --git a/src/spiffworkflow_backend/services/script_unit_test_runner.py b/src/spiffworkflow_backend/services/script_unit_test_runner.py index 1fafb548..310f53e9 100644 --- a/src/spiffworkflow_backend/services/script_unit_test_runner.py +++ b/src/spiffworkflow_backend/services/script_unit_test_runner.py @@ -45,6 +45,7 @@ class ScriptUnitTestRunner: context = input_context.copy() try: + cls._script_engine.environment.clear_state() cls._script_engine._execute(context=context, script=script) except SyntaxError as ex: return ScriptUnitTestResult( @@ -77,6 +78,7 @@ class ScriptUnitTestRunner: error=f"Failed to execute script: {error_message}", ) + context = cls._script_engine.environment.last_result() result_as_boolean = context == expected_output_context script_unit_test_result = ScriptUnitTestResult( diff --git a/tests/spiffworkflow_backend/scripts/test_get_localtime.py b/tests/spiffworkflow_backend/scripts/test_get_localtime.py index 90e4158d..8116ec42 100644 --- a/tests/spiffworkflow_backend/scripts/test_get_localtime.py +++ b/tests/spiffworkflow_backend/scripts/test_get_localtime.py @@ -87,7 +87,8 @@ class TestGetLocaltime(BaseTest): ) assert spiff_task - data = spiff_task.data + + data = ProcessInstanceProcessor._script_engine.environment.last_result() some_time = data["some_time"] localtime = data["localtime"] timezone = data["timezone"] From 4641b523b7cbf059b42f85b0ad3bec6e0cd12ff3 Mon Sep 17 00:00:00 2001 From: jbirddog <100367399+jbirddog@users.noreply.github.com> Date: Thu, 2 Feb 2023 14:44:37 -0500 Subject: [PATCH 15/24] File download from workflow data (#122) --- src/spiffworkflow_backend/api.yml | 39 +++++++++++++ .../routes/process_api_blueprint.py | 57 ++++++++++++++++++- src/spiffworkflow_backend/routes/user.py | 5 ++ .../scripts/markdown_file_download_link.py | 51 +++++++++++++++++ 4 files changed, 151 insertions(+), 1 deletion(-) create mode 100644 src/spiffworkflow_backend/scripts/markdown_file_download_link.py diff --git a/src/spiffworkflow_backend/api.yml b/src/spiffworkflow_backend/api.yml index 825a24b4..326d55b6 100755 --- a/src/spiffworkflow_backend/api.yml +++ b/src/spiffworkflow_backend/api.yml @@ -1605,6 +1605,45 @@ paths: schema: $ref: "#/components/schemas/Workflow" + /process-data-file-download/{modified_process_model_identifier}/{process_instance_id}/{process_data_identifier}: + parameters: + - name: modified_process_model_identifier + in: path + required: true + description: The modified id of an existing process model + schema: + type: string + - name: process_instance_id + in: path + required: true + description: The unique id of an existing process instance. + schema: + type: integer + - name: process_data_identifier + in: path + required: true + description: The identifier of the process data. + schema: + type: string + - name: index + in: query + required: false + description: The optional index of the value if key's value is an array + schema: + type: integer + get: + operationId: spiffworkflow_backend.routes.process_api_blueprint.process_data_file_download + summary: Download the file referneced in the process data value. + tags: + - Data Objects + responses: + "200": + description: Fetch succeeded. + content: + application/json: + schema: + $ref: "#/components/schemas/Workflow" + /send-event/{modified_process_model_identifier}/{process_instance_id}: parameters: - name: modified_process_model_identifier diff --git a/src/spiffworkflow_backend/routes/process_api_blueprint.py b/src/spiffworkflow_backend/routes/process_api_blueprint.py index 0e9bd581..82263475 100644 --- a/src/spiffworkflow_backend/routes/process_api_blueprint.py +++ b/src/spiffworkflow_backend/routes/process_api_blueprint.py @@ -1,7 +1,9 @@ """APIs for dealing with process groups, process models, and process instances.""" +import base64 import json from typing import Any from typing import Dict +from typing import Optional import flask.wrappers from flask import Blueprint @@ -81,10 +83,12 @@ def process_list() -> Any: return SpecReferenceSchema(many=True).dump(references) -def process_data_show( +def _process_data_fetcher( process_instance_id: int, process_data_identifier: str, modified_process_model_identifier: str, + download_file_data: bool, + index: Optional[int] = None, ) -> flask.wrappers.Response: """Process_data_show.""" process_instance = _find_process_instance_by_id_or_raise(process_instance_id) @@ -94,6 +98,26 @@ def process_data_show( if process_data_identifier in all_process_data: process_data_value = all_process_data[process_data_identifier] + if process_data_value is not None and index is not None: + process_data_value = process_data_value[index] + + if ( + download_file_data + and isinstance(process_data_value, str) + and process_data_value.startswith("data:") + ): + parts = process_data_value.split(";") + mimetype = parts[0][4:] + filename = parts[1] + base64_value = parts[2].split(",")[1] + file_contents = base64.b64decode(base64_value) + + return Response( + file_contents, + mimetype=mimetype, + headers={"Content-disposition": f"attachment; filename={filename}"}, + ) + return make_response( jsonify( { @@ -105,6 +129,37 @@ def process_data_show( ) +def process_data_show( + process_instance_id: int, + process_data_identifier: str, + modified_process_model_identifier: str, +) -> flask.wrappers.Response: + """Process_data_show.""" + return _process_data_fetcher( + process_instance_id, + process_data_identifier, + modified_process_model_identifier, + False, + None, + ) + + +def process_data_file_download( + process_instance_id: int, + process_data_identifier: str, + modified_process_model_identifier: str, + index: Optional[int] = None, +) -> flask.wrappers.Response: + """Process_data_file_download.""" + return _process_data_fetcher( + process_instance_id, + process_data_identifier, + modified_process_model_identifier, + True, + index, + ) + + # sample body: # {"ref": "refs/heads/main", "repository": {"name": "sample-process-models", # "full_name": "sartography/sample-process-models", "private": False .... }} diff --git a/src/spiffworkflow_backend/routes/user.py b/src/spiffworkflow_backend/routes/user.py index 6873198a..6fd7d39c 100644 --- a/src/spiffworkflow_backend/routes/user.py +++ b/src/spiffworkflow_backend/routes/user.py @@ -17,6 +17,7 @@ from flask import request from werkzeug.wrappers import Response from spiffworkflow_backend.exceptions.api_error import ApiError +from spiffworkflow_backend.helpers.api_version import V1_API_PATH_PREFIX from spiffworkflow_backend.models.user import UserModel from spiffworkflow_backend.services.authentication_service import AuthenticationService from spiffworkflow_backend.services.authentication_service import ( @@ -58,6 +59,10 @@ def verify_token( if not token and "Authorization" in request.headers: token = request.headers["Authorization"].removeprefix("Bearer ") + if not token and "access_token" in request.cookies: + if request.path.startswith(f"{V1_API_PATH_PREFIX}/process-data-file-download/"): + token = request.cookies["access_token"] + # This should never be set here but just in case _clear_auth_tokens_from_thread_local_data() diff --git a/src/spiffworkflow_backend/scripts/markdown_file_download_link.py b/src/spiffworkflow_backend/scripts/markdown_file_download_link.py new file mode 100644 index 00000000..3952525b --- /dev/null +++ b/src/spiffworkflow_backend/scripts/markdown_file_download_link.py @@ -0,0 +1,51 @@ +"""Markdown_file_download_link.""" +from typing import Any +from urllib.parse import unquote + +from flask import current_app + +from spiffworkflow_backend.models.process_model import ProcessModelInfo +from spiffworkflow_backend.models.script_attributes_context import ( + ScriptAttributesContext, +) +from spiffworkflow_backend.scripts.script import Script + + +class GetMarkdownFileDownloadLink(Script): + """GetMarkdownFileDownloadLink.""" + + @staticmethod + def requires_privileged_permissions() -> bool: + """We have deemed this function safe to run without elevated permissions.""" + return False + + def get_description(self) -> str: + """Get_description.""" + return """Returns a string which is a string in markdown format.""" + + def run( + self, + script_attributes_context: ScriptAttributesContext, + *_args: Any, + **kwargs: Any, + ) -> Any: + """Run.""" + # example input: + # "data:application/pdf;name=Harmeet_1234.pdf;base64,JV...." + process_data_identifier = kwargs["key"] + parts = kwargs["file_data"].split(";") + file_index = kwargs["file_index"] + label = unquote(parts[1].split("=")[1]) + process_model_identifier = script_attributes_context.process_model_identifier + modified_process_model_identifier = ( + ProcessModelInfo.modify_process_identifier_for_path_param( + process_model_identifier + ) + ) + process_instance_id = script_attributes_context.process_instance_id + url = current_app.config["SPIFFWORKFLOW_BACKEND_URL"] + url += f"/v1.0/process-data-file-download/{modified_process_model_identifier}/" + f"{process_instance_id}/{process_data_identifier}?index={file_index}" + link = f"[{label}]({url})" + + return link From 9baf0cb242b1a0d146853ca1c73af050760a72b4 Mon Sep 17 00:00:00 2001 From: Jon Herron Date: Thu, 2 Feb 2023 15:04:57 -0500 Subject: [PATCH 16/24] Quick fix for url building --- .../scripts/markdown_file_download_link.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/spiffworkflow_backend/scripts/markdown_file_download_link.py b/src/spiffworkflow_backend/scripts/markdown_file_download_link.py index 3952525b..d1b3af7f 100644 --- a/src/spiffworkflow_backend/scripts/markdown_file_download_link.py +++ b/src/spiffworkflow_backend/scripts/markdown_file_download_link.py @@ -44,8 +44,8 @@ class GetMarkdownFileDownloadLink(Script): ) process_instance_id = script_attributes_context.process_instance_id url = current_app.config["SPIFFWORKFLOW_BACKEND_URL"] - url += f"/v1.0/process-data-file-download/{modified_process_model_identifier}/" - f"{process_instance_id}/{process_data_identifier}?index={file_index}" + url += f"/v1.0/process-data-file-download/{modified_process_model_identifier}/" + \ + f"{process_instance_id}/{process_data_identifier}?index={file_index}" link = f"[{label}]({url})" return link From e9913d83a199c4dae3e9e1b7c92d5a07ac40fc92 Mon Sep 17 00:00:00 2001 From: burnettk Date: Thu, 2 Feb 2023 15:40:01 -0500 Subject: [PATCH 17/24] simplify spiff integration post serializer update, w/ elizabeth and jon --- poetry.lock | 5 +- .../scripts/markdown_file_download_link.py | 6 ++- .../services/process_instance_processor.py | 51 +++---------------- 3 files changed, 15 insertions(+), 47 deletions(-) diff --git a/poetry.lock b/poetry.lock index 733c84ac..570faf85 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1825,7 +1825,7 @@ lxml = "*" type = "git" url = "https://github.com/sartography/SpiffWorkflow" reference = "main" -resolved_reference = "64737498caa36c25b12f5216bdc9c30338b2a1fa" +resolved_reference = "0e61be85c47474a33037e6f398e64c96e02f13ad" [[package]] name = "SQLAlchemy" @@ -2546,6 +2546,7 @@ greenlet = [ {file = "greenlet-2.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d5b0ff9878333823226d270417f24f4d06f235cb3e54d1103b71ea537a6a86ce"}, {file = "greenlet-2.0.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:be9e0fb2ada7e5124f5282d6381903183ecc73ea019568d6d63d33f25b2a9000"}, {file = "greenlet-2.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0b493db84d124805865adc587532ebad30efa68f79ad68f11b336e0a51ec86c2"}, + {file = "greenlet-2.0.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:0459d94f73265744fee4c2d5ec44c6f34aa8a31017e6e9de770f7bcf29710be9"}, {file = "greenlet-2.0.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:a20d33124935d27b80e6fdacbd34205732660e0a1d35d8b10b3328179a2b51a1"}, {file = "greenlet-2.0.1-cp37-cp37m-win32.whl", hash = "sha256:ea688d11707d30e212e0110a1aac7f7f3f542a259235d396f88be68b649e47d1"}, {file = "greenlet-2.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:afe07421c969e259e9403c3bb658968702bc3b78ec0b6fde3ae1e73440529c23"}, @@ -2554,6 +2555,7 @@ greenlet = [ {file = "greenlet-2.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:659f167f419a4609bc0516fb18ea69ed39dbb25594934bd2dd4d0401660e8a1e"}, {file = "greenlet-2.0.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:356e4519d4dfa766d50ecc498544b44c0249b6de66426041d7f8b751de4d6b48"}, {file = "greenlet-2.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:811e1d37d60b47cb8126e0a929b58c046251f28117cb16fcd371eed61f66b764"}, + {file = "greenlet-2.0.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:d38ffd0e81ba8ef347d2be0772e899c289b59ff150ebbbbe05dc61b1246eb4e0"}, {file = "greenlet-2.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:0109af1138afbfb8ae647e31a2b1ab030f58b21dd8528c27beaeb0093b7938a9"}, {file = "greenlet-2.0.1-cp38-cp38-win32.whl", hash = "sha256:88c8d517e78acdf7df8a2134a3c4b964415b575d2840a2746ddb1cc6175f8608"}, {file = "greenlet-2.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:d6ee1aa7ab36475035eb48c01efae87d37936a8173fc4d7b10bb02c2d75dd8f6"}, @@ -2562,6 +2564,7 @@ greenlet = [ {file = "greenlet-2.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:505138d4fa69462447a562a7c2ef723c6025ba12ac04478bc1ce2fcc279a2db5"}, {file = "greenlet-2.0.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cce1e90dd302f45716a7715517c6aa0468af0bf38e814ad4eab58e88fc09f7f7"}, {file = "greenlet-2.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9e9744c657d896c7b580455e739899e492a4a452e2dd4d2b3e459f6b244a638d"}, + {file = "greenlet-2.0.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:662e8f7cad915ba75d8017b3e601afc01ef20deeeabf281bd00369de196d7726"}, {file = "greenlet-2.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:41b825d65f31e394b523c84db84f9383a2f7eefc13d987f308f4663794d2687e"}, {file = "greenlet-2.0.1-cp39-cp39-win32.whl", hash = "sha256:db38f80540083ea33bdab614a9d28bcec4b54daa5aff1668d7827a9fc769ae0a"}, {file = "greenlet-2.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:b23d2a46d53210b498e5b701a1913697671988f4bf8e10f935433f6e7c332fb6"}, diff --git a/src/spiffworkflow_backend/scripts/markdown_file_download_link.py b/src/spiffworkflow_backend/scripts/markdown_file_download_link.py index d1b3af7f..25f81cc7 100644 --- a/src/spiffworkflow_backend/scripts/markdown_file_download_link.py +++ b/src/spiffworkflow_backend/scripts/markdown_file_download_link.py @@ -44,8 +44,10 @@ class GetMarkdownFileDownloadLink(Script): ) process_instance_id = script_attributes_context.process_instance_id url = current_app.config["SPIFFWORKFLOW_BACKEND_URL"] - url += f"/v1.0/process-data-file-download/{modified_process_model_identifier}/" + \ - f"{process_instance_id}/{process_data_identifier}?index={file_index}" + url += ( + f"/v1.0/process-data-file-download/{modified_process_model_identifier}/" + + f"{process_instance_id}/{process_data_identifier}?index={file_index}" + ) link = f"[{label}]({url})" return link diff --git a/src/spiffworkflow_backend/services/process_instance_processor.py b/src/spiffworkflow_backend/services/process_instance_processor.py index 40458838..b45add69 100644 --- a/src/spiffworkflow_backend/services/process_instance_processor.py +++ b/src/spiffworkflow_backend/services/process_instance_processor.py @@ -38,36 +38,14 @@ from SpiffWorkflow.bpmn.specs.events.StartEvent import StartEvent # type: ignor from SpiffWorkflow.bpmn.specs.SubWorkflowTask import SubWorkflowTask # type: ignore from SpiffWorkflow.bpmn.workflow import BpmnWorkflow # type: ignore from SpiffWorkflow.dmn.parser.BpmnDmnParser import BpmnDmnParser # type: ignore -from SpiffWorkflow.dmn.serializer.task_spec_converters import BusinessRuleTaskConverter # type: ignore +from SpiffWorkflow.dmn.serializer.task_spec import BusinessRuleTaskConverter # type: ignore from SpiffWorkflow.exceptions import WorkflowException # type: ignore from SpiffWorkflow.exceptions import WorkflowTaskException from SpiffWorkflow.serializer.exceptions import MissingSpecError # type: ignore -from SpiffWorkflow.spiff.serializer.task_spec_converters import BoundaryEventConverter # type: ignore -from SpiffWorkflow.spiff.serializer.task_spec_converters import ( - CallActivityTaskConverter, -) -from SpiffWorkflow.spiff.serializer.task_spec_converters import EndEventConverter -from SpiffWorkflow.spiff.serializer.task_spec_converters import ( +from SpiffWorkflow.spiff.serializer.config import SPIFF_SPEC_CONFIG # type: ignore +from SpiffWorkflow.spiff.serializer.task_spec_converters import ( # type: ignore EventBasedGatewayConverter, ) -from SpiffWorkflow.spiff.serializer.task_spec_converters import ( - IntermediateCatchEventConverter, -) -from SpiffWorkflow.spiff.serializer.task_spec_converters import ( - IntermediateThrowEventConverter, -) -from SpiffWorkflow.spiff.serializer.task_spec_converters import ManualTaskConverter -from SpiffWorkflow.spiff.serializer.task_spec_converters import NoneTaskConverter -from SpiffWorkflow.spiff.serializer.task_spec_converters import ReceiveTaskConverter -from SpiffWorkflow.spiff.serializer.task_spec_converters import ScriptTaskConverter -from SpiffWorkflow.spiff.serializer.task_spec_converters import SendTaskConverter -from SpiffWorkflow.spiff.serializer.task_spec_converters import ServiceTaskConverter -from SpiffWorkflow.spiff.serializer.task_spec_converters import StartEventConverter -from SpiffWorkflow.spiff.serializer.task_spec_converters import SubWorkflowTaskConverter -from SpiffWorkflow.spiff.serializer.task_spec_converters import ( - TransactionSubprocessConverter, -) -from SpiffWorkflow.spiff.serializer.task_spec_converters import UserTaskConverter from SpiffWorkflow.task import Task as SpiffTask # type: ignore from SpiffWorkflow.task import TaskState from SpiffWorkflow.util.deep_merge import DeepMerge # type: ignore @@ -110,6 +88,8 @@ from spiffworkflow_backend.services.service_task_service import ServiceTaskDeleg from spiffworkflow_backend.services.spec_file_service import SpecFileService from spiffworkflow_backend.services.user_service import UserService +SPIFF_SPEC_CONFIG["task_specs"].append(BusinessRuleTaskConverter) + # Sorry about all this crap. I wanted to move this thing to another file, but # importing a bunch of types causes circular imports. @@ -408,26 +388,9 @@ class ProcessInstanceProcessor: _script_engine = CustomBpmnScriptEngine() SERIALIZER_VERSION = "1.0-spiffworkflow-backend" + wf_spec_converter = BpmnWorkflowSerializer.configure_workflow_spec_converter( - [ - BoundaryEventConverter, - BusinessRuleTaskConverter, - CallActivityTaskConverter, - EndEventConverter, - IntermediateCatchEventConverter, - IntermediateThrowEventConverter, - EventBasedGatewayConverter, - ManualTaskConverter, - NoneTaskConverter, - ReceiveTaskConverter, - ScriptTaskConverter, - SendTaskConverter, - ServiceTaskConverter, - StartEventConverter, - SubWorkflowTaskConverter, - TransactionSubprocessConverter, - UserTaskConverter, - ] + SPIFF_SPEC_CONFIG ) _serializer = BpmnWorkflowSerializer(wf_spec_converter, version=SERIALIZER_VERSION) _event_serializer = EventBasedGatewayConverter() From 5b7805d046c7e0c64e1bc66259b21f2c041cb558 Mon Sep 17 00:00:00 2001 From: burnettk Date: Thu, 2 Feb 2023 19:00:58 -0500 Subject: [PATCH 18/24] try to improve exception handling by avoiding raising ApiError from services --- .../exceptions/api_error.py | 31 +++++++++++++--- .../services/authentication_service.py | 24 ++++++++++--- .../services/authorization_service.py | 36 ++++++++----------- 3 files changed, 62 insertions(+), 29 deletions(-) diff --git a/src/spiffworkflow_backend/exceptions/api_error.py b/src/spiffworkflow_backend/exceptions/api_error.py index 02a66a20..ab5bf1c3 100644 --- a/src/spiffworkflow_backend/exceptions/api_error.py +++ b/src/spiffworkflow_backend/exceptions/api_error.py @@ -20,6 +20,11 @@ from SpiffWorkflow.exceptions import WorkflowTaskException from SpiffWorkflow.specs.base import TaskSpec # type: ignore from SpiffWorkflow.task import Task # type: ignore +from spiffworkflow_backend.services.authentication_service import NotAuthorizedError +from spiffworkflow_backend.services.authentication_service import TokenInvalidError +from spiffworkflow_backend.services.authentication_service import TokenNotProvidedError +from spiffworkflow_backend.services.authentication_service import UserNotLoggedInError + api_error_blueprint = Blueprint("api_error_blueprint", __name__) @@ -172,7 +177,12 @@ def handle_exception(exception: Exception) -> flask.wrappers.Response: set_user_sentry_context() sentry_link = None - if not isinstance(exception, ApiError) or exception.error_code != "invalid_token": + # we want to capture_exception to log the exception to sentry, but we don't want to log: + # 1. ApiErrors that are just invalid tokens + # 2. NotAuthorizedError + if ( + not isinstance(exception, ApiError) or exception.error_code != "invalid_token" + ) and not isinstance(exception, NotAuthorizedError): id = capture_exception(exception) if isinstance(exception, ApiError): @@ -193,17 +203,30 @@ def handle_exception(exception: Exception) -> flask.wrappers.Response: # an event id or send out tags like username current_app.logger.exception(exception) + error_code = "internal_server_error" + status_code = 500 + if ( + isinstance(exception, NotAuthorizedError) + or isinstance(exception, TokenNotProvidedError) + or isinstance(exception, TokenInvalidError) + ): + error_code = "not_authorized" + status_code = 403 + if isinstance(exception, UserNotLoggedInError): + error_code = "not_authenticated" + status_code = 401 + # set api_exception like this to avoid confusing mypy - # and what type the object is + # about what type the object is api_exception = None if isinstance(exception, ApiError): api_exception = exception else: api_exception = ApiError( - error_code="internal_server_error", + error_code=error_code, message=f"{exception.__class__.__name__}", sentry_link=sentry_link, - status_code=500, + status_code=status_code, ) return make_response(jsonify(api_exception), api_exception.status_code) diff --git a/src/spiffworkflow_backend/services/authentication_service.py b/src/spiffworkflow_backend/services/authentication_service.py index 1793aab6..5c9c4708 100644 --- a/src/spiffworkflow_backend/services/authentication_service.py +++ b/src/spiffworkflow_backend/services/authentication_service.py @@ -11,7 +11,6 @@ from flask import current_app from flask import redirect from werkzeug.wrappers import Response -from spiffworkflow_backend.exceptions.api_error import ApiError from spiffworkflow_backend.models.db import db from spiffworkflow_backend.models.refresh_token import RefreshTokenModel @@ -20,7 +19,21 @@ class MissingAccessTokenError(Exception): """MissingAccessTokenError.""" +class NotAuthorizedError(Exception): + pass + + +class RefreshTokenStorageError(Exception): + pass + + +class UserNotLoggedInError(Exception): + pass + + # These could be either 'id' OR 'access' tokens and we can't always know which + + class TokenExpiredError(Exception): """TokenExpiredError.""" @@ -29,6 +42,10 @@ class TokenInvalidError(Exception): """TokenInvalidError.""" +class TokenNotProvidedError(Exception): + pass + + class AuthenticationProviderTypes(enum.Enum): """AuthenticationServiceProviders.""" @@ -183,9 +200,8 @@ class AuthenticationService: db.session.commit() except Exception as e: db.session.rollback() - raise ApiError( - error_code="store_refresh_token_error", - message=f"We could not store the refresh token. Original error is {e}", + raise RefreshTokenStorageError( + f"We could not store the refresh token. Original error is {e}", ) from e @staticmethod diff --git a/src/spiffworkflow_backend/services/authorization_service.py b/src/spiffworkflow_backend/services/authorization_service.py index 19f9f418..a72effd4 100644 --- a/src/spiffworkflow_backend/services/authorization_service.py +++ b/src/spiffworkflow_backend/services/authorization_service.py @@ -21,7 +21,6 @@ from SpiffWorkflow.task import Task as SpiffTask # type: ignore from sqlalchemy import or_ from sqlalchemy import text -from spiffworkflow_backend.exceptions.api_error import ApiError from spiffworkflow_backend.helpers.api_version import V1_API_PATH_PREFIX from spiffworkflow_backend.models.db import db from spiffworkflow_backend.models.group import GroupModel @@ -34,6 +33,11 @@ from spiffworkflow_backend.models.user import UserModel from spiffworkflow_backend.models.user import UserNotFoundError from spiffworkflow_backend.models.user_group_assignment import UserGroupAssignmentModel from spiffworkflow_backend.routes.openid_blueprint import openid_blueprint +from spiffworkflow_backend.services.authentication_service import NotAuthorizedError +from spiffworkflow_backend.services.authentication_service import TokenExpiredError +from spiffworkflow_backend.services.authentication_service import TokenInvalidError +from spiffworkflow_backend.services.authentication_service import TokenNotProvidedError +from spiffworkflow_backend.services.authentication_service import UserNotLoggedInError from spiffworkflow_backend.services.group_service import GroupService from spiffworkflow_backend.services.user_service import UserService @@ -98,20 +102,16 @@ class AuthorizationService: def verify_sha256_token(cls, auth_header: Optional[str]) -> None: """Verify_sha256_token.""" if auth_header is None: - raise ApiError( - error_code="unauthorized", - message="", - status_code=403, + raise TokenNotProvidedError( + "unauthorized", ) received_sign = auth_header.split("sha256=")[-1].strip() secret = current_app.config["GITHUB_WEBHOOK_SECRET"].encode() expected_sign = HMAC(key=secret, msg=request.data, digestmod=sha256).hexdigest() if not compare_digest(received_sign, expected_sign): - raise ApiError( - error_code="unauthorized", - message="", - status_code=403, + raise TokenInvalidError( + "unauthorized", ) @classmethod @@ -393,10 +393,8 @@ class AuthorizationService: authorization_exclusion_list = ["permissions_check"] if not hasattr(g, "user"): - raise ApiError( - error_code="user_not_logged_in", - message="User is not logged in. Please log in", - status_code=401, + raise UserNotLoggedInError( + "User is not logged in. Please log in", ) api_view_function = current_app.view_functions[request.endpoint] @@ -416,13 +414,11 @@ class AuthorizationService: if has_permission: return None - raise ApiError( - error_code="unauthorized", - message=( + raise NotAuthorizedError( + ( f"User {g.user.username} is not authorized to perform requested action:" f" {permission_string} - {request.path}" ), - status_code=403, ) @staticmethod @@ -440,13 +436,11 @@ class AuthorizationService: payload = jwt.decode(auth_token, options={"verify_signature": False}) return payload except jwt.ExpiredSignatureError as exception: - raise ApiError( - "token_expired", + raise TokenExpiredError( "The Authentication token you provided expired and must be renewed.", ) from exception except jwt.InvalidTokenError as exception: - raise ApiError( - "token_invalid", + raise TokenInvalidError( ( "The Authentication token you provided is invalid. You need a new" " token. " From b17142bff8ac9351c8a6e2954e49ae7f737d81d7 Mon Sep 17 00:00:00 2001 From: burnettk Date: Thu, 2 Feb 2023 21:55:26 -0500 Subject: [PATCH 19/24] import EventBasedGatewayConverter from correct package --- .../services/process_instance_processor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/spiffworkflow_backend/services/process_instance_processor.py b/src/spiffworkflow_backend/services/process_instance_processor.py index b45add69..022560c6 100644 --- a/src/spiffworkflow_backend/services/process_instance_processor.py +++ b/src/spiffworkflow_backend/services/process_instance_processor.py @@ -43,7 +43,7 @@ from SpiffWorkflow.exceptions import WorkflowException # type: ignore from SpiffWorkflow.exceptions import WorkflowTaskException from SpiffWorkflow.serializer.exceptions import MissingSpecError # type: ignore from SpiffWorkflow.spiff.serializer.config import SPIFF_SPEC_CONFIG # type: ignore -from SpiffWorkflow.spiff.serializer.task_spec_converters import ( # type: ignore +from SpiffWorkflow.bpmn.serializer.task_spec import ( # type: ignore EventBasedGatewayConverter, ) from SpiffWorkflow.task import Task as SpiffTask # type: ignore From 0fced76dbc2d31a2ca1530e79713460699ec431f Mon Sep 17 00:00:00 2001 From: burnettk Date: Thu, 2 Feb 2023 22:04:34 -0500 Subject: [PATCH 20/24] couple last serializer updates --- .../services/process_instance_processor.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/spiffworkflow_backend/services/process_instance_processor.py b/src/spiffworkflow_backend/services/process_instance_processor.py index 022560c6..c9d43f92 100644 --- a/src/spiffworkflow_backend/services/process_instance_processor.py +++ b/src/spiffworkflow_backend/services/process_instance_processor.py @@ -30,6 +30,9 @@ from SpiffWorkflow.bpmn.PythonScriptEngine import PythonScriptEngine # type: ig from SpiffWorkflow.bpmn.PythonScriptEngineEnvironment import BasePythonScriptEngineEnvironment # type: ignore from SpiffWorkflow.bpmn.PythonScriptEngineEnvironment import Box from SpiffWorkflow.bpmn.PythonScriptEngineEnvironment import BoxedTaskDataEnvironment +from SpiffWorkflow.bpmn.serializer.task_spec import ( # type: ignore + EventBasedGatewayConverter, +) from SpiffWorkflow.bpmn.serializer.workflow import BpmnWorkflowSerializer # type: ignore from SpiffWorkflow.bpmn.specs.BpmnProcessSpec import BpmnProcessSpec # type: ignore from SpiffWorkflow.bpmn.specs.events.EndEvent import EndEvent # type: ignore @@ -43,9 +46,6 @@ from SpiffWorkflow.exceptions import WorkflowException # type: ignore from SpiffWorkflow.exceptions import WorkflowTaskException from SpiffWorkflow.serializer.exceptions import MissingSpecError # type: ignore from SpiffWorkflow.spiff.serializer.config import SPIFF_SPEC_CONFIG # type: ignore -from SpiffWorkflow.bpmn.serializer.task_spec import ( # type: ignore - EventBasedGatewayConverter, -) from SpiffWorkflow.task import Task as SpiffTask # type: ignore from SpiffWorkflow.task import TaskState from SpiffWorkflow.util.deep_merge import DeepMerge # type: ignore @@ -393,7 +393,7 @@ class ProcessInstanceProcessor: SPIFF_SPEC_CONFIG ) _serializer = BpmnWorkflowSerializer(wf_spec_converter, version=SERIALIZER_VERSION) - _event_serializer = EventBasedGatewayConverter() + _event_serializer = EventBasedGatewayConverter(wf_spec_converter) PROCESS_INSTANCE_ID_KEY = "process_instance_id" VALIDATION_PROCESS_KEY = "validate_only" @@ -971,7 +971,7 @@ class ProcessInstanceProcessor: def send_bpmn_event(self, event_data: dict[str, Any]) -> None: """Send an event to the workflow.""" payload = event_data.pop("payload", None) - event_definition = self._event_serializer.restore(event_data) + event_definition = self._event_serializer.registry.restore(event_data) if payload is not None: event_definition.payload = payload current_app.logger.info( From a163d6f7ed650ceb1f2a8458feabbfe2ec7a0472 Mon Sep 17 00:00:00 2001 From: burnettk Date: Fri, 3 Feb 2023 11:06:40 -0500 Subject: [PATCH 21/24] clean up sentry notification and avoid logger.exception when we do not want sentry --- .../exceptions/api_error.py | 38 ++++++++++++++----- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/src/spiffworkflow_backend/exceptions/api_error.py b/src/spiffworkflow_backend/exceptions/api_error.py index ab5bf1c3..46d2ad54 100644 --- a/src/spiffworkflow_backend/exceptions/api_error.py +++ b/src/spiffworkflow_backend/exceptions/api_error.py @@ -171,18 +171,30 @@ def set_user_sentry_context() -> None: set_tag("username", username) +def should_notify_sentry(exception: Exception) -> bool: + """Determine if we should notify sentry. + + We want to capture_exception to log the exception to sentry, but we don't want to log: + 1. ApiErrors that are just invalid tokens + 2. NotAuthorizedError. we usually call check-permissions before calling an API to + make sure we'll have access, but there are some cases + where it's more convenient to just make the call from the frontend and handle the 403 appropriately. + """ + if isinstance(exception, ApiError): + if exception.error_code == "invalid_token": + return False + if isinstance(exception, NotAuthorizedError): + return False + return True + + @api_error_blueprint.app_errorhandler(Exception) # type: ignore def handle_exception(exception: Exception) -> flask.wrappers.Response: """Handles unexpected exceptions.""" set_user_sentry_context() sentry_link = None - # we want to capture_exception to log the exception to sentry, but we don't want to log: - # 1. ApiErrors that are just invalid tokens - # 2. NotAuthorizedError - if ( - not isinstance(exception, ApiError) or exception.error_code != "invalid_token" - ) and not isinstance(exception, NotAuthorizedError): + if should_notify_sentry(exception): id = capture_exception(exception) if isinstance(exception, ApiError): @@ -198,10 +210,16 @@ def handle_exception(exception: Exception) -> flask.wrappers.Response: f"https://sentry.io/{organization_slug}/{project_slug}/events/{id}" ) - # !!!NOTE!!!: do this after sentry stuff since calling logger.exception - # seems to break the sentry sdk context where we no longer get back - # an event id or send out tags like username - current_app.logger.exception(exception) + # !!!NOTE!!!: do this after sentry stuff since calling logger.exception + # seems to break the sentry sdk context where we no longer get back + # an event id or send out tags like username + current_app.logger.exception(exception) + else: + current_app.logger.error( + f"Received exception: {exception}. Since we do not want this particular" + " exception in sentry, we cannot use logger.exception, so there will be no" + " backtrace. see api_error.py" + ) error_code = "internal_server_error" status_code = 500 From 698b3af880caa6f68b480311a9534079e33b652f Mon Sep 17 00:00:00 2001 From: burnettk Date: Fri, 3 Feb 2023 12:51:57 -0500 Subject: [PATCH 22/24] make test_user_lists more complete and correct --- keycloak/bin/add_test_users_to_keycloak | 6 +++ keycloak/realm_exports/sartography-realm.json | 2 +- keycloak/test_user_lists/sartography | 9 ++-- keycloak/test_user_lists/status | 45 ++++++++++++------- 4 files changed, 42 insertions(+), 20 deletions(-) diff --git a/keycloak/bin/add_test_users_to_keycloak b/keycloak/bin/add_test_users_to_keycloak index 5ad11e13..9a045ffe 100755 --- a/keycloak/bin/add_test_users_to_keycloak +++ b/keycloak/bin/add_test_users_to_keycloak @@ -7,7 +7,13 @@ function error_handler() { trap 'error_handler ${LINENO} $?' ERR set -o errtrace -o errexit -o nounset -o pipefail +# you can get a list of users from the keycloak realm file like: +# grep '"email" :' keycloak/realm_exports/spiffworkflow-realm.json | awk -F : '{print $2}' | sed -E 's/ "//g' | sed -E 's/",//g' > s + +# we keep some of these in keycloak/test_user_lists +# spiffworkflow-realm.json is a mashup of the status and sartography user lists. user_file_with_one_email_per_line="${1:-}" + keycloak_realm="${2:-spiffworkflow}" if [[ -z "${1:-}" ]]; then >&2 echo "usage: $(basename "$0") [user_file_with_one_email_per_line]" diff --git a/keycloak/realm_exports/sartography-realm.json b/keycloak/realm_exports/sartography-realm.json index 37704ea5..20c19e24 100644 --- a/keycloak/realm_exports/sartography-realm.json +++ b/keycloak/realm_exports/sartography-realm.json @@ -547,7 +547,7 @@ "enabled" : true, "totp" : false, "emailVerified" : false, - "email" : "kevin@sartography.com", + "email" : "kb@sartography.com", "credentials" : [ { "id" : "4057e784-689d-47c0-a164-035a69e78edf", "type" : "password", diff --git a/keycloak/test_user_lists/sartography b/keycloak/test_user_lists/sartography index 4f98a51e..b6f685b8 100644 --- a/keycloak/test_user_lists/sartography +++ b/keycloak/test_user_lists/sartography @@ -1,8 +1,11 @@ +admin@spiffworkflow.org alex@sartography.com dan@sartography.com -kevin@sartography.com -jason@sartography.com -mike@sartography.com +daniel@sartography.com elizabeth@sartography.com +jason@sartography.com jon@sartography.com +kb@sartography.com +madhurya@sartography.com +mike@sartography.com natalia@sartography.com diff --git a/keycloak/test_user_lists/status b/keycloak/test_user_lists/status index 667c4f03..cb510747 100644 --- a/keycloak/test_user_lists/status +++ b/keycloak/test_user_lists/status @@ -1,25 +1,38 @@ +admin@spiffworkflow.org +amir@status.im +app.program.lead@status.im +core@status.im +dao.project.lead@status.im +desktop.program.lead@status.im +desktop.project.lead@status.im +fin1@status.im +fin@status.im finance.lead@status.im -legal.lead@status.im -program.lead@status.im -services.lead@status.im finance.sme@status.im -infra.sme@status.im -legal.sme@status.im -security.sme@status.im -ppg.ba@status.im -peopleops.partner@status.im -peopleops.talent@status.im +finance_user1@status.im +harmeet@status.im infra.program-lead@status.im infra.project-lead@status.im -dao.project.lead@status.im -desktop.project.lead@status.im -app.program.lead@status.im -desktop.program.lead@status.im -legal.program-lead.sme@status.im -legal.project-lead.sme@status.im -legal1.sme@status.im +infra.sme@status.im infra1.sme@status.im infra2.sme@status.im +jakub@status.im +jarrad@status.im +lead1@status.im +lead@status.im +legal.lead@status.im +legal.program-lead.sme@status.im +legal.project-lead.sme@status.im +legal.sme@status.im +legal1.sme@status.im +manuchehr@status.im +peopleops.partner@status.im +peopleops.talent@status.im +ppg.ba@status.im +program.lead@status.im +sasha@status.im security.program-lead.sme@status.im security.project-lead.sme@status.im +security.sme@status.im security1.sme@status.im +services.lead@status.im From 83b9c90149bd97e4e0ac5bda6259a3420d9c4f29 Mon Sep 17 00:00:00 2001 From: burnettk Date: Fri, 3 Feb 2023 13:02:50 -0500 Subject: [PATCH 23/24] remove service accounts, formalize j, add madhurya --- keycloak/bin/export_keycloak_realms | 3 + .../realm_exports/spiffworkflow-realm.json | 106 ++++++++---------- keycloak/test_user_lists/sartography | 2 + 3 files changed, 50 insertions(+), 61 deletions(-) diff --git a/keycloak/bin/export_keycloak_realms b/keycloak/bin/export_keycloak_realms index f205d0d7..7e55ae6f 100755 --- a/keycloak/bin/export_keycloak_realms +++ b/keycloak/bin/export_keycloak_realms @@ -21,6 +21,9 @@ docker exec keycloak /opt/keycloak/bin/kc.sh export --dir "${docker_container_pa docker cp "keycloak:${docker_container_path}" "$local_tmp_dir" for realm in $realms ; do + if ! grep -Eq '\-realm$' <<< "$realm"; then + realm="${realm}-realm" + fi cp "${local_tmp_dir}/hey/${realm}.json" "${script_dir}/../realm_exports/" done diff --git a/keycloak/realm_exports/spiffworkflow-realm.json b/keycloak/realm_exports/spiffworkflow-realm.json index 634caef7..c81e57ad 100644 --- a/keycloak/realm_exports/spiffworkflow-realm.json +++ b/keycloak/realm_exports/spiffworkflow-realm.json @@ -903,7 +903,7 @@ "emailVerified" : false, "firstName" : "", "lastName" : "", - "email" : "j@status.im", + "email" : "j@sartography.com", "credentials" : [ { "id" : "e71ec785-9133-4b7d-8015-1978379af0bb", "type" : "password", @@ -1163,6 +1163,26 @@ "realmRoles" : [ "default-roles-spiffworkflow" ], "notBefore" : 0, "groups" : [ ] + }, { + "id" : "99ce8a54-2941-4767-8ddf-52320b3708bd", + "createdTimestamp" : 1675447085191, + "username" : "madhurya", + "enabled" : true, + "totp" : false, + "emailVerified" : false, + "email" : "madhurya@sartography.com", + "credentials" : [ { + "id" : "4fa2bf1f-188e-42e3-9633-01d436864206", + "type" : "password", + "createdDate" : 1675447085252, + "secretData" : "{\"value\":\"6ZApQ7kx4YDc5ojW9eyFiSKMz5l3/Zl5PIScHEW1gtP3lrnnWqWgwcP+8cWkKdm3im+XrZwDQHjuGjGN5Rbjyw==\",\"salt\":\"HT3fCh245v8etRFIprXsyw==\",\"additionalParameters\":{}}", + "credentialData" : "{\"hashIterations\":27500,\"algorithm\":\"pbkdf2-sha256\",\"additionalParameters\":{}}" + } ], + "disableableCredentialTypes" : [ ], + "requiredActions" : [ ], + "realmRoles" : [ "default-roles-spiffworkflow" ], + "notBefore" : 0, + "groups" : [ ] }, { "id" : "6f5bfa09-7494-4a2f-b871-cf327048cac7", "createdTimestamp" : 1665517010600, @@ -1405,42 +1425,6 @@ "realmRoles" : [ "default-roles-spiffworkflow" ], "notBefore" : 0, "groups" : [ ] - }, { - "id" : "487d3a85-89dd-4839-957a-c3f6d70551f6", - "createdTimestamp" : 1657115173081, - "username" : "service-account-spiffworkflow-backend", - "enabled" : true, - "totp" : false, - "emailVerified" : false, - "email" : "service-account@status.im", - "serviceAccountClientId" : "spiffworkflow-backend", - "credentials" : [ ], - "disableableCredentialTypes" : [ ], - "requiredActions" : [ ], - "realmRoles" : [ "default-roles-spiffworkflow" ], - "clientRoles" : { - "spiffworkflow-backend" : [ "uma_protection" ] - }, - "notBefore" : 0, - "groups" : [ ] - }, { - "id" : "22de68b1-4b06-4bc2-8da6-0c577e7e62ad", - "createdTimestamp" : 1657055472800, - "username" : "service-account-withauth", - "enabled" : true, - "totp" : false, - "emailVerified" : false, - "email" : "service-account-withauth@status.im", - "serviceAccountClientId" : "withAuth", - "credentials" : [ ], - "disableableCredentialTypes" : [ ], - "requiredActions" : [ ], - "realmRoles" : [ "default-roles-spiffworkflow" ], - "clientRoles" : { - "withAuth" : [ "uma_protection" ] - }, - "notBefore" : 0, - "groups" : [ ] }, { "id" : "3d45bb85-0a2d-4b15-8a19-d26a5619d359", "createdTimestamp" : 1674148694810, @@ -2674,7 +2658,7 @@ "subType" : "authenticated", "subComponents" : { }, "config" : { - "allowed-protocol-mapper-types" : [ "oidc-address-mapper", "oidc-sha256-pairwise-sub-mapper", "saml-role-list-mapper", "oidc-usermodel-attribute-mapper", "saml-user-property-mapper", "saml-user-attribute-mapper", "oidc-usermodel-property-mapper", "oidc-full-name-mapper" ] + "allowed-protocol-mapper-types" : [ "oidc-usermodel-property-mapper", "saml-role-list-mapper", "saml-user-attribute-mapper", "oidc-address-mapper", "saml-user-property-mapper", "oidc-usermodel-attribute-mapper", "oidc-sha256-pairwise-sub-mapper", "oidc-full-name-mapper" ] } }, { "id" : "d68e938d-dde6-47d9-bdc8-8e8523eb08cd", @@ -2692,7 +2676,7 @@ "subType" : "anonymous", "subComponents" : { }, "config" : { - "allowed-protocol-mapper-types" : [ "saml-role-list-mapper", "oidc-sha256-pairwise-sub-mapper", "oidc-usermodel-attribute-mapper", "saml-user-attribute-mapper", "oidc-full-name-mapper", "saml-user-property-mapper", "oidc-address-mapper", "oidc-usermodel-property-mapper" ] + "allowed-protocol-mapper-types" : [ "saml-role-list-mapper", "saml-user-property-mapper", "oidc-usermodel-property-mapper", "oidc-full-name-mapper", "oidc-usermodel-attribute-mapper", "oidc-sha256-pairwise-sub-mapper", "oidc-address-mapper", "saml-user-attribute-mapper" ] } }, { "id" : "3854361d-3fe5-47fb-9417-a99592e3dc5c", @@ -2782,7 +2766,7 @@ "internationalizationEnabled" : false, "supportedLocales" : [ ], "authenticationFlows" : [ { - "id" : "feafc299-fede-4880-9e23-eb81aca22808", + "id" : "8facbab5-bca2-42c6-8608-ed94dacefe92", "alias" : "Account verification options", "description" : "Method with which to verity the existing account", "providerId" : "basic-flow", @@ -2804,7 +2788,7 @@ "userSetupAllowed" : false } ] }, { - "id" : "ce7904d0-9182-49a2-aa71-a7b43e21f3ac", + "id" : "be52bd38-2def-41e7-a021-69bae78e92b7", "alias" : "Authentication Options", "description" : "Authentication options.", "providerId" : "basic-flow", @@ -2833,7 +2817,7 @@ "userSetupAllowed" : false } ] }, { - "id" : "d9c6909a-5cc1-4ddf-b297-dbfcf6e609a6", + "id" : "ee18f6d1-9ca3-4535-a7a0-9759f3841513", "alias" : "Browser - Conditional OTP", "description" : "Flow to determine if the OTP is required for the authentication", "providerId" : "basic-flow", @@ -2855,7 +2839,7 @@ "userSetupAllowed" : false } ] }, { - "id" : "083a589e-a486-42b6-ae73-1ec983967ff5", + "id" : "c76481eb-7997-4231-abac-632afd97631f", "alias" : "Direct Grant - Conditional OTP", "description" : "Flow to determine if the OTP is required for the authentication", "providerId" : "basic-flow", @@ -2877,7 +2861,7 @@ "userSetupAllowed" : false } ] }, { - "id" : "7f0248b0-2d51-4175-9fd2-52b606a39e26", + "id" : "14fe94d2-f3ef-4349-9cbe-79921c013108", "alias" : "First broker login - Conditional OTP", "description" : "Flow to determine if the OTP is required for the authentication", "providerId" : "basic-flow", @@ -2899,7 +2883,7 @@ "userSetupAllowed" : false } ] }, { - "id" : "44465f1f-c700-4ec0-a234-d95c994c9e25", + "id" : "533c45e3-10d9-480b-9c9b-c2f746fb6f66", "alias" : "Handle Existing Account", "description" : "Handle what to do if there is existing account with same email/username like authenticated identity provider", "providerId" : "basic-flow", @@ -2921,7 +2905,7 @@ "userSetupAllowed" : false } ] }, { - "id" : "8cf09055-5b98-4fc8-b867-3dffacdec21b", + "id" : "1161d043-26ba-420c-baed-b220bcef40f1", "alias" : "Reset - Conditional OTP", "description" : "Flow to determine if the OTP should be reset or not. Set to REQUIRED to force.", "providerId" : "basic-flow", @@ -2943,7 +2927,7 @@ "userSetupAllowed" : false } ] }, { - "id" : "16b50b3e-4240-4f49-a85e-1bfd40def300", + "id" : "cbba8afb-920f-4ae0-85f3-6bc520485dc2", "alias" : "User creation or linking", "description" : "Flow for the existing/non-existing user alternatives", "providerId" : "basic-flow", @@ -2966,7 +2950,7 @@ "userSetupAllowed" : false } ] }, { - "id" : "2aa981ae-d67e-49fb-95a4-91de1e5ab724", + "id" : "7b349cd1-fb1c-4d04-b5b5-885352277562", "alias" : "Verify Existing Account by Re-authentication", "description" : "Reauthentication of existing account", "providerId" : "basic-flow", @@ -2988,7 +2972,7 @@ "userSetupAllowed" : false } ] }, { - "id" : "cf8406f7-09c3-4614-a898-99c9d66746f6", + "id" : "de10b07d-98b5-483c-b193-b1b93229478f", "alias" : "browser", "description" : "browser based authentication", "providerId" : "basic-flow", @@ -3024,7 +3008,7 @@ "userSetupAllowed" : false } ] }, { - "id" : "e1ec7d6e-7612-4c5b-afce-c7f4fddbf6ec", + "id" : "4504d37b-3a2d-4cc9-b300-29482d86c72e", "alias" : "clients", "description" : "Base authentication for clients", "providerId" : "client-flow", @@ -3060,7 +3044,7 @@ "userSetupAllowed" : false } ] }, { - "id" : "f5862b09-6e01-4c88-b44e-26dc59d71b80", + "id" : "9d86bdff-ba8e-433a-8536-a49c0af5faf2", "alias" : "direct grant", "description" : "OpenID Connect Resource Owner Grant", "providerId" : "basic-flow", @@ -3089,7 +3073,7 @@ "userSetupAllowed" : false } ] }, { - "id" : "7caa8611-8b13-437e-83b2-556899b5444f", + "id" : "546d31fc-a885-46eb-94bd-171d04f16a7c", "alias" : "docker auth", "description" : "Used by Docker clients to authenticate against the IDP", "providerId" : "basic-flow", @@ -3104,7 +3088,7 @@ "userSetupAllowed" : false } ] }, { - "id" : "91d40deb-344f-4e0b-a845-98b2fc4a633a", + "id" : "70e5d629-4338-4aec-8671-fc7cf4c450b1", "alias" : "first broker login", "description" : "Actions taken after first broker login with identity provider account, which is not yet linked to any Keycloak account", "providerId" : "basic-flow", @@ -3127,7 +3111,7 @@ "userSetupAllowed" : false } ] }, { - "id" : "f221b5e6-1bcc-4b37-ba61-4d3bc6a30a8b", + "id" : "7213dc19-6e0b-4241-bef6-2409346a2745", "alias" : "forms", "description" : "Username, password, otp and other auth forms.", "providerId" : "basic-flow", @@ -3149,7 +3133,7 @@ "userSetupAllowed" : false } ] }, { - "id" : "3ed8e597-19af-4ec8-b532-a97311f52de3", + "id" : "f91a8499-8cf5-408c-b85d-40e85a3f6ee3", "alias" : "http challenge", "description" : "An authentication flow based on challenge-response HTTP Authentication Schemes", "providerId" : "basic-flow", @@ -3171,7 +3155,7 @@ "userSetupAllowed" : false } ] }, { - "id" : "3970fd16-3786-4eb3-9efe-453d0984b18b", + "id" : "9ec3751c-619e-4edc-a14f-4ac9c60b056f", "alias" : "registration", "description" : "registration flow", "providerId" : "basic-flow", @@ -3187,7 +3171,7 @@ "userSetupAllowed" : false } ] }, { - "id" : "e26b27b4-c957-491c-bb6d-9d226b22399c", + "id" : "8048e711-8e77-4b85-8b26-243948a7c2f4", "alias" : "registration form", "description" : "registration form", "providerId" : "form-flow", @@ -3223,7 +3207,7 @@ "userSetupAllowed" : false } ] }, { - "id" : "3ae37429-a623-42e3-a4a1-f9586b96b730", + "id" : "5a08de49-dd24-4e53-a656-9fac52fc6d2b", "alias" : "reset credentials", "description" : "Reset credentials for a user if they forgot their password or something", "providerId" : "basic-flow", @@ -3259,7 +3243,7 @@ "userSetupAllowed" : false } ] }, { - "id" : "7606ecd5-eb13-4aee-bd9f-3ec4ce77c59c", + "id" : "42bc970f-3ee5-429c-a543-e8078808d371", "alias" : "saml ecp", "description" : "SAML ECP Profile Authentication Flow", "providerId" : "basic-flow", @@ -3275,13 +3259,13 @@ } ] } ], "authenticatorConfig" : [ { - "id" : "058b3c89-4ea4-43fa-b337-e523b1d93ec3", + "id" : "23f4f930-3290-4a63-ac96-f7ddc04fbce2", "alias" : "create unique user config", "config" : { "require.password.update.after.registration" : "false" } }, { - "id" : "21410ac7-4b82-4f19-aae2-43ac33ba3f8f", + "id" : "4cfa7fa4-1a9b-4464-9510-460208e345eb", "alias" : "review profile config", "config" : { "update.profile.on.first.login" : "missing" diff --git a/keycloak/test_user_lists/sartography b/keycloak/test_user_lists/sartography index b6f685b8..1b7166bb 100644 --- a/keycloak/test_user_lists/sartography +++ b/keycloak/test_user_lists/sartography @@ -3,9 +3,11 @@ alex@sartography.com dan@sartography.com daniel@sartography.com elizabeth@sartography.com +j@sartography.com jason@sartography.com jon@sartography.com kb@sartography.com +kevin@sartography.com madhurya@sartography.com mike@sartography.com natalia@sartography.com From 2d4f9d4581a68fe886dd2de96653f14b3f1dc006 Mon Sep 17 00:00:00 2001 From: burnettk Date: Fri, 3 Feb 2023 13:11:39 -0500 Subject: [PATCH 24/24] add more users, and try to prevent sentry notification again --- .../realm_exports/spiffworkflow-realm.json | 244 ++++++++++++++++-- keycloak/test_user_lists/status | 8 + .../exceptions/api_error.py | 6 +- 3 files changed, 230 insertions(+), 28 deletions(-) diff --git a/keycloak/realm_exports/spiffworkflow-realm.json b/keycloak/realm_exports/spiffworkflow-realm.json index c81e57ad..722f1276 100644 --- a/keycloak/realm_exports/spiffworkflow-realm.json +++ b/keycloak/realm_exports/spiffworkflow-realm.json @@ -1083,6 +1083,26 @@ "realmRoles" : [ "default-roles-spiffworkflow" ], "notBefore" : 0, "groups" : [ ] + }, { + "id" : "e911fb0f-fd07-4886-acbf-d00930d293d3", + "createdTimestamp" : 1675447845512, + "username" : "legal.program-lead", + "enabled" : true, + "totp" : false, + "emailVerified" : false, + "email" : "legal.program-lead@status.im", + "credentials" : [ { + "id" : "9676d8d3-1e8c-4f5d-b5f7-49745cecf8fd", + "type" : "password", + "createdDate" : 1675447845577, + "secretData" : "{\"value\":\"vTffScfGXIjWWyDDfzo7JPiJe9VjAtrmds382EeV7N+wYNapJmLTVModkBsmGPy4TmWLc9BoysQynOaanSGi9Q==\",\"salt\":\"67ZxTEnar8aq4LZLhSNTFg==\",\"additionalParameters\":{}}", + "credentialData" : "{\"hashIterations\":27500,\"algorithm\":\"pbkdf2-sha256\",\"additionalParameters\":{}}" + } ], + "disableableCredentialTypes" : [ ], + "requiredActions" : [ ], + "realmRoles" : [ "default-roles-spiffworkflow" ], + "notBefore" : 0, + "groups" : [ ] }, { "id" : "eff82d12-9a67-4002-b3c5-37811bd45199", "createdTimestamp" : 1675349217585, @@ -1103,6 +1123,26 @@ "realmRoles" : [ "default-roles-spiffworkflow" ], "notBefore" : 0, "groups" : [ ] + }, { + "id" : "4ed2b5a2-16c2-4029-ae97-d75c60f2147f", + "createdTimestamp" : 1675447845616, + "username" : "legal.project-lead", + "enabled" : true, + "totp" : false, + "emailVerified" : false, + "email" : "legal.project-lead@status.im", + "credentials" : [ { + "id" : "fd0b0d0a-8a3e-48c9-b17b-023e87057048", + "type" : "password", + "createdDate" : 1675447845652, + "secretData" : "{\"value\":\"l/DPfNBcHINV8lCf9nEyCJkFvaMGnLqcd1Y8t9taLqxb8r/ofY2ce79C19JCHDQJXRPRuCsMoobuFhhNR6aQmg==\",\"salt\":\"2ivCPrNc56396ldlwpQP6Q==\",\"additionalParameters\":{}}", + "credentialData" : "{\"hashIterations\":27500,\"algorithm\":\"pbkdf2-sha256\",\"additionalParameters\":{}}" + } ], + "disableableCredentialTypes" : [ ], + "requiredActions" : [ ], + "realmRoles" : [ "default-roles-spiffworkflow" ], + "notBefore" : 0, + "groups" : [ ] }, { "id" : "8cd6feba-5ca6-4cfb-bc1a-a52c80595783", "createdTimestamp" : 1675349217698, @@ -1305,6 +1345,86 @@ "realmRoles" : [ "default-roles-spiffworkflow" ], "notBefore" : 0, "groups" : [ ] + }, { + "id" : "9f703c96-02f1-403c-b070-25feb86cfe21", + "createdTimestamp" : 1675447845811, + "username" : "ppg.ba.program-lead", + "enabled" : true, + "totp" : false, + "emailVerified" : false, + "email" : "ppg.ba.program-lead@status.im", + "credentials" : [ { + "id" : "bf74118b-b28f-4d2f-8bfa-7b9d1a8345f2", + "type" : "password", + "createdDate" : 1675447845847, + "secretData" : "{\"value\":\"wFUAB6E98gE222nCfsKe6P3kSZxeOSjhflsxon8kw/dY4ZwN0KMwvlYuNhmoptTLqDQJyqUiydmlMK0NS4JjTQ==\",\"salt\":\"YCPk4Tc3eXcoes78oLhDEg==\",\"additionalParameters\":{}}", + "credentialData" : "{\"hashIterations\":27500,\"algorithm\":\"pbkdf2-sha256\",\"additionalParameters\":{}}" + } ], + "disableableCredentialTypes" : [ ], + "requiredActions" : [ ], + "realmRoles" : [ "default-roles-spiffworkflow" ], + "notBefore" : 0, + "groups" : [ ] + }, { + "id" : "81a1727b-c846-4af9-8d95-1c50b1deb0d5", + "createdTimestamp" : 1675447845879, + "username" : "ppg.ba.project-lead", + "enabled" : true, + "totp" : false, + "emailVerified" : false, + "email" : "ppg.ba.project-lead@status.im", + "credentials" : [ { + "id" : "6411830d-6015-4cf2-bac6-d49c26510319", + "type" : "password", + "createdDate" : 1675447845915, + "secretData" : "{\"value\":\"1+m8twycOEbA4X61zN7dLENqp2IxxQZrXKaf3mEuzmxouHrgxvmXudwC6DWyfjXvLm7gxWlaa4cofBFwr1idig==\",\"salt\":\"UEKUSScYv2xY+rJ8vlvF4A==\",\"additionalParameters\":{}}", + "credentialData" : "{\"hashIterations\":27500,\"algorithm\":\"pbkdf2-sha256\",\"additionalParameters\":{}}" + } ], + "disableableCredentialTypes" : [ ], + "requiredActions" : [ ], + "realmRoles" : [ "default-roles-spiffworkflow" ], + "notBefore" : 0, + "groups" : [ ] + }, { + "id" : "1d4d471a-b3ef-4750-97c4-a9e64eb8f414", + "createdTimestamp" : 1675447845942, + "username" : "ppg.ba.sme", + "enabled" : true, + "totp" : false, + "emailVerified" : false, + "email" : "ppg.ba.sme@status.im", + "credentials" : [ { + "id" : "6512f88a-cbcc-4d79-be17-1d132ba11e64", + "type" : "password", + "createdDate" : 1675447845977, + "secretData" : "{\"value\":\"EErx/3vG+lh4DgrJUzkBv4cLT3sK1gS+T9KD5V/JpvJUmJpRFQqpk+YxC/nC/kTGLIpRDdCIN690T84FlOIjew==\",\"salt\":\"FPeVGnFbt9TRNiORMB5LMQ==\",\"additionalParameters\":{}}", + "credentialData" : "{\"hashIterations\":27500,\"algorithm\":\"pbkdf2-sha256\",\"additionalParameters\":{}}" + } ], + "disableableCredentialTypes" : [ ], + "requiredActions" : [ ], + "realmRoles" : [ "default-roles-spiffworkflow" ], + "notBefore" : 0, + "groups" : [ ] + }, { + "id" : "2dade29f-c6dc-445b-bdf0-eed316bdb638", + "createdTimestamp" : 1675447846003, + "username" : "ppg.ba.sme1", + "enabled" : true, + "totp" : false, + "emailVerified" : false, + "email" : "ppg.ba.sme1@status.im", + "credentials" : [ { + "id" : "ccf2d138-020a-4a29-b63d-1f4d2f415639", + "type" : "password", + "createdDate" : 1675447846038, + "secretData" : "{\"value\":\"BtSJtW/8lCtyrDPTXzhsyT/32H+pOHx9thKqJV30dOEZ9wcSQbrRSHoQbXwLos+sIiA82X3wm+qObdQoD5guVQ==\",\"salt\":\"nSbgxYpVGaMz2ArmqLCN6Q==\",\"additionalParameters\":{}}", + "credentialData" : "{\"hashIterations\":27500,\"algorithm\":\"pbkdf2-sha256\",\"additionalParameters\":{}}" + } ], + "disableableCredentialTypes" : [ ], + "requiredActions" : [ ], + "realmRoles" : [ "default-roles-spiffworkflow" ], + "notBefore" : 0, + "groups" : [ ] }, { "id" : "c3ea06ee-c497-48e6-8816-43c8ef68bd8b", "createdTimestamp" : 1674148694747, @@ -1345,6 +1465,26 @@ "realmRoles" : [ "default-roles-spiffworkflow" ], "notBefore" : 0, "groups" : [ ] + }, { + "id" : "c21c075d-9ac5-40a1-964a-c1d6ffe17257", + "createdTimestamp" : 1675447845680, + "username" : "security.program-lead", + "enabled" : true, + "totp" : false, + "emailVerified" : false, + "email" : "security.program-lead@status.im", + "credentials" : [ { + "id" : "d1401dbd-a88b-44a6-b13c-fff13ee07e0c", + "type" : "password", + "createdDate" : 1675447845718, + "secretData" : "{\"value\":\"3D76RpIFG0/ixbSBeJfCc61kyL8PvVn/khA8FOy6RLg2hrZbs1Uwl8SmplnSUll1wD5a/BoobsO7v1XW4TCvwQ==\",\"salt\":\"YtDRRmBV4SBlO/oX23r2EQ==\",\"additionalParameters\":{}}", + "credentialData" : "{\"hashIterations\":27500,\"algorithm\":\"pbkdf2-sha256\",\"additionalParameters\":{}}" + } ], + "disableableCredentialTypes" : [ ], + "requiredActions" : [ ], + "realmRoles" : [ "default-roles-spiffworkflow" ], + "notBefore" : 0, + "groups" : [ ] }, { "id" : "ace0432f-1818-4210-8bcf-15533abfb3ce", "createdTimestamp" : 1675349217958, @@ -1365,6 +1505,26 @@ "realmRoles" : [ "default-roles-spiffworkflow" ], "notBefore" : 0, "groups" : [ ] + }, { + "id" : "34dfacfd-24b5-414e-ac3e-9b013399aee2", + "createdTimestamp" : 1675447845747, + "username" : "security.project-lead", + "enabled" : true, + "totp" : false, + "emailVerified" : false, + "email" : "security.project-lead@status.im", + "credentials" : [ { + "id" : "cb5d8a8a-e7d0-40e4-878b-a33608cb76c8", + "type" : "password", + "createdDate" : 1675447845784, + "secretData" : "{\"value\":\"rudimVOjVwJeO/1RLuyHySEaSQMzjHqPQrh5Pmfr4L2PgP/1oDKLVB38pKOohlbTarDcbAfMHB7AFYAPn9kuIg==\",\"salt\":\"cOkkUBOx/4AVUSa3Ozsiuw==\",\"additionalParameters\":{}}", + "credentialData" : "{\"hashIterations\":27500,\"algorithm\":\"pbkdf2-sha256\",\"additionalParameters\":{}}" + } ], + "disableableCredentialTypes" : [ ], + "requiredActions" : [ ], + "realmRoles" : [ "default-roles-spiffworkflow" ], + "notBefore" : 0, + "groups" : [ ] }, { "id" : "6272ac80-1d79-4e3c-a5c1-b31660560318", "createdTimestamp" : 1675349218020, @@ -1425,6 +1585,40 @@ "realmRoles" : [ "default-roles-spiffworkflow" ], "notBefore" : 0, "groups" : [ ] + }, { + "id" : "b768e3ef-f905-4493-976c-bc3408c04bec", + "createdTimestamp" : 1675447832524, + "username" : "service-account-spiffworkflow-backend", + "enabled" : true, + "totp" : false, + "emailVerified" : false, + "serviceAccountClientId" : "spiffworkflow-backend", + "credentials" : [ ], + "disableableCredentialTypes" : [ ], + "requiredActions" : [ ], + "realmRoles" : [ "default-roles-spiffworkflow" ], + "clientRoles" : { + "spiffworkflow-backend" : [ "uma_protection" ] + }, + "notBefore" : 0, + "groups" : [ ] + }, { + "id" : "b6fb214b-cb8a-4403-9308-ac6d4e13ef26", + "createdTimestamp" : 1675447832560, + "username" : "service-account-withauth", + "enabled" : true, + "totp" : false, + "emailVerified" : false, + "serviceAccountClientId" : "withAuth", + "credentials" : [ ], + "disableableCredentialTypes" : [ ], + "requiredActions" : [ ], + "realmRoles" : [ "default-roles-spiffworkflow" ], + "clientRoles" : { + "withAuth" : [ "uma_protection" ] + }, + "notBefore" : 0, + "groups" : [ ] }, { "id" : "3d45bb85-0a2d-4b15-8a19-d26a5619d359", "createdTimestamp" : 1674148694810, @@ -2658,7 +2852,7 @@ "subType" : "authenticated", "subComponents" : { }, "config" : { - "allowed-protocol-mapper-types" : [ "oidc-usermodel-property-mapper", "saml-role-list-mapper", "saml-user-attribute-mapper", "oidc-address-mapper", "saml-user-property-mapper", "oidc-usermodel-attribute-mapper", "oidc-sha256-pairwise-sub-mapper", "oidc-full-name-mapper" ] + "allowed-protocol-mapper-types" : [ "oidc-usermodel-property-mapper", "saml-user-property-mapper", "saml-user-attribute-mapper", "oidc-usermodel-attribute-mapper", "oidc-address-mapper", "oidc-sha256-pairwise-sub-mapper", "saml-role-list-mapper", "oidc-full-name-mapper" ] } }, { "id" : "d68e938d-dde6-47d9-bdc8-8e8523eb08cd", @@ -2676,7 +2870,7 @@ "subType" : "anonymous", "subComponents" : { }, "config" : { - "allowed-protocol-mapper-types" : [ "saml-role-list-mapper", "saml-user-property-mapper", "oidc-usermodel-property-mapper", "oidc-full-name-mapper", "oidc-usermodel-attribute-mapper", "oidc-sha256-pairwise-sub-mapper", "oidc-address-mapper", "saml-user-attribute-mapper" ] + "allowed-protocol-mapper-types" : [ "oidc-full-name-mapper", "saml-role-list-mapper", "saml-user-attribute-mapper", "oidc-usermodel-attribute-mapper", "oidc-address-mapper", "saml-user-property-mapper", "oidc-sha256-pairwise-sub-mapper", "oidc-usermodel-property-mapper" ] } }, { "id" : "3854361d-3fe5-47fb-9417-a99592e3dc5c", @@ -2766,7 +2960,7 @@ "internationalizationEnabled" : false, "supportedLocales" : [ ], "authenticationFlows" : [ { - "id" : "8facbab5-bca2-42c6-8608-ed94dacefe92", + "id" : "cb39eda2-18c2-4b03-9d7c-672a2bd47d19", "alias" : "Account verification options", "description" : "Method with which to verity the existing account", "providerId" : "basic-flow", @@ -2788,7 +2982,7 @@ "userSetupAllowed" : false } ] }, { - "id" : "be52bd38-2def-41e7-a021-69bae78e92b7", + "id" : "96d4e28f-51ad-4737-87b4-5a10484ceb8b", "alias" : "Authentication Options", "description" : "Authentication options.", "providerId" : "basic-flow", @@ -2817,7 +3011,7 @@ "userSetupAllowed" : false } ] }, { - "id" : "ee18f6d1-9ca3-4535-a7a0-9759f3841513", + "id" : "8f4c884d-93cd-4404-bc3a-1fa717b070c5", "alias" : "Browser - Conditional OTP", "description" : "Flow to determine if the OTP is required for the authentication", "providerId" : "basic-flow", @@ -2839,7 +3033,7 @@ "userSetupAllowed" : false } ] }, { - "id" : "c76481eb-7997-4231-abac-632afd97631f", + "id" : "166d1879-dd61-4fb4-b4f6-0a4d69f49da8", "alias" : "Direct Grant - Conditional OTP", "description" : "Flow to determine if the OTP is required for the authentication", "providerId" : "basic-flow", @@ -2861,7 +3055,7 @@ "userSetupAllowed" : false } ] }, { - "id" : "14fe94d2-f3ef-4349-9cbe-79921c013108", + "id" : "18cab8f9-f010-4226-a86e-8da2f1632304", "alias" : "First broker login - Conditional OTP", "description" : "Flow to determine if the OTP is required for the authentication", "providerId" : "basic-flow", @@ -2883,7 +3077,7 @@ "userSetupAllowed" : false } ] }, { - "id" : "533c45e3-10d9-480b-9c9b-c2f746fb6f66", + "id" : "04d8d1d1-5253-4644-b55d-8c9317818b33", "alias" : "Handle Existing Account", "description" : "Handle what to do if there is existing account with same email/username like authenticated identity provider", "providerId" : "basic-flow", @@ -2905,7 +3099,7 @@ "userSetupAllowed" : false } ] }, { - "id" : "1161d043-26ba-420c-baed-b220bcef40f1", + "id" : "2bf21e1d-ff7e-4d52-8be7-31355945c302", "alias" : "Reset - Conditional OTP", "description" : "Flow to determine if the OTP should be reset or not. Set to REQUIRED to force.", "providerId" : "basic-flow", @@ -2927,7 +3121,7 @@ "userSetupAllowed" : false } ] }, { - "id" : "cbba8afb-920f-4ae0-85f3-6bc520485dc2", + "id" : "fa8636a5-9969-41a5-9fef-9c825cceb819", "alias" : "User creation or linking", "description" : "Flow for the existing/non-existing user alternatives", "providerId" : "basic-flow", @@ -2950,7 +3144,7 @@ "userSetupAllowed" : false } ] }, { - "id" : "7b349cd1-fb1c-4d04-b5b5-885352277562", + "id" : "8656a884-6645-40b5-b075-c40736e27811", "alias" : "Verify Existing Account by Re-authentication", "description" : "Reauthentication of existing account", "providerId" : "basic-flow", @@ -2972,7 +3166,7 @@ "userSetupAllowed" : false } ] }, { - "id" : "de10b07d-98b5-483c-b193-b1b93229478f", + "id" : "0d88d334-bfa4-4cf1-9fa3-17d0df0151d1", "alias" : "browser", "description" : "browser based authentication", "providerId" : "basic-flow", @@ -3008,7 +3202,7 @@ "userSetupAllowed" : false } ] }, { - "id" : "4504d37b-3a2d-4cc9-b300-29482d86c72e", + "id" : "9b195d67-e3e6-4983-8607-533b739ebd97", "alias" : "clients", "description" : "Base authentication for clients", "providerId" : "client-flow", @@ -3044,7 +3238,7 @@ "userSetupAllowed" : false } ] }, { - "id" : "9d86bdff-ba8e-433a-8536-a49c0af5faf2", + "id" : "fd0273a1-f6f4-4df1-a057-54ac4e91f4a9", "alias" : "direct grant", "description" : "OpenID Connect Resource Owner Grant", "providerId" : "basic-flow", @@ -3073,7 +3267,7 @@ "userSetupAllowed" : false } ] }, { - "id" : "546d31fc-a885-46eb-94bd-171d04f16a7c", + "id" : "b457cba8-ef31-473b-a481-c095b2f4eb48", "alias" : "docker auth", "description" : "Used by Docker clients to authenticate against the IDP", "providerId" : "basic-flow", @@ -3088,7 +3282,7 @@ "userSetupAllowed" : false } ] }, { - "id" : "70e5d629-4338-4aec-8671-fc7cf4c450b1", + "id" : "97519504-fd69-4c08-bd27-15d26fbc9b76", "alias" : "first broker login", "description" : "Actions taken after first broker login with identity provider account, which is not yet linked to any Keycloak account", "providerId" : "basic-flow", @@ -3111,7 +3305,7 @@ "userSetupAllowed" : false } ] }, { - "id" : "7213dc19-6e0b-4241-bef6-2409346a2745", + "id" : "fc6a4468-1a78-410d-ac97-cf9f05814850", "alias" : "forms", "description" : "Username, password, otp and other auth forms.", "providerId" : "basic-flow", @@ -3133,7 +3327,7 @@ "userSetupAllowed" : false } ] }, { - "id" : "f91a8499-8cf5-408c-b85d-40e85a3f6ee3", + "id" : "97a25d8a-25a0-4bf4-be6d-a6f019cf3a32", "alias" : "http challenge", "description" : "An authentication flow based on challenge-response HTTP Authentication Schemes", "providerId" : "basic-flow", @@ -3155,7 +3349,7 @@ "userSetupAllowed" : false } ] }, { - "id" : "9ec3751c-619e-4edc-a14f-4ac9c60b056f", + "id" : "671e8ec7-af31-4c54-b6bb-96ebe69881de", "alias" : "registration", "description" : "registration flow", "providerId" : "basic-flow", @@ -3171,7 +3365,7 @@ "userSetupAllowed" : false } ] }, { - "id" : "8048e711-8e77-4b85-8b26-243948a7c2f4", + "id" : "24d6aaaa-5202-4401-99c3-bb15925bd5be", "alias" : "registration form", "description" : "registration form", "providerId" : "form-flow", @@ -3207,7 +3401,7 @@ "userSetupAllowed" : false } ] }, { - "id" : "5a08de49-dd24-4e53-a656-9fac52fc6d2b", + "id" : "f948bd43-ff05-4245-be30-a0a0dad2b7f0", "alias" : "reset credentials", "description" : "Reset credentials for a user if they forgot their password or something", "providerId" : "basic-flow", @@ -3243,7 +3437,7 @@ "userSetupAllowed" : false } ] }, { - "id" : "42bc970f-3ee5-429c-a543-e8078808d371", + "id" : "7e4aaea7-05ca-4aa0-b934-4c81614620a8", "alias" : "saml ecp", "description" : "SAML ECP Profile Authentication Flow", "providerId" : "basic-flow", @@ -3259,13 +3453,13 @@ } ] } ], "authenticatorConfig" : [ { - "id" : "23f4f930-3290-4a63-ac96-f7ddc04fbce2", + "id" : "14ca1058-25e7-41f6-85ce-ad0bfce2c67c", "alias" : "create unique user config", "config" : { "require.password.update.after.registration" : "false" } }, { - "id" : "4cfa7fa4-1a9b-4464-9510-460208e345eb", + "id" : "16803de1-f7dc-4293-acde-fd0eae264377", "alias" : "review profile config", "config" : { "update.profile.on.first.login" : "missing" @@ -3360,4 +3554,4 @@ "clientPolicies" : { "policies" : [ ] } -} +} \ No newline at end of file diff --git a/keycloak/test_user_lists/status b/keycloak/test_user_lists/status index cb510747..66da936e 100644 --- a/keycloak/test_user_lists/status +++ b/keycloak/test_user_lists/status @@ -22,17 +22,25 @@ lead1@status.im lead@status.im legal.lead@status.im legal.program-lead.sme@status.im +legal.program-lead@status.im legal.project-lead.sme@status.im +legal.project-lead@status.im legal.sme@status.im legal1.sme@status.im manuchehr@status.im peopleops.partner@status.im peopleops.talent@status.im +ppg.ba.program-lead@status.im +ppg.ba.project-lead@status.im +ppg.ba.sme1@status.im +ppg.ba.sme@status.im ppg.ba@status.im program.lead@status.im sasha@status.im security.program-lead.sme@status.im +security.program-lead@status.im security.project-lead.sme@status.im +security.project-lead@status.im security.sme@status.im security1.sme@status.im services.lead@status.im diff --git a/src/spiffworkflow_backend/exceptions/api_error.py b/src/spiffworkflow_backend/exceptions/api_error.py index 46d2ad54..886e138e 100644 --- a/src/spiffworkflow_backend/exceptions/api_error.py +++ b/src/spiffworkflow_backend/exceptions/api_error.py @@ -215,10 +215,10 @@ def handle_exception(exception: Exception) -> flask.wrappers.Response: # an event id or send out tags like username current_app.logger.exception(exception) else: - current_app.logger.error( + current_app.logger.warning( f"Received exception: {exception}. Since we do not want this particular" - " exception in sentry, we cannot use logger.exception, so there will be no" - " backtrace. see api_error.py" + " exception in sentry, we cannot use logger.exception or logger.error, so" + " there will be no backtrace. see api_error.py" ) error_code = "internal_server_error"