From 4dfd01042e1c418c13910e3921c8080ac995b8d5 Mon Sep 17 00:00:00 2001 From: jasquat Date: Thu, 22 Sep 2022 13:08:31 -0400 Subject: [PATCH 01/12] refactor to be able to get all process instances w/ burnettk --- src/spiffworkflow_backend/api.yml | 34 +++++++++++++------ .../routes/process_api_blueprint.py | 27 ++++++++------- 2 files changed, 39 insertions(+), 22 deletions(-) diff --git a/src/spiffworkflow_backend/api.yml b/src/spiffworkflow_backend/api.yml index 0df544a5..fb9582de 100755 --- a/src/spiffworkflow_backend/api.yml +++ b/src/spiffworkflow_backend/api.yml @@ -224,11 +224,11 @@ paths: schema: $ref: "#/components/schemas/WorkflowSpecCategory" - /process-groups/{process_group_id}/process-models: + /process-models: parameters: - name: process_group_id - in: path - required: true + in: query + required: false description: The group containing the models we want to return schema: type: string @@ -390,17 +390,17 @@ paths: schema: $ref: "#/components/schemas/WorkflowSpec" - /process-models/{process_group_id}/{process_model_id}/process-instances: + /process-instances: parameters: - - name: process_group_id - in: path - required: true + - name: process_group_identifier + in: query + required: false description: The unique id of an existing process group schema: type: string - - name: process_model_id - in: path - required: true + - name: process_model_identifier + in: query + required: false description: The unique id of an existing workflow specification. schema: type: string @@ -461,6 +461,20 @@ paths: type: array items: $ref: "#/components/schemas/Workflow" + /process-models/{process_group_id}/{process_model_id}/process-instances: + parameters: + - name: process_group_id + in: path + required: true + description: The unique id of an existing process group + schema: + type: string + - name: process_model_id + in: path + required: true + description: The unique id of an existing workflow specification. + schema: + type: string # process_instance_create post: operationId: spiffworkflow_backend.routes.process_api_blueprint.process_instance_create diff --git a/src/spiffworkflow_backend/routes/process_api_blueprint.py b/src/spiffworkflow_backend/routes/process_api_blueprint.py index 58d4fa27..9ecee852 100644 --- a/src/spiffworkflow_backend/routes/process_api_blueprint.py +++ b/src/spiffworkflow_backend/routes/process_api_blueprint.py @@ -527,8 +527,8 @@ def message_start( def process_instance_list( - process_group_id: str, - process_model_id: str, + process_group_identifier: Optional[str] = None, + process_model_identifier: Optional[str] = None, page: int = 1, per_page: int = 100, start_from: Optional[int] = None, @@ -538,11 +538,14 @@ def process_instance_list( process_status: Optional[str] = None, ) -> flask.wrappers.Response: """Process_instance_list.""" - process_model = get_process_model(process_model_id, process_group_id) - results = ProcessInstanceModel.query.filter_by( - process_model_identifier=process_model.id - ) + process_instance_query = ProcessInstanceModel.query + if process_model_identifier is not None and process_group_identifier is not None: + process_model = get_process_model(process_model_identifier, process_group_identifier) + + process_instance_query = process_instance_query.filter_by( + process_model_identifier=process_model.id + ) # this can never happen. obviously the class has the columns it defines. this is just to appease mypy. if ( @@ -558,17 +561,17 @@ def process_instance_list( ) if start_from is not None: - results = results.filter(ProcessInstanceModel.start_in_seconds >= start_from) + process_instance_query = process_instance_query.filter(ProcessInstanceModel.start_in_seconds >= start_from) if start_till is not None: - results = results.filter(ProcessInstanceModel.start_in_seconds <= start_till) + process_instance_query = process_instance_query.filter(ProcessInstanceModel.start_in_seconds <= start_till) if end_from is not None: - results = results.filter(ProcessInstanceModel.end_in_seconds >= end_from) + process_instance_query = process_instance_query.filter(ProcessInstanceModel.end_in_seconds >= end_from) if end_till is not None: - results = results.filter(ProcessInstanceModel.end_in_seconds <= end_till) + process_instance_query = process_instance_query.filter(ProcessInstanceModel.end_in_seconds <= end_till) if process_status is not None: - results = results.filter(ProcessInstanceModel.status == process_status) + process_instance_query = process_instance_query.filter(ProcessInstanceModel.status == process_status) - process_instances = results.order_by( + process_instances = process_instance_query.order_by( ProcessInstanceModel.start_in_seconds.desc(), ProcessInstanceModel.id.desc() # type: ignore ).paginate(page, per_page, False) From f02a825cec6507c8c15a9c7cb36dae9371043893 Mon Sep 17 00:00:00 2001 From: jasquat Date: Thu, 22 Sep 2022 15:40:01 -0400 Subject: [PATCH 02/12] updated process_models_list to allow getting all process models and udpated tests w/ burnettk --- src/spiffworkflow_backend/api.yml | 4 +- .../routes/process_api_blueprint.py | 31 +++++++--- .../integration/test_process_api.py | 60 +++++++++---------- 3 files changed, 53 insertions(+), 42 deletions(-) diff --git a/src/spiffworkflow_backend/api.yml b/src/spiffworkflow_backend/api.yml index fb9582de..f5650f04 100755 --- a/src/spiffworkflow_backend/api.yml +++ b/src/spiffworkflow_backend/api.yml @@ -226,7 +226,7 @@ paths: /process-models: parameters: - - name: process_group_id + - name: process_group_identifier in: query required: false description: The group containing the models we want to return @@ -259,8 +259,6 @@ paths: type: array items: $ref: "#/components/schemas/WorkflowSpec" - - /process-models: # process_model_add post: operationId: spiffworkflow_backend.routes.process_api_blueprint.process_model_add diff --git a/src/spiffworkflow_backend/routes/process_api_blueprint.py b/src/spiffworkflow_backend/routes/process_api_blueprint.py index 9ecee852..7f848d96 100644 --- a/src/spiffworkflow_backend/routes/process_api_blueprint.py +++ b/src/spiffworkflow_backend/routes/process_api_blueprint.py @@ -198,10 +198,12 @@ def process_model_show(process_group_id: str, process_model_id: str) -> Any: def process_model_list( - process_group_id: str, page: int = 1, per_page: int = 100 + process_group_identifier: Optional[str] = None, page: int = 1, per_page: int = 100 ) -> flask.wrappers.Response: """Process model list!""" - process_models = ProcessModelService().get_process_models(process_group_id) + process_models = ProcessModelService().get_process_models( + process_group_id=process_group_identifier + ) batch = ProcessModelService().get_batch( process_models, page=page, per_page=per_page ) @@ -538,10 +540,11 @@ def process_instance_list( process_status: Optional[str] = None, ) -> flask.wrappers.Response: """Process_instance_list.""" - process_instance_query = ProcessInstanceModel.query if process_model_identifier is not None and process_group_identifier is not None: - process_model = get_process_model(process_model_identifier, process_group_identifier) + process_model = get_process_model( + process_model_identifier, process_group_identifier + ) process_instance_query = process_instance_query.filter_by( process_model_identifier=process_model.id @@ -561,15 +564,25 @@ def process_instance_list( ) if start_from is not None: - process_instance_query = process_instance_query.filter(ProcessInstanceModel.start_in_seconds >= start_from) + process_instance_query = process_instance_query.filter( + ProcessInstanceModel.start_in_seconds >= start_from + ) if start_till is not None: - process_instance_query = process_instance_query.filter(ProcessInstanceModel.start_in_seconds <= start_till) + process_instance_query = process_instance_query.filter( + ProcessInstanceModel.start_in_seconds <= start_till + ) if end_from is not None: - process_instance_query = process_instance_query.filter(ProcessInstanceModel.end_in_seconds >= end_from) + process_instance_query = process_instance_query.filter( + ProcessInstanceModel.end_in_seconds >= end_from + ) if end_till is not None: - process_instance_query = process_instance_query.filter(ProcessInstanceModel.end_in_seconds <= end_till) + process_instance_query = process_instance_query.filter( + ProcessInstanceModel.end_in_seconds <= end_till + ) if process_status is not None: - process_instance_query = process_instance_query.filter(ProcessInstanceModel.status == process_status) + process_instance_query = process_instance_query.filter( + ProcessInstanceModel.status == process_status + ) process_instances = process_instance_query.order_by( ProcessInstanceModel.start_in_seconds.desc(), ProcessInstanceModel.id.desc() # type: ignore diff --git a/tests/spiffworkflow_backend/integration/test_process_api.py b/tests/spiffworkflow_backend/integration/test_process_api.py index de33f3e7..074e2d4d 100644 --- a/tests/spiffworkflow_backend/integration/test_process_api.py +++ b/tests/spiffworkflow_backend/integration/test_process_api.py @@ -216,7 +216,7 @@ class TestProcessApi(BaseTest): # get all models response = client.get( - f"/v1.0/process-groups/{group_id}/process-models", + f"/v1.0/process-models?process_group_identifier={group_id}", headers=logged_in_headers(user), ) assert response.json is not None @@ -227,7 +227,7 @@ class TestProcessApi(BaseTest): # get first page, 1 per page response = client.get( - f"/v1.0/process-groups/{group_id}/process-models?page=1&per_page=1", + f"/v1.0/process-models?page=1&per_page=1&process_group_identifier={group_id}", headers=logged_in_headers(user), ) assert response.json is not None @@ -239,7 +239,7 @@ class TestProcessApi(BaseTest): # get second page, 1 per page response = client.get( - f"/v1.0/process-groups/{group_id}/process-models?page=2&per_page=1", + f"/v1.0/process-models?page=2&per_page=1&process_group_identifier={group_id}", headers=logged_in_headers(user), ) assert response.json is not None @@ -251,7 +251,7 @@ class TestProcessApi(BaseTest): # get first page, 3 per page response = client.get( - f"/v1.0/process-groups/{group_id}/process-models?page=1&per_page=3", + f"/v1.0/process-models?page=1&per_page=3&process_group_identifier={group_id}", headers=logged_in_headers(user), ) assert response.json is not None @@ -263,7 +263,7 @@ class TestProcessApi(BaseTest): # get second page, 3 per page response = client.get( - f"/v1.0/process-groups/{group_id}/process-models?page=2&per_page=3", + f"/v1.0/process-models?page=2&per_page=3&process_group_identifier={group_id}", headers=logged_in_headers(user), ) # there should only be 2 left @@ -436,11 +436,11 @@ class TestProcessApi(BaseTest): """Test_process_model_file_update.""" self.create_spec_file(client) - spec = load_test_spec("random_fact") + process_model = load_test_spec("random_fact") data = {"key1": "THIS DATA"} user = self.find_or_create_user() response = client.put( - f"/v1.0/process-models/{spec.process_group_id}/{spec.id}/files/random_fact.svg", + f"/v1.0/process-models/{process_model.process_group_id}/{process_model.id}/files/random_fact.svg", data=data, follow_redirects=True, content_type="multipart/form-data", @@ -457,11 +457,11 @@ class TestProcessApi(BaseTest): """Test_process_model_file_update.""" self.create_spec_file(client) - spec = load_test_spec("random_fact") + process_model = load_test_spec("random_fact") data = {"file": (io.BytesIO(b""), "random_fact.svg")} user = self.find_or_create_user() response = client.put( - f"/v1.0/process-models/{spec.process_group_id}/{spec.id}/files/random_fact.svg", + f"/v1.0/process-models/{process_model.process_group_id}/{process_model.id}/files/random_fact.svg", data=data, follow_redirects=True, content_type="multipart/form-data", @@ -478,12 +478,12 @@ class TestProcessApi(BaseTest): """Test_process_model_file_update.""" original_file = self.create_spec_file(client) - spec = load_test_spec("random_fact") + process_model = load_test_spec("random_fact") new_file_contents = b"THIS_IS_NEW_DATA" data = {"file": (io.BytesIO(new_file_contents), "random_fact.svg")} user = self.find_or_create_user() response = client.put( - f"/v1.0/process-models/{spec.process_group_id}/{spec.id}/files/random_fact.svg", + f"/v1.0/process-models/{process_model.process_group_id}/{process_model.id}/files/random_fact.svg", data=data, follow_redirects=True, content_type="multipart/form-data", @@ -495,7 +495,7 @@ class TestProcessApi(BaseTest): assert response.json["ok"] response = client.get( - f"/v1.0/process-models/{spec.process_group_id}/{spec.id}/files/random_fact.svg", + f"/v1.0/process-models/{process_model.process_group_id}/{process_model.id}/files/random_fact.svg", headers=logged_in_headers(user), ) assert response.status_code == 200 @@ -509,10 +509,10 @@ class TestProcessApi(BaseTest): """Test_process_model_file_update.""" self.create_spec_file(client) - spec = load_test_spec("random_fact") + process_model = load_test_spec("random_fact") user = self.find_or_create_user() response = client.delete( - f"/v1.0/process-models/INCORRECT-NON-EXISTENT-GROUP/{spec.id}/files/random_fact.svg", + f"/v1.0/process-models/INCORRECT-NON-EXISTENT-GROUP/{process_model.id}/files/random_fact.svg", follow_redirects=True, headers=logged_in_headers(user), ) @@ -527,10 +527,10 @@ class TestProcessApi(BaseTest): """Test_process_model_file_update.""" self.create_spec_file(client) - spec = load_test_spec("random_fact") + process_model = load_test_spec("random_fact") user = self.find_or_create_user() response = client.delete( - f"/v1.0/process-models/{spec.process_group_id}/{spec.id}/files/random_fact_DOES_NOT_EXIST.svg", + f"/v1.0/process-models/{process_model.process_group_id}/{process_model.id}/files/random_fact_DOES_NOT_EXIST.svg", follow_redirects=True, headers=logged_in_headers(user), ) @@ -545,10 +545,10 @@ class TestProcessApi(BaseTest): """Test_process_model_file_update.""" self.create_spec_file(client) - spec = load_test_spec("random_fact") + process_model = load_test_spec("random_fact") user = self.find_or_create_user() response = client.delete( - f"/v1.0/process-models/{spec.process_group_id}/{spec.id}/files/random_fact.svg", + f"/v1.0/process-models/{process_model.process_group_id}/{process_model.id}/files/random_fact.svg", follow_redirects=True, headers=logged_in_headers(user), ) @@ -558,7 +558,7 @@ class TestProcessApi(BaseTest): assert response.json["ok"] response = client.get( - f"/v1.0/process-models/{spec.process_group_id}/{spec.id}/files/random_fact.svg", + f"/v1.0/process-models/{process_model.process_group_id}/{process_model.id}/files/random_fact.svg", headers=logged_in_headers(user), ) assert response.status_code == 404 @@ -586,9 +586,9 @@ class TestProcessApi(BaseTest): ) -> None: """Test_get_workflow_from_workflow_spec.""" user = self.find_or_create_user() - spec = load_test_spec("hello_world") + process_model = load_test_spec("hello_world") response = client.post( - f"/v1.0/process-models/{spec.process_group_id}/{spec.id}/process-instances", + f"/v1.0/process-models/{process_model.process_group_id}/{process_model.id}/process-instances", headers=logged_in_headers(user), ) assert response.status_code == 201 @@ -931,7 +931,7 @@ class TestProcessApi(BaseTest): ) response = client.get( - f"/v1.0/process-models/{test_process_group_id}/{process_model_dir_name}/process-instances", + "/v1.0/process-instances", headers=logged_in_headers(user), ) assert response.status_code == 200 @@ -979,7 +979,7 @@ class TestProcessApi(BaseTest): ) response = client.get( - f"/v1.0/process-models/{test_process_group_id}/{process_model_dir_name}/process-instances?per_page=2&page=3", + "/v1.0/process-instances?per_page=2&page=3", headers=logged_in_headers(user), ) assert response.status_code == 200 @@ -990,7 +990,7 @@ class TestProcessApi(BaseTest): assert response.json["pagination"]["total"] == 5 response = client.get( - f"/v1.0/process-models/{test_process_group_id}/{process_model_dir_name}/process-instances?per_page=2&page=1", + "/v1.0/process-instances?per_page=2&page=1", headers=logged_in_headers(user), ) assert response.status_code == 200 @@ -1027,7 +1027,7 @@ class TestProcessApi(BaseTest): # Without filtering we should get all 5 instances response = client.get( - f"/v1.0/process-models/{test_process_group_id}/{test_process_model_id}/process-instances", + f"/v1.0/process-instances?process_group_identifier={test_process_group_id}&process_model_identifier={test_process_model_id}", headers=logged_in_headers(user), ) assert response.json is not None @@ -1038,7 +1038,7 @@ class TestProcessApi(BaseTest): # we should get 1 instance each time for i in range(5): response = client.get( - f"/v1.0/process-models/{test_process_group_id}/{test_process_model_id}/process-instances?process_status={ProcessInstanceStatus[statuses[i]].value}", + f"/v1.0/process-instances?process_status={ProcessInstanceStatus[statuses[i]].value}&process_group_identifier={test_process_group_id}&process_model_identifier={test_process_model_id}", headers=logged_in_headers(user), ) assert response.json is not None @@ -1049,7 +1049,7 @@ class TestProcessApi(BaseTest): # filter by start/end seconds # start > 1000 - this should eliminate the first response = client.get( - f"/v1.0/process-models/{test_process_group_id}/{test_process_model_id}/process-instances?start_from=1001", + "/v1.0/process-instances?start_from=1001", headers=logged_in_headers(user), ) assert response.json is not None @@ -1060,7 +1060,7 @@ class TestProcessApi(BaseTest): # start > 2000, end < 5000 - this should eliminate the first 2 and the last response = client.get( - f"/v1.0/process-models/{test_process_group_id}/{test_process_model_id}/process-instances?start_from=2001&end_till=5999", + "/v1.0/process-instances?start_from=2001&end_till=5999", headers=logged_in_headers(user), ) assert response.json is not None @@ -1071,7 +1071,7 @@ class TestProcessApi(BaseTest): # start > 1000, start < 4000 - this should eliminate the first and the last 2 response = client.get( - f"/v1.0/process-models/{test_process_group_id}/{test_process_model_id}/process-instances?start_from=1001&start_till=3999", + "/v1.0/process-instances?start_from=1001&start_till=3999", headers=logged_in_headers(user), ) assert response.json is not None @@ -1082,7 +1082,7 @@ class TestProcessApi(BaseTest): # end > 2000, end < 6000 - this should eliminate the first and the last response = client.get( - f"/v1.0/process-models/{test_process_group_id}/{test_process_model_id}/process-instances?end_from=2001&end_till=5999", + "/v1.0/process-instances?end_from=2001&end_till=5999", headers=logged_in_headers(user), ) assert response.json is not None From 98fd9d845b47a143c85a1255882088d23a98c990 Mon Sep 17 00:00:00 2001 From: jasquat Date: Thu, 22 Sep 2022 16:48:14 -0400 Subject: [PATCH 03/12] set task name in logs if available and catch errors to ensure we set the log level back to the correct value w/ burnettk --- .../{1ccfcb7baf22_.py => e9763012b98b_.py} | 7 ++++--- poetry.lock | 2 +- src/spiffworkflow_backend/models/spiff_logging.py | 1 + .../services/logging_service.py | 2 ++ .../services/process_instance_processor.py | 15 ++++++++++----- 5 files changed, 18 insertions(+), 9 deletions(-) rename migrations/versions/{1ccfcb7baf22_.py => e9763012b98b_.py} (99%) diff --git a/migrations/versions/1ccfcb7baf22_.py b/migrations/versions/e9763012b98b_.py similarity index 99% rename from migrations/versions/1ccfcb7baf22_.py rename to migrations/versions/e9763012b98b_.py index abc8d137..64519620 100644 --- a/migrations/versions/1ccfcb7baf22_.py +++ b/migrations/versions/e9763012b98b_.py @@ -1,8 +1,8 @@ """empty message -Revision ID: 1ccfcb7baf22 +Revision ID: e9763012b98b Revises: -Create Date: 2022-09-21 16:05:59.406883 +Create Date: 2022-09-22 16:15:16.490215 """ from alembic import op @@ -10,7 +10,7 @@ import sqlalchemy as sa # revision identifiers, used by Alembic. -revision = '1ccfcb7baf22' +revision = 'e9763012b98b' down_revision = None branch_labels = None depends_on = None @@ -210,6 +210,7 @@ def upgrade(): sa.Column('process_instance_id', sa.Integer(), nullable=False), sa.Column('bpmn_process_identifier', sa.String(length=255), nullable=False), sa.Column('bpmn_task_identifier', sa.String(length=255), nullable=False), + sa.Column('bpmn_task_name', sa.String(length=255), nullable=True), sa.Column('bpmn_task_type', sa.String(length=255), nullable=True), sa.Column('spiff_task_guid', sa.String(length=50), nullable=False), sa.Column('timestamp', sa.DECIMAL(precision=17, scale=6), nullable=False), diff --git a/poetry.lock b/poetry.lock index 9dbcc5a7..d1cee94a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1866,7 +1866,7 @@ pytz = "*" type = "git" url = "https://github.com/sartography/SpiffWorkflow" reference = "main" -resolved_reference = "082d116990fcada49c3287f585cfd3a46c55de0f" +resolved_reference = "8d79389816db6d70dfde34c2f83a2872cdf1148f" [[package]] name = "sqlalchemy" diff --git a/src/spiffworkflow_backend/models/spiff_logging.py b/src/spiffworkflow_backend/models/spiff_logging.py index 894d7500..a655ec51 100644 --- a/src/spiffworkflow_backend/models/spiff_logging.py +++ b/src/spiffworkflow_backend/models/spiff_logging.py @@ -19,6 +19,7 @@ class SpiffLoggingModel(SpiffworkflowBaseDBModel): process_instance_id: int = db.Column(ForeignKey(ProcessInstanceModel.id), nullable=False) # type: ignore bpmn_process_identifier: str = db.Column(db.String(255), nullable=False) bpmn_task_identifier: str = db.Column(db.String(255), nullable=False) + bpmn_task_name: str = db.Column(db.String(255), nullable=True) bpmn_task_type: str = db.Column(db.String(255), nullable=True) spiff_task_guid: str = db.Column(db.String(50), nullable=False) timestamp: float = db.Column(db.DECIMAL(17, 6), nullable=False) diff --git a/src/spiffworkflow_backend/services/logging_service.py b/src/spiffworkflow_backend/services/logging_service.py index 93c5ed6b..870031f3 100644 --- a/src/spiffworkflow_backend/services/logging_service.py +++ b/src/spiffworkflow_backend/services/logging_service.py @@ -176,6 +176,7 @@ class DBHandler(logging.Handler): bpmn_process_identifier = record.workflow # type: ignore spiff_task_guid = str(record.task_id) # type: ignore bpmn_task_identifier = str(record.task_spec) # type: ignore + bpmn_task_name = record.task_name if hasattr(record, "task_name") else None # type: ignore bpmn_task_type = record.task_type if hasattr(record, "task_type") else None # type: ignore timestamp = record.created message = record.msg if hasattr(record, "msg") else None @@ -184,6 +185,7 @@ class DBHandler(logging.Handler): process_instance_id=record.process_instance_id, # type: ignore bpmn_process_identifier=bpmn_process_identifier, spiff_task_guid=spiff_task_guid, + bpmn_task_name=bpmn_task_name, bpmn_task_identifier=bpmn_task_identifier, bpmn_task_type=bpmn_task_type, message=message, diff --git a/src/spiffworkflow_backend/services/process_instance_processor.py b/src/spiffworkflow_backend/services/process_instance_processor.py index 5ce14990..d3c3cccd 100644 --- a/src/spiffworkflow_backend/services/process_instance_processor.py +++ b/src/spiffworkflow_backend/services/process_instance_processor.py @@ -368,12 +368,17 @@ class ProcessInstanceProcessor: original_spiff_logger_log_level = spiff_logger.level spiff_logger.setLevel(logging.WARNING) - bpmn_process_instance = ( - ProcessInstanceProcessor._serializer.deserialize_json( - process_instance_model.bpmn_json + try: + bpmn_process_instance = ( + ProcessInstanceProcessor._serializer.deserialize_json( + process_instance_model.bpmn_json + ) ) - ) - spiff_logger.setLevel(original_spiff_logger_log_level) + except Exception as err: + raise(err) + finally: + spiff_logger.setLevel(original_spiff_logger_log_level) + bpmn_process_instance.script_engine = ( ProcessInstanceProcessor._script_engine ) From d689aad90efc432fc58369353b6465246a8b6c11 Mon Sep 17 00:00:00 2001 From: jasquat Date: Thu, 22 Sep 2022 17:23:30 -0400 Subject: [PATCH 04/12] cascade process instance delete to message instances w/ burnettk --- src/spiffworkflow_backend/models/message_correlation.py | 4 ++++ src/spiffworkflow_backend/models/message_instance.py | 1 + src/spiffworkflow_backend/models/process_instance.py | 2 ++ 3 files changed, 7 insertions(+) diff --git a/src/spiffworkflow_backend/models/message_correlation.py b/src/spiffworkflow_backend/models/message_correlation.py index ec97200b..8ab9f920 100644 --- a/src/spiffworkflow_backend/models/message_correlation.py +++ b/src/spiffworkflow_backend/models/message_correlation.py @@ -1,6 +1,8 @@ """Message_correlation.""" from dataclasses import dataclass +from sqlalchemy.orm import relationship + from flask_bpmn.models.db import db from flask_bpmn.models.db import SpiffworkflowBaseDBModel from sqlalchemy import ForeignKey @@ -36,3 +38,5 @@ class MessageCorrelationModel(SpiffworkflowBaseDBModel): value = db.Column(db.String(255), nullable=False, index=True) updated_at_in_seconds: int = db.Column(db.Integer) created_at_in_seconds: int = db.Column(db.Integer) + + message_correlations_message_instances = relationship("MessageCorrelationMessageInstanceModel", cascade="delete") diff --git a/src/spiffworkflow_backend/models/message_instance.py b/src/spiffworkflow_backend/models/message_instance.py index 99556502..bac4eb17 100644 --- a/src/spiffworkflow_backend/models/message_instance.py +++ b/src/spiffworkflow_backend/models/message_instance.py @@ -42,6 +42,7 @@ class MessageInstanceModel(SpiffworkflowBaseDBModel): process_instance_id: int = db.Column(ForeignKey(ProcessInstanceModel.id), nullable=False) # type: ignore message_model_id: int = db.Column(ForeignKey(MessageModel.id), nullable=False) message_model = relationship("MessageModel") + message_correlations_message_instances = relationship("MessageCorrelationMessageInstanceModel", cascade="delete") message_type: str = db.Column(db.String(20), nullable=False) payload: str = db.Column(db.JSON) diff --git a/src/spiffworkflow_backend/models/process_instance.py b/src/spiffworkflow_backend/models/process_instance.py index ea9fa615..ac0bb315 100644 --- a/src/spiffworkflow_backend/models/process_instance.py +++ b/src/spiffworkflow_backend/models/process_instance.py @@ -95,6 +95,8 @@ class ProcessInstanceModel(SpiffworkflowBaseDBModel): active_tasks = relationship("ActiveTaskModel", cascade="delete") # type: ignore task_events = relationship("TaskEventModel", cascade="delete") # type: ignore spiff_logs = relationship("SpiffLoggingModel", cascade="delete") # type: ignore + message_instances = relationship("MessageInstanceModel", cascade="delete") # type: ignore + message_correlations = relationship("MessageCorrelationModel", cascade="delete") # type: ignore bpmn_json: str | None = deferred(db.Column(db.JSON)) # type: ignore start_in_seconds: int | None = db.Column(db.Integer) From 9aa55dca6a5fbcaac7d06f81863ddb8a864724b1 Mon Sep 17 00:00:00 2001 From: burnettk Date: Thu, 22 Sep 2022 17:37:10 -0400 Subject: [PATCH 05/12] lint and pull new spiff --- poetry.lock | 2 +- .../models/message_correlation.py | 13 ++++++++++--- .../models/message_instance.py | 10 +++++++++- .../services/process_instance_processor.py | 2 +- 4 files changed, 21 insertions(+), 6 deletions(-) diff --git a/poetry.lock b/poetry.lock index d1cee94a..d770cbef 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1866,7 +1866,7 @@ pytz = "*" type = "git" url = "https://github.com/sartography/SpiffWorkflow" reference = "main" -resolved_reference = "8d79389816db6d70dfde34c2f83a2872cdf1148f" +resolved_reference = "dec9b4b942378d030ae73f1365dfbf108e6f7f8c" [[package]] name = "sqlalchemy" diff --git a/src/spiffworkflow_backend/models/message_correlation.py b/src/spiffworkflow_backend/models/message_correlation.py index 8ab9f920..baec8270 100644 --- a/src/spiffworkflow_backend/models/message_correlation.py +++ b/src/spiffworkflow_backend/models/message_correlation.py @@ -1,17 +1,22 @@ """Message_correlation.""" from dataclasses import dataclass - -from sqlalchemy.orm import relationship +from typing import TYPE_CHECKING from flask_bpmn.models.db import db from flask_bpmn.models.db import SpiffworkflowBaseDBModel from sqlalchemy import ForeignKey +from sqlalchemy.orm import relationship from spiffworkflow_backend.models.message_correlation_property import ( MessageCorrelationPropertyModel, ) from spiffworkflow_backend.models.process_instance import ProcessInstanceModel +if TYPE_CHECKING: + from spiffworkflow_backend.models.message_correlation_message_instance import ( # noqa: F401 + MessageCorrelationMessageInstanceModel, + ) + @dataclass class MessageCorrelationModel(SpiffworkflowBaseDBModel): @@ -39,4 +44,6 @@ class MessageCorrelationModel(SpiffworkflowBaseDBModel): updated_at_in_seconds: int = db.Column(db.Integer) created_at_in_seconds: int = db.Column(db.Integer) - message_correlations_message_instances = relationship("MessageCorrelationMessageInstanceModel", cascade="delete") + message_correlations_message_instances = relationship( + "MessageCorrelationMessageInstanceModel", cascade="delete" + ) diff --git a/src/spiffworkflow_backend/models/message_instance.py b/src/spiffworkflow_backend/models/message_instance.py index bac4eb17..c7b7f5fc 100644 --- a/src/spiffworkflow_backend/models/message_instance.py +++ b/src/spiffworkflow_backend/models/message_instance.py @@ -3,6 +3,7 @@ import enum from dataclasses import dataclass from typing import Any from typing import Optional +from typing import TYPE_CHECKING from flask_bpmn.models.db import db from flask_bpmn.models.db import SpiffworkflowBaseDBModel @@ -15,6 +16,11 @@ from sqlalchemy.orm.events import event from spiffworkflow_backend.models.message_model import MessageModel from spiffworkflow_backend.models.process_instance import ProcessInstanceModel +if TYPE_CHECKING: + from spiffworkflow_backend.models.message_correlation_message_instance import ( # noqa: F401 + MessageCorrelationMessageInstanceModel, + ) + class MessageTypes(enum.Enum): """MessageTypes.""" @@ -42,7 +48,9 @@ class MessageInstanceModel(SpiffworkflowBaseDBModel): process_instance_id: int = db.Column(ForeignKey(ProcessInstanceModel.id), nullable=False) # type: ignore message_model_id: int = db.Column(ForeignKey(MessageModel.id), nullable=False) message_model = relationship("MessageModel") - message_correlations_message_instances = relationship("MessageCorrelationMessageInstanceModel", cascade="delete") + message_correlations_message_instances = relationship( + "MessageCorrelationMessageInstanceModel", cascade="delete" + ) message_type: str = db.Column(db.String(20), nullable=False) payload: str = db.Column(db.JSON) diff --git a/src/spiffworkflow_backend/services/process_instance_processor.py b/src/spiffworkflow_backend/services/process_instance_processor.py index d3c3cccd..17c9a8b0 100644 --- a/src/spiffworkflow_backend/services/process_instance_processor.py +++ b/src/spiffworkflow_backend/services/process_instance_processor.py @@ -375,7 +375,7 @@ class ProcessInstanceProcessor: ) ) except Exception as err: - raise(err) + raise (err) finally: spiff_logger.setLevel(original_spiff_logger_log_level) From 6a7fcdfb4eb88675e62a298c454e9debf25263b0 Mon Sep 17 00:00:00 2001 From: jasquat Date: Fri, 23 Sep 2022 09:19:06 -0400 Subject: [PATCH 06/12] use instance path instead of root_path for nox --- src/spiffworkflow_backend/services/logging_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/spiffworkflow_backend/services/logging_service.py b/src/spiffworkflow_backend/services/logging_service.py index 2bb3ebaa..94a14da3 100644 --- a/src/spiffworkflow_backend/services/logging_service.py +++ b/src/spiffworkflow_backend/services/logging_service.py @@ -133,7 +133,7 @@ def setup_logger(app: Flask) -> None: spiff_logger_filehandler = None if app.config["SPIFFWORKFLOW_BACKEND_LOG_TO_FILE"]: spiff_logger_filehandler = logging.FileHandler( - f"{app.root_path}/../../log/{app.env}.log" + f"{app.instance_path}/../../log/{app.env}.log" ) spiff_logger_filehandler.setLevel(spiff_log_level) spiff_logger_filehandler.setFormatter(log_formatter) From 8bb45c67d37bf787e80076fb6916ae241564bf4a Mon Sep 17 00:00:00 2001 From: jasquat Date: Fri, 23 Sep 2022 09:45:12 -0400 Subject: [PATCH 07/12] avoid checking for mysql specific error messages so other dbs can still work --- .../spiffworkflow_backend/integration/test_secret_service.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/spiffworkflow_backend/integration/test_secret_service.py b/tests/spiffworkflow_backend/integration/test_secret_service.py index 36ce6c9b..18d58c69 100644 --- a/tests/spiffworkflow_backend/integration/test_secret_service.py +++ b/tests/spiffworkflow_backend/integration/test_secret_service.py @@ -94,7 +94,7 @@ class TestSecretService(SecretServiceTestHelpers): self.add_test_secret(user) with pytest.raises(ApiError) as ae: self.add_test_secret(user) - assert "Duplicate entry" in ae.value.message + assert "IntegrityError" in ae.value.message def test_get_secret(self, app: Flask, with_db_and_bpmn_file_cleanup: None) -> None: """Test_get_secret.""" @@ -245,7 +245,6 @@ class TestSecretService(SecretServiceTestHelpers): ) assert "Resource already exists" in ae.value.message assert "IntegrityError" in ae.value.message - assert "Duplicate entry" in ae.value.message def test_secret_add_allowed_process_bad_user_fails( self, app: Flask, client: FlaskClient, with_db_and_bpmn_file_cleanup: None @@ -286,7 +285,6 @@ class TestSecretService(SecretServiceTestHelpers): allowed_relative_path=process_model_relative_path, ) assert "Resource does not exist" in ae.value.message - print("test_secret_add_allowed_process_bad_secret") def test_secret_delete_allowed_process( self, app: Flask, client: FlaskClient, with_db_and_bpmn_file_cleanup: None @@ -438,7 +436,6 @@ class TestSecretServiceApi(SecretServiceTestHelpers): headers=self.logged_in_headers(user), ) assert secret_response.status_code == 404 - print("test_delete_secret_bad_key") def test_add_secret_allowed_process( self, app: Flask, client: FlaskClient, with_db_and_bpmn_file_cleanup: None From bc9d820e1d4fb2e114a90997570e31b44bebf3de Mon Sep 17 00:00:00 2001 From: jasquat Date: Fri, 23 Sep 2022 10:01:22 -0400 Subject: [PATCH 08/12] python 3.9 does not like the pipe style optional arg types --- src/spiffworkflow_backend/routes/process_api_blueprint.py | 2 +- src/spiffworkflow_backend/services/secret_service.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/spiffworkflow_backend/routes/process_api_blueprint.py b/src/spiffworkflow_backend/routes/process_api_blueprint.py index 65816cd5..ae0c31aa 100644 --- a/src/spiffworkflow_backend/routes/process_api_blueprint.py +++ b/src/spiffworkflow_backend/routes/process_api_blueprint.py @@ -1088,7 +1088,7 @@ def get_spiff_task_from_process_instance( # # Methods for secrets CRUD - maybe move somewhere else: # -def get_secret(key: str) -> str | None: +def get_secret(key: str) -> Optional[str]: """Get_secret.""" return SecretService.get_secret(key) diff --git a/src/spiffworkflow_backend/services/secret_service.py b/src/spiffworkflow_backend/services/secret_service.py index 9321ea52..de215a1c 100644 --- a/src/spiffworkflow_backend/services/secret_service.py +++ b/src/spiffworkflow_backend/services/secret_service.py @@ -54,7 +54,7 @@ class SecretService: return secret_model @staticmethod - def get_secret(key: str) -> str | None: + def get_secret(key: str) -> Optional[str]: """Get_secret.""" secret: SecretModel = ( db.session.query(SecretModel).filter(SecretModel.key == key).first() From 115e126873dc4de7b20698dd6bab596d77ea8df8 Mon Sep 17 00:00:00 2001 From: jasquat Date: Fri, 23 Sep 2022 10:33:12 -0400 Subject: [PATCH 09/12] log db validation error to see what postgres is doing --- src/spiffworkflow_backend/services/logging_service.py | 6 +++++- src/spiffworkflow_backend/services/secret_service.py | 8 ++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/spiffworkflow_backend/services/logging_service.py b/src/spiffworkflow_backend/services/logging_service.py index 94a14da3..9d98119e 100644 --- a/src/spiffworkflow_backend/services/logging_service.py +++ b/src/spiffworkflow_backend/services/logging_service.py @@ -1,6 +1,7 @@ """Logging_service.""" import json import logging +import re from typing import Any from typing import Optional @@ -113,6 +114,8 @@ def setup_logger(app: Flask) -> None: "%(asctime)s - %(name)s - %(levelname)s - %(message)s" ) + app.logger.debug("Printing log to create app logger") + # the json formatter is nice for real environments but makes # debugging locally a little more difficult if app.env != "development": @@ -140,7 +143,8 @@ def setup_logger(app: Flask) -> None: # make all loggers act the same for name in logging.root.manager.loggerDict: - if "spiff" not in name: + # use a regex so spiffworkflow_backend isn't filtered out + if not re.match(r"^spiff\b", name): the_logger = logging.getLogger(name) the_logger.setLevel(log_level) if spiff_logger_filehandler: diff --git a/src/spiffworkflow_backend/services/secret_service.py b/src/spiffworkflow_backend/services/secret_service.py index de215a1c..06fedaa3 100644 --- a/src/spiffworkflow_backend/services/secret_service.py +++ b/src/spiffworkflow_backend/services/secret_service.py @@ -1,6 +1,7 @@ """Secret_service.""" from typing import Optional +from flask import current_app from flask_bpmn.api.api_error import ApiError from flask_bpmn.models.db import db from sqlalchemy.exc import IntegrityError @@ -46,11 +47,14 @@ class SecretService: try: db.session.commit() except Exception as e: - raise ApiError( + ae = ApiError( code="create_secret_error", message=f"There was an error creating a secret with key: {key} and value ending with: {value[:-4]}. " f"Original error is {e}", - ) from e + ) + # log the error so we can see what postgres is doing + current_app.logger.error(ae) + raise ae from e return secret_model @staticmethod From da4c20524a91a461ace12ddcedf3bac9568e2be5 Mon Sep 17 00:00:00 2001 From: jasquat Date: Fri, 23 Sep 2022 10:46:03 -0400 Subject: [PATCH 10/12] code for api error codes instead of db errors since each db raises different errors for the samething --- src/spiffworkflow_backend/services/secret_service.py | 8 ++------ .../integration/test_secret_service.py | 4 ++-- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/src/spiffworkflow_backend/services/secret_service.py b/src/spiffworkflow_backend/services/secret_service.py index 06fedaa3..de215a1c 100644 --- a/src/spiffworkflow_backend/services/secret_service.py +++ b/src/spiffworkflow_backend/services/secret_service.py @@ -1,7 +1,6 @@ """Secret_service.""" from typing import Optional -from flask import current_app from flask_bpmn.api.api_error import ApiError from flask_bpmn.models.db import db from sqlalchemy.exc import IntegrityError @@ -47,14 +46,11 @@ class SecretService: try: db.session.commit() except Exception as e: - ae = ApiError( + raise ApiError( code="create_secret_error", message=f"There was an error creating a secret with key: {key} and value ending with: {value[:-4]}. " f"Original error is {e}", - ) - # log the error so we can see what postgres is doing - current_app.logger.error(ae) - raise ae from e + ) from e return secret_model @staticmethod diff --git a/tests/spiffworkflow_backend/integration/test_secret_service.py b/tests/spiffworkflow_backend/integration/test_secret_service.py index 18d58c69..abc38a52 100644 --- a/tests/spiffworkflow_backend/integration/test_secret_service.py +++ b/tests/spiffworkflow_backend/integration/test_secret_service.py @@ -94,7 +94,7 @@ class TestSecretService(SecretServiceTestHelpers): self.add_test_secret(user) with pytest.raises(ApiError) as ae: self.add_test_secret(user) - assert "IntegrityError" in ae.value.message + assert ae.value.code == "create_secret_error" def test_get_secret(self, app: Flask, with_db_and_bpmn_file_cleanup: None) -> None: """Test_get_secret.""" @@ -151,6 +151,7 @@ class TestSecretService(SecretServiceTestHelpers): with pytest.raises(ApiError) as ae: SecretService.update_secret(secret.key + "x", "some_new_value", user.id) assert "Resource does not exist" in ae.value.message + assert ae.value.code == "update_secret_error" def test_delete_secret( self, app: Flask, client: FlaskClient, with_db_and_bpmn_file_cleanup: None @@ -244,7 +245,6 @@ class TestSecretService(SecretServiceTestHelpers): allowed_relative_path=process_model_relative_path, ) assert "Resource already exists" in ae.value.message - assert "IntegrityError" in ae.value.message def test_secret_add_allowed_process_bad_user_fails( self, app: Flask, client: FlaskClient, with_db_and_bpmn_file_cleanup: None From aea56fc7500469a45fc0994828e2f6b0d2a7babf Mon Sep 17 00:00:00 2001 From: jasquat Date: Fri, 23 Sep 2022 14:19:57 -0400 Subject: [PATCH 11/12] allow process_status filter to be a csv string --- .../routes/process_api_blueprint.py | 3 ++- .../services/process_instance_processor.py | 2 +- .../integration/test_process_api.py | 10 ++++++++++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/spiffworkflow_backend/routes/process_api_blueprint.py b/src/spiffworkflow_backend/routes/process_api_blueprint.py index 998e4dff..898d9f11 100644 --- a/src/spiffworkflow_backend/routes/process_api_blueprint.py +++ b/src/spiffworkflow_backend/routes/process_api_blueprint.py @@ -583,8 +583,9 @@ def process_instance_list( ProcessInstanceModel.end_in_seconds <= end_till ) if process_status is not None: + process_status_array = process_status.split(",") process_instance_query = process_instance_query.filter( - ProcessInstanceModel.status == process_status + ProcessInstanceModel.status.in_(process_status_array) # type: ignore ) process_instances = process_instance_query.order_by( diff --git a/src/spiffworkflow_backend/services/process_instance_processor.py b/src/spiffworkflow_backend/services/process_instance_processor.py index 17c9a8b0..8fc15192 100644 --- a/src/spiffworkflow_backend/services/process_instance_processor.py +++ b/src/spiffworkflow_backend/services/process_instance_processor.py @@ -357,7 +357,7 @@ class ProcessInstanceProcessor: @staticmethod def __get_bpmn_process_instance( process_instance_model: ProcessInstanceModel, - spec: WorkflowSpec = None, + spec: Optional[WorkflowSpec] = None, validate_only: bool = False, subprocesses: Optional[IdToBpmnProcessSpecMapping] = None, ) -> BpmnWorkflow: diff --git a/tests/spiffworkflow_backend/integration/test_process_api.py b/tests/spiffworkflow_backend/integration/test_process_api.py index 192a7270..1aea2422 100644 --- a/tests/spiffworkflow_backend/integration/test_process_api.py +++ b/tests/spiffworkflow_backend/integration/test_process_api.py @@ -1050,6 +1050,16 @@ class TestProcessApi(BaseTest): assert len(results) == 1 assert results[0]["status"] == ProcessInstanceStatus[statuses[i]].value + response = client.get( + f"/v1.0/process-instances?process_status=not_started,complete&process_group_identifier={test_process_group_id}&process_model_identifier={test_process_model_id}", + headers=self.logged_in_headers(user), + ) + assert response.json is not None + results = response.json["results"] + assert len(results) == 2 + assert results[0]["status"] in ['complete', 'not_started'] + assert results[1]["status"] in ['complete', 'not_started'] + # filter by start/end seconds # start > 1000 - this should eliminate the first response = client.get( From ccdeaceda5731a881570403db047a488a34d4453 Mon Sep 17 00:00:00 2001 From: jasquat Date: Fri, 23 Sep 2022 16:12:46 -0400 Subject: [PATCH 12/12] add dmn files when processing a call activity w/ burnettk cullerton --- .../services/process_instance_processor.py | 8 ++++++-- .../spiffworkflow_backend/integration/test_process_api.py | 4 ++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/spiffworkflow_backend/services/process_instance_processor.py b/src/spiffworkflow_backend/services/process_instance_processor.py index 8fc15192..8545445d 100644 --- a/src/spiffworkflow_backend/services/process_instance_processor.py +++ b/src/spiffworkflow_backend/services/process_instance_processor.py @@ -568,10 +568,14 @@ class ProcessInstanceProcessor: bpmn_process_identifier ) new_bpmn_files.add(new_bpmn_file_full_path) + dmn_file_glob = os.path.join( + os.path.dirname(new_bpmn_file_full_path), "*.dmn" + ) + parser.add_dmn_files_by_glob(dmn_file_glob) processed_identifiers.add(bpmn_process_identifier) - for new_bpmn_file_full_path in new_bpmn_files: - parser.add_bpmn_file(new_bpmn_file_full_path) + if new_bpmn_files: + parser.add_bpmn_files(new_bpmn_files) ProcessInstanceProcessor.update_spiff_parser_with_all_process_dependency_files( parser, processed_identifiers ) diff --git a/tests/spiffworkflow_backend/integration/test_process_api.py b/tests/spiffworkflow_backend/integration/test_process_api.py index 1aea2422..f08a66b2 100644 --- a/tests/spiffworkflow_backend/integration/test_process_api.py +++ b/tests/spiffworkflow_backend/integration/test_process_api.py @@ -1057,8 +1057,8 @@ class TestProcessApi(BaseTest): assert response.json is not None results = response.json["results"] assert len(results) == 2 - assert results[0]["status"] in ['complete', 'not_started'] - assert results[1]["status"] in ['complete', 'not_started'] + assert results[0]["status"] in ["complete", "not_started"] + assert results[1]["status"] in ["complete", "not_started"] # filter by start/end seconds # start > 1000 - this should eliminate the first