From f1b67180686f8bc76f4ffc8d71d5dcf056c17bc7 Mon Sep 17 00:00:00 2001 From: Aaron Louie Date: Tue, 10 Mar 2020 15:46:14 -0400 Subject: [PATCH 1/7] Adds task counts --- crc/api.yml | 6 ++++++ crc/api/workflow.py | 7 ++++--- crc/models/api_models.py | 37 ++++++++++++++++++++++++++++++++----- 3 files changed, 42 insertions(+), 8 deletions(-) diff --git a/crc/api.yml b/crc/api.yml index ab80f01d..cd3ea6fe 100644 --- a/crc/api.yml +++ b/crc/api.yml @@ -749,6 +749,12 @@ components: type: string is_latest_spec: type: boolean + num_tasks_total: + type: integer + num_tasks_complete: + type: integer + num_tasks_incomplete: + type: integer example: id: 291234 diff --git a/crc/api/workflow.py b/crc/api/workflow.py index e14023ea..8b4d6554 100644 --- a/crc/api/workflow.py +++ b/crc/api/workflow.py @@ -61,7 +61,6 @@ def delete_workflow_specification(spec_id): session.query(WorkflowModel).filter_by(workflow_spec_id=spec_id).delete() session.query(WorkflowSpecModel).filter_by(id=spec_id).delete() - session.commit() @@ -94,6 +93,7 @@ def delete(workflow_id): session.query(WorkflowModel).filter_by(id=workflow_id).delete() session.commit() + def get_task(workflow_id, task_id): workflow = session.query(WorkflowModel).filter_by(id=workflow_id).first() return workflow.bpmn_workflow().get_task(task_id) @@ -111,5 +111,6 @@ def update_task(workflow_id, task_id, body): workflow_model.bpmn_workflow_json = processor.serialize() session.add(workflow_model) session.commit() - return WorkflowApiSchema().dump(__get_workflow_api_model(processor) - ) + return WorkflowApiSchema().dump(__get_workflow_api_model(processor)) + + diff --git a/crc/models/api_models.py b/crc/models/api_models.py index c8575440..55ca3082 100644 --- a/crc/models/api_models.py +++ b/crc/models/api_models.py @@ -108,17 +108,44 @@ class WorkflowApi(object): class WorkflowApiSchema(ma.Schema): class Meta: model = WorkflowApi - fields = ["id", "status", - "user_tasks", "last_task", "next_task", - "workflow_spec_id", - "spec_version", "is_latest_spec"] + fields = ["id", "status", "user_tasks", "last_task", "next_task", + "workflow_spec_id", "spec_version", "is_latest_spec", + "num_tasks_total", "num_tasks_complete", "num_tasks_incomplete"] unknown = INCLUDE status = EnumField(WorkflowStatus) user_tasks = marshmallow.fields.List(marshmallow.fields.Nested(TaskSchema, dump_only=True)) last_task = marshmallow.fields.Nested(TaskSchema, dump_only=True) next_task = marshmallow.fields.Nested(TaskSchema, dump_only=True, required=False) + num_tasks_total = marshmallow.fields.Method('get_num_tasks_total') + num_tasks_complete = marshmallow.fields.Method('get_num_tasks_complete') + num_tasks_incomplete = marshmallow.fields.Method('get_num_tasks_incomplete') @marshmallow.post_load def make_workflow(self, data, **kwargs): - return WorkflowApi(**data) + keys = [ + 'id', + 'status', + 'user_tasks', + 'last_task', + 'next_task', + 'workflow_spec_id', + 'spec_version', + 'is_latest_spec' + ] + filtered_fields = {key: data[key] for key in keys} + return WorkflowApi(**filtered_fields) + + def get_num_tasks_total(self, obj): + tasks = list(obj.user_tasks) + return len(tasks) + + def get_num_tasks_complete(self, obj): + complete_states = ['CANCELLED', 'COMPLETED'] + tasks = list(obj.user_tasks) + return sum(1 for t in tasks if t.state in complete_states) + + def get_num_tasks_incomplete(self, obj): + incomplete_states = ['MAYBE', 'LIKELY', 'FUTURE', 'WAITING', 'READY'] + tasks = list(obj.user_tasks) + return sum(1 for t in tasks if t.state in incomplete_states) From be17c2159c83276e55f55628ac4bd5625e6d1c86 Mon Sep 17 00:00:00 2001 From: Aaron Louie Date: Tue, 10 Mar 2020 21:29:57 -0400 Subject: [PATCH 2/7] Adds workflow stats and task events models and schemas. Updates workflow stats and logs task completion when task data is updated. --- crc/api/workflow.py | 55 +++++++++++++++++++++++++- crc/models/api_models.py | 17 -------- crc/models/stats.py | 41 +++++++++++++++++++ crc/models/study.py | 2 - migrations/versions/90dd63672e0a_.py | 59 ++++++++++++++++++++++++++++ 5 files changed, 154 insertions(+), 20 deletions(-) create mode 100644 crc/models/stats.py create mode 100644 migrations/versions/90dd63672e0a_.py diff --git a/crc/api/workflow.py b/crc/api/workflow.py index 8b4d6554..61245a11 100644 --- a/crc/api/workflow.py +++ b/crc/api/workflow.py @@ -1,4 +1,6 @@ import uuid +from datetime import datetime +from flask import g from crc.api.file import delete_file from crc import session @@ -7,6 +9,7 @@ from crc.models.api_models import Task, WorkflowApi, WorkflowApiSchema from crc.models.workflow import WorkflowModel, WorkflowSpecModelSchema, WorkflowSpecModel from crc.services.workflow_processor import WorkflowProcessor from crc.models.file import FileModel +from crc.models.stats import WorkflowStatsModel, WorkflowStatsModelSchema, TaskEventModel, TaskEventModelSchema def all_specifications(): @@ -99,6 +102,22 @@ def get_task(workflow_id, task_id): return workflow.bpmn_workflow().get_task(task_id) +def log_task_complete(workflow_model, task_id): + task_event = TaskEventModel( + study_id=workflow_model.study_id, + user_uid=g.user.uid, + workflow_id=workflow_model.id, + workflow_spec_id=workflow_model.workflow_spec_id, + spec_version=workflow_model.spec_version, + task_id=task_id, + task_state='COMPLETE', + date=datetime.now(), + ) + session.add(task_event) + session.commit() + return TaskEventModelSchema().dump(task_event) + + def update_task(workflow_id, task_id, body): workflow_model = session.query(WorkflowModel).filter_by(id=workflow_id).first() processor = WorkflowProcessor(workflow_model) @@ -111,6 +130,40 @@ def update_task(workflow_id, task_id, body): workflow_model.bpmn_workflow_json = processor.serialize() session.add(workflow_model) session.commit() - return WorkflowApiSchema().dump(__get_workflow_api_model(processor)) + + workflow_api_model =__get_workflow_api_model(processor) + update_workflow_stats(workflow_model, workflow_api_model) + log_task_complete(workflow_model, task_id) + return WorkflowApiSchema().dump(workflow_api_model) + + +def update_workflow_stats(workflow_model, workflow_api_model): + stats = session.query(WorkflowStatsModel)\ + .filter_by(study_id=workflow_model.study_id)\ + .filter_by(workflow_id=workflow_model.id)\ + .filter_by(workflow_spec_id=workflow_model.workflow_spec_id)\ + .filter_by(spec_version=workflow_model.spec_version)\ + .first() + + if stats is None: + stats = WorkflowStatsModel( + study_id=workflow_model.study_id, + workflow_id=workflow_model.id, + workflow_spec_id=workflow_model.workflow_spec_id, + spec_version=workflow_model.spec_version, + ) + + complete_states = ['CANCELLED', 'COMPLETED'] + incomplete_states = ['MAYBE', 'LIKELY', 'FUTURE', 'WAITING', 'READY'] + tasks = list(workflow_api_model.user_tasks) + stats.num_tasks_total = len(tasks) + stats.num_tasks_complete = sum(1 for t in tasks if t.state in complete_states) + stats.num_tasks_incomplete = sum(1 for t in tasks if t.state in incomplete_states) + stats.last_updated = datetime.now() + + session.add(stats) + session.commit() + return WorkflowStatsModelSchema().dump(stats) + diff --git a/crc/models/api_models.py b/crc/models/api_models.py index 55ca3082..0e325f9d 100644 --- a/crc/models/api_models.py +++ b/crc/models/api_models.py @@ -117,9 +117,6 @@ class WorkflowApiSchema(ma.Schema): user_tasks = marshmallow.fields.List(marshmallow.fields.Nested(TaskSchema, dump_only=True)) last_task = marshmallow.fields.Nested(TaskSchema, dump_only=True) next_task = marshmallow.fields.Nested(TaskSchema, dump_only=True, required=False) - num_tasks_total = marshmallow.fields.Method('get_num_tasks_total') - num_tasks_complete = marshmallow.fields.Method('get_num_tasks_complete') - num_tasks_incomplete = marshmallow.fields.Method('get_num_tasks_incomplete') @marshmallow.post_load def make_workflow(self, data, **kwargs): @@ -135,17 +132,3 @@ class WorkflowApiSchema(ma.Schema): ] filtered_fields = {key: data[key] for key in keys} return WorkflowApi(**filtered_fields) - - def get_num_tasks_total(self, obj): - tasks = list(obj.user_tasks) - return len(tasks) - - def get_num_tasks_complete(self, obj): - complete_states = ['CANCELLED', 'COMPLETED'] - tasks = list(obj.user_tasks) - return sum(1 for t in tasks if t.state in complete_states) - - def get_num_tasks_incomplete(self, obj): - incomplete_states = ['MAYBE', 'LIKELY', 'FUTURE', 'WAITING', 'READY'] - tasks = list(obj.user_tasks) - return sum(1 for t in tasks if t.state in incomplete_states) diff --git a/crc/models/stats.py b/crc/models/stats.py new file mode 100644 index 00000000..70132b22 --- /dev/null +++ b/crc/models/stats.py @@ -0,0 +1,41 @@ +from marshmallow_sqlalchemy import ModelSchema + +from crc import db + + +class WorkflowStatsModel(db.Model): + __tablename__ = 'workflow_stats' + id = db.Column(db.Integer, primary_key=True) + study_id = db.Column(db.Integer, db.ForeignKey('study.id'), nullable=False) + workflow_id = db.Column(db.Integer, db.ForeignKey('workflow.id'), nullable=False) + workflow_spec_id = db.Column(db.String, db.ForeignKey('workflow_spec.id')) + spec_version = db.Column(db.String) + num_tasks_total = db.Column(db.Integer) + num_tasks_complete = db.Column(db.Integer) + num_tasks_incomplete = db.Column(db.Integer) + last_updated = db.Column(db.DateTime) + + +class WorkflowStatsModelSchema(ModelSchema): + class Meta: + model = WorkflowStatsModel + include_fk = True # Includes foreign keys + + +class TaskEventModel(db.Model): + __tablename__ = 'task_event' + id = db.Column(db.Integer, primary_key=True) + study_id = db.Column(db.Integer, db.ForeignKey('study.id'), nullable=False) + user_uid = db.Column(db.String, db.ForeignKey('user.uid'), nullable=False) + workflow_id = db.Column(db.Integer, db.ForeignKey('workflow.id'), nullable=False) + workflow_spec_id = db.Column(db.String, db.ForeignKey('workflow_spec.id')) + spec_version = db.Column(db.String) + task_id = db.Column(db.String) + task_state = db.Column(db.String) + date = db.Column(db.DateTime) + + +class TaskEventModelSchema(ModelSchema): + class Meta: + model = TaskEventModel + include_fk = True # Includes foreign keys diff --git a/crc/models/study.py b/crc/models/study.py index 815acb49..ac7d708d 100644 --- a/crc/models/study.py +++ b/crc/models/study.py @@ -28,5 +28,3 @@ class StudyModelSchema(ModelSchema): include_fk = True # Includes foreign keys protocol_builder_status = EnumField(ProtocolBuilderStatus) - - diff --git a/migrations/versions/90dd63672e0a_.py b/migrations/versions/90dd63672e0a_.py new file mode 100644 index 00000000..819e4234 --- /dev/null +++ b/migrations/versions/90dd63672e0a_.py @@ -0,0 +1,59 @@ +"""empty message + +Revision ID: 90dd63672e0a +Revises: 8856126b6658 +Create Date: 2020-03-10 21:16:38.827156 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '90dd63672e0a' +down_revision = '8856126b6658' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('task_event', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('study_id', sa.Integer(), nullable=False), + sa.Column('user_uid', sa.String(), nullable=False), + sa.Column('workflow_id', sa.Integer(), nullable=False), + sa.Column('workflow_spec_id', sa.String(), nullable=True), + sa.Column('spec_version', sa.String(), nullable=True), + sa.Column('task_id', sa.String(), nullable=True), + sa.Column('task_state', sa.String(), nullable=True), + sa.Column('date', sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint(['study_id'], ['study.id'], ), + sa.ForeignKeyConstraint(['user_uid'], ['user.uid'], ), + sa.ForeignKeyConstraint(['workflow_id'], ['workflow.id'], ), + sa.ForeignKeyConstraint(['workflow_spec_id'], ['workflow_spec.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('workflow_stats', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('study_id', sa.Integer(), nullable=False), + sa.Column('workflow_id', sa.Integer(), nullable=False), + sa.Column('workflow_spec_id', sa.String(), nullable=True), + sa.Column('spec_version', sa.String(), nullable=True), + sa.Column('num_tasks_total', sa.Integer(), nullable=True), + sa.Column('num_tasks_complete', sa.Integer(), nullable=True), + sa.Column('num_tasks_incomplete', sa.Integer(), nullable=True), + sa.Column('last_updated', sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint(['study_id'], ['study.id'], ), + sa.ForeignKeyConstraint(['workflow_id'], ['workflow.id'], ), + sa.ForeignKeyConstraint(['workflow_spec_id'], ['workflow_spec.id'], ), + sa.PrimaryKeyConstraint('id') + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('workflow_stats') + op.drop_table('task_event') + # ### end Alembic commands ### From 1119bb2b6ccb7bf07d65e6df46e42b58088c3935 Mon Sep 17 00:00:00 2001 From: Aaron Louie Date: Wed, 11 Mar 2020 12:35:37 -0400 Subject: [PATCH 3/7] Tests that task counts and events are logged. --- tests/test_tasks_api.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test_tasks_api.py b/tests/test_tasks_api.py index fabe6d98..9676a9a2 100644 --- a/tests/test_tasks_api.py +++ b/tests/test_tasks_api.py @@ -7,6 +7,7 @@ from crc.models.file import FileModelSchema from crc.models.study import StudyModel from crc.models.workflow import WorkflowSpecModelSchema, WorkflowModel, WorkflowStatus from crc.services.workflow_processor import WorkflowProcessor +from crc.models.stats import WorkflowStatsModel, TaskEventModel from tests.base_test import BaseTest @@ -40,6 +41,19 @@ class TestTasksApi(BaseTest): data=json.dumps(dict_data)) self.assert_success(rv) json_data = json.loads(rv.get_data(as_text=True)) + + num_stats = session.query(WorkflowStatsModel)\ + .filter_by(workflow_id=workflow.id)\ + .filter_by(workflow_spec_id=workflow.workflow_spec_id)\ + .count() + self.assertGreater(num_stats, 0) + + num_task_events = session.query(TaskEventModel)\ + .filter_by(workflow_id=workflow.id)\ + .filter_by(task_id=task.id)\ + .count() + self.assertGreater(num_task_events, 0) + workflow = WorkflowApiSchema().load(json_data) return workflow From bbfe9291e033acc3ccde4f8b2a439afefbdb658b Mon Sep 17 00:00:00 2001 From: Aaron Louie Date: Wed, 11 Mar 2020 14:28:53 -0400 Subject: [PATCH 4/7] Adds endpoint to get workflow stats. Adds a test for the endpoint. --- crc/api.yml | 21 +++++++++++++++ crc/api/stats.py | 57 ++++++++++++++++++++++++++++++++++++++++ crc/api/workflow.py | 58 +++-------------------------------------- tests/test_tasks_api.py | 41 ++++++++++++++++++++++++----- 4 files changed, 116 insertions(+), 61 deletions(-) create mode 100644 crc/api/stats.py diff --git a/crc/api.yml b/crc/api.yml index cd3ea6fe..5d120aaa 100644 --- a/crc/api.yml +++ b/crc/api.yml @@ -500,6 +500,27 @@ paths: responses: '204': description: The workflow was removed + /workflow/{workflow_id}/stats: + parameters: + - name: workflow_id + in: path + required: true + description: The id of the workflow + schema: + type: integer + format: int32 + get: + operationId: crc.api.stats.get_workflow_stats + summary: Stats about a given workflow + tags: + - Workflows and Tasks + responses: + '200': + description: Returns counts of complete, incomplete, and total number of tasks + content: + application/json: + schema: + $ref: "#/components/schemas/Workflow" # /v1.0/workflow/0/task/0 /workflow/{workflow_id}/task/{task_id}: parameters: diff --git a/crc/api/stats.py b/crc/api/stats.py new file mode 100644 index 00000000..1c78939c --- /dev/null +++ b/crc/api/stats.py @@ -0,0 +1,57 @@ +from datetime import datetime + +from flask import g + +from crc import session +from crc.models.stats import WorkflowStatsModel, WorkflowStatsModelSchema, TaskEventModel, TaskEventModelSchema + + +def get_workflow_stats(workflow_id): + schema = WorkflowStatsModelSchema() + workflow_model = session.query(WorkflowStatsModel).filter_by(id=workflow_id).first() + return schema.dump(workflow_model) + + +def update_workflow_stats(workflow_model, workflow_api_model): + stats = session.query(WorkflowStatsModel) \ + .filter_by(study_id=workflow_model.study_id) \ + .filter_by(workflow_id=workflow_model.id) \ + .filter_by(workflow_spec_id=workflow_model.workflow_spec_id) \ + .filter_by(spec_version=workflow_model.spec_version) \ + .first() + + if stats is None: + stats = WorkflowStatsModel( + study_id=workflow_model.study_id, + workflow_id=workflow_model.id, + workflow_spec_id=workflow_model.workflow_spec_id, + spec_version=workflow_model.spec_version, + ) + + complete_states = ['CANCELLED', 'COMPLETED'] + incomplete_states = ['MAYBE', 'LIKELY', 'FUTURE', 'WAITING', 'READY'] + tasks = list(workflow_api_model.user_tasks) + stats.num_tasks_total = len(tasks) + stats.num_tasks_complete = sum(1 for t in tasks if t.state in complete_states) + stats.num_tasks_incomplete = sum(1 for t in tasks if t.state in incomplete_states) + stats.last_updated = datetime.now() + + session.add(stats) + session.commit() + return WorkflowStatsModelSchema().dump(stats) + + +def log_task_complete(workflow_model, task_id): + task_event = TaskEventModel( + study_id=workflow_model.study_id, + user_uid=g.user.uid, + workflow_id=workflow_model.id, + workflow_spec_id=workflow_model.workflow_spec_id, + spec_version=workflow_model.spec_version, + task_id=task_id, + task_state='COMPLETE', + date=datetime.now(), + ) + session.add(task_event) + session.commit() + return TaskEventModelSchema().dump(task_event) diff --git a/crc/api/workflow.py b/crc/api/workflow.py index 61245a11..26060eab 100644 --- a/crc/api/workflow.py +++ b/crc/api/workflow.py @@ -1,15 +1,13 @@ import uuid -from datetime import datetime -from flask import g -from crc.api.file import delete_file +from api.stats import update_workflow_stats, log_task_complete from crc import session from crc.api.common import ApiError, ApiErrorSchema +from crc.api.file import delete_file from crc.models.api_models import Task, WorkflowApi, WorkflowApiSchema +from crc.models.file import FileModel from crc.models.workflow import WorkflowModel, WorkflowSpecModelSchema, WorkflowSpecModel from crc.services.workflow_processor import WorkflowProcessor -from crc.models.file import FileModel -from crc.models.stats import WorkflowStatsModel, WorkflowStatsModelSchema, TaskEventModel, TaskEventModelSchema def all_specifications(): @@ -102,22 +100,6 @@ def get_task(workflow_id, task_id): return workflow.bpmn_workflow().get_task(task_id) -def log_task_complete(workflow_model, task_id): - task_event = TaskEventModel( - study_id=workflow_model.study_id, - user_uid=g.user.uid, - workflow_id=workflow_model.id, - workflow_spec_id=workflow_model.workflow_spec_id, - spec_version=workflow_model.spec_version, - task_id=task_id, - task_state='COMPLETE', - date=datetime.now(), - ) - session.add(task_event) - session.commit() - return TaskEventModelSchema().dump(task_event) - - def update_task(workflow_id, task_id, body): workflow_model = session.query(WorkflowModel).filter_by(id=workflow_id).first() processor = WorkflowProcessor(workflow_model) @@ -131,39 +113,7 @@ def update_task(workflow_id, task_id, body): session.add(workflow_model) session.commit() - workflow_api_model =__get_workflow_api_model(processor) + workflow_api_model = __get_workflow_api_model(processor) update_workflow_stats(workflow_model, workflow_api_model) log_task_complete(workflow_model, task_id) return WorkflowApiSchema().dump(workflow_api_model) - - -def update_workflow_stats(workflow_model, workflow_api_model): - stats = session.query(WorkflowStatsModel)\ - .filter_by(study_id=workflow_model.study_id)\ - .filter_by(workflow_id=workflow_model.id)\ - .filter_by(workflow_spec_id=workflow_model.workflow_spec_id)\ - .filter_by(spec_version=workflow_model.spec_version)\ - .first() - - if stats is None: - stats = WorkflowStatsModel( - study_id=workflow_model.study_id, - workflow_id=workflow_model.id, - workflow_spec_id=workflow_model.workflow_spec_id, - spec_version=workflow_model.spec_version, - ) - - complete_states = ['CANCELLED', 'COMPLETED'] - incomplete_states = ['MAYBE', 'LIKELY', 'FUTURE', 'WAITING', 'READY'] - tasks = list(workflow_api_model.user_tasks) - stats.num_tasks_total = len(tasks) - stats.num_tasks_complete = sum(1 for t in tasks if t.state in complete_states) - stats.num_tasks_incomplete = sum(1 for t in tasks if t.state in incomplete_states) - stats.last_updated = datetime.now() - - session.add(stats) - session.commit() - return WorkflowStatsModelSchema().dump(stats) - - - diff --git a/tests/test_tasks_api.py b/tests/test_tasks_api.py index 9676a9a2..e100696c 100644 --- a/tests/test_tasks_api.py +++ b/tests/test_tasks_api.py @@ -4,10 +4,10 @@ import os from crc import session, app from crc.models.api_models import WorkflowApiSchema, Task from crc.models.file import FileModelSchema +from crc.models.stats import WorkflowStatsModel, TaskEventModel from crc.models.study import StudyModel from crc.models.workflow import WorkflowSpecModelSchema, WorkflowModel, WorkflowStatus from crc.services.workflow_processor import WorkflowProcessor -from crc.models.stats import WorkflowStatsModel, TaskEventModel from tests.base_test import BaseTest @@ -42,15 +42,15 @@ class TestTasksApi(BaseTest): self.assert_success(rv) json_data = json.loads(rv.get_data(as_text=True)) - num_stats = session.query(WorkflowStatsModel)\ - .filter_by(workflow_id=workflow.id)\ - .filter_by(workflow_spec_id=workflow.workflow_spec_id)\ + num_stats = session.query(WorkflowStatsModel) \ + .filter_by(workflow_id=workflow.id) \ + .filter_by(workflow_spec_id=workflow.workflow_spec_id) \ .count() self.assertGreater(num_stats, 0) - num_task_events = session.query(TaskEventModel)\ - .filter_by(workflow_id=workflow.id)\ - .filter_by(task_id=task.id)\ + num_task_events = session.query(TaskEventModel) \ + .filter_by(workflow_id=workflow.id) \ + .filter_by(task_id=task.id) \ .count() self.assertGreater(num_task_events, 0) @@ -240,3 +240,30 @@ class TestTasksApi(BaseTest): workflow_api = self.get_workflow_api(workflow, hard_reset=True) self.assertTrue(workflow_api.spec_version.startswith("v2 ")) self.assertTrue(workflow_api.is_latest_spec) + + def test_get_workflow_stats(self): + self.load_example_data() + workflow = self.create_workflow('exclusive_gateway') + + response_before = self.app.get('/v1.0/workflow/%i/stats' % workflow.id, + content_type="application/json", + headers=self.logged_in_headers()) + self.assert_success(response_before) + db_stats_before = session.query(WorkflowStatsModel).filter_by(workflow_id=workflow.id).first() + self.assertIsNone(db_stats_before) + + # Start the workflow. + tasks = self.get_workflow_api(workflow).user_tasks + self.complete_form(workflow, tasks[0], {"has_bananas": True}) + + response_after = self.app.get('/v1.0/workflow/%i/stats' % workflow.id, + content_type="application/json", + headers=self.logged_in_headers()) + self.assert_success(response_after) + db_stats_after = session.query(WorkflowStatsModel).filter_by(workflow_id=workflow.id).first() + self.assertIsNotNone(db_stats_after) + self.assertGreater(db_stats_after.num_tasks_complete, 0) + self.assertGreater(db_stats_after.num_tasks_incomplete, 0) + self.assertGreater(db_stats_after.num_tasks_total, 0) + self.assertEqual(db_stats_after.num_tasks_total, + db_stats_after.num_tasks_complete + db_stats_after.num_tasks_incomplete) From e977ee9e0de600ae3e1a73a54e7d269294dfc92f Mon Sep 17 00:00:00 2001 From: Aaron Louie Date: Wed, 11 Mar 2020 15:16:42 -0400 Subject: [PATCH 5/7] Updates task counts when getting workflow --- crc/api/workflow.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/crc/api/workflow.py b/crc/api/workflow.py index 26060eab..5f27d03c 100644 --- a/crc/api/workflow.py +++ b/crc/api/workflow.py @@ -1,6 +1,6 @@ import uuid -from api.stats import update_workflow_stats, log_task_complete +from crc.api.stats import update_workflow_stats, log_task_complete from crc import session from crc.api.common import ApiError, ApiErrorSchema from crc.api.file import delete_file @@ -67,7 +67,7 @@ def delete_workflow_specification(spec_id): def __get_workflow_api_model(processor: WorkflowProcessor): spiff_tasks = processor.get_all_user_tasks() - user_tasks = map(Task.from_spiff, spiff_tasks) + user_tasks = list(map(Task.from_spiff, spiff_tasks)) workflow_api = WorkflowApi( id=processor.get_workflow_id(), status=processor.get_status(), @@ -84,10 +84,12 @@ def __get_workflow_api_model(processor: WorkflowProcessor): def get_workflow(workflow_id, soft_reset=False, hard_reset=False): - schema = WorkflowApiSchema() workflow_model = session.query(WorkflowModel).filter_by(id=workflow_id).first() processor = WorkflowProcessor(workflow_model, soft_reset=soft_reset, hard_reset=hard_reset) - return schema.dump(__get_workflow_api_model(processor)) + + workflow_api_model = __get_workflow_api_model(processor) + update_workflow_stats(workflow_model, workflow_api_model) + return WorkflowApiSchema().dump(workflow_api_model) def delete(workflow_id): From 3262833102695bb8231fdf6eabaed32f84bff51c Mon Sep 17 00:00:00 2001 From: Aaron Louie Date: Wed, 11 Mar 2020 15:27:22 -0400 Subject: [PATCH 6/7] Requires login --- crc/api/stats.py | 5 ++++- crc/api/study.py | 1 - crc/api/workflow.py | 10 +++++++++- tests/test_study_api.py | 2 +- tests/test_tasks_api.py | 4 +++- tests/test_workflow_spec_api.py | 8 +++++--- 6 files changed, 22 insertions(+), 8 deletions(-) diff --git a/crc/api/stats.py b/crc/api/stats.py index 1c78939c..8bc63a15 100644 --- a/crc/api/stats.py +++ b/crc/api/stats.py @@ -2,16 +2,18 @@ from datetime import datetime from flask import g -from crc import session +from crc import session, auth from crc.models.stats import WorkflowStatsModel, WorkflowStatsModelSchema, TaskEventModel, TaskEventModelSchema +@auth.login_required def get_workflow_stats(workflow_id): schema = WorkflowStatsModelSchema() workflow_model = session.query(WorkflowStatsModel).filter_by(id=workflow_id).first() return schema.dump(workflow_model) +@auth.login_required def update_workflow_stats(workflow_model, workflow_api_model): stats = session.query(WorkflowStatsModel) \ .filter_by(study_id=workflow_model.study_id) \ @@ -41,6 +43,7 @@ def update_workflow_stats(workflow_model, workflow_api_model): return WorkflowStatsModelSchema().dump(stats) +@auth.login_required def log_task_complete(workflow_model, task_id): task_event = TaskEventModel( study_id=workflow_model.study_id, diff --git a/crc/api/study.py b/crc/api/study.py index bfc01dc8..9a77c36d 100644 --- a/crc/api/study.py +++ b/crc/api/study.py @@ -33,7 +33,6 @@ def add_study(body): return StudyModelSchema().dump(study) - @auth.login_required def update_study(study_id, body): if study_id is None: diff --git a/crc/api/workflow.py b/crc/api/workflow.py index 5f27d03c..49990bec 100644 --- a/crc/api/workflow.py +++ b/crc/api/workflow.py @@ -1,7 +1,7 @@ import uuid from crc.api.stats import update_workflow_stats, log_task_complete -from crc import session +from crc import session, auth from crc.api.common import ApiError, ApiErrorSchema from crc.api.file import delete_file from crc.models.api_models import Task, WorkflowApi, WorkflowApiSchema @@ -15,6 +15,7 @@ def all_specifications(): return schema.dump(session.query(WorkflowSpecModel).all()) +@auth.login_required def add_workflow_specification(body): new_spec = WorkflowSpecModelSchema().load(body, session=session) session.add(new_spec) @@ -22,6 +23,7 @@ def add_workflow_specification(body): return WorkflowSpecModelSchema().dump(new_spec) +@auth.login_required def get_workflow_specification(spec_id): if spec_id is None: error = ApiError('unknown_spec', 'Please provide a valid Workflow Specification ID.') @@ -36,6 +38,7 @@ def get_workflow_specification(spec_id): return WorkflowSpecModelSchema().dump(spec) +@auth.login_required def update_workflow_specification(spec_id, body): spec = WorkflowSpecModelSchema().load(body, session=session) spec.id = spec_id @@ -44,6 +47,7 @@ def update_workflow_specification(spec_id, body): return WorkflowSpecModelSchema().dump(spec) +@auth.login_required def delete_workflow_specification(spec_id): if spec_id is None: error = ApiError('unknown_spec', 'Please provide a valid Workflow Specification ID.') @@ -83,6 +87,7 @@ def __get_workflow_api_model(processor: WorkflowProcessor): return workflow_api +@auth.login_required def get_workflow(workflow_id, soft_reset=False, hard_reset=False): workflow_model = session.query(WorkflowModel).filter_by(id=workflow_id).first() processor = WorkflowProcessor(workflow_model, soft_reset=soft_reset, hard_reset=hard_reset) @@ -92,16 +97,19 @@ def get_workflow(workflow_id, soft_reset=False, hard_reset=False): return WorkflowApiSchema().dump(workflow_api_model) +@auth.login_required def delete(workflow_id): session.query(WorkflowModel).filter_by(id=workflow_id).delete() session.commit() +@auth.login_required def get_task(workflow_id, task_id): workflow = session.query(WorkflowModel).filter_by(id=workflow_id).first() return workflow.bpmn_workflow().get_task(task_id) +@auth.login_required def update_task(workflow_id, task_id, body): workflow_model = session.query(WorkflowModel).filter_by(id=workflow_id).first() processor = WorkflowProcessor(workflow_model) diff --git a/tests/test_study_api.py b/tests/test_study_api.py index 57a287f7..0b49f100 100644 --- a/tests/test_study_api.py +++ b/tests/test_study_api.py @@ -174,7 +174,7 @@ class TestStudyApi(BaseTest): self.assertEqual(1, session.query(WorkflowModel).count()) json_data = json.loads(rv.get_data(as_text=True)) workflow = WorkflowApiSchema().load(json_data) - rv = self.app.delete('/v1.0/workflow/%i' % workflow.id) + rv = self.app.delete('/v1.0/workflow/%i' % workflow.id, headers=self.logged_in_headers()) self.assert_success(rv) self.assertEqual(0, session.query(WorkflowModel).count()) diff --git a/tests/test_tasks_api.py b/tests/test_tasks_api.py index e100696c..af1a105c 100644 --- a/tests/test_tasks_api.py +++ b/tests/test_tasks_api.py @@ -29,6 +29,7 @@ class TestTasksApi(BaseTest): def get_workflow_api(self, workflow, soft_reset=False, hard_reset=False): rv = self.app.get('/v1.0/workflow/%i?soft_reset=%s&hard_reset=%s' % (workflow.id, str(soft_reset), str(hard_reset)), + headers=self.logged_in_headers(), content_type="application/json") json_data = json.loads(rv.get_data(as_text=True)) workflow_api = WorkflowApiSchema().load(json_data) @@ -37,6 +38,7 @@ class TestTasksApi(BaseTest): def complete_form(self, workflow, task, dict_data): rv = self.app.put('/v1.0/workflow/%i/task/%s/data' % (workflow.id, task.id), + headers=self.logged_in_headers(), content_type="application/json", data=json.dumps(dict_data)) self.assert_success(rv) @@ -158,7 +160,7 @@ class TestTasksApi(BaseTest): self.assertIsNotNone(workflow_api.next_task) self.assertEquals("EndEvent_0evb22x", workflow_api.next_task['name']) self.assertTrue(workflow_api.status == WorkflowStatus.complete) - rv = self.app.get('/v1.0/file?workflow_id=%i' % workflow.id) + rv = self.app.get('/v1.0/file?workflow_id=%i' % workflow.id, headers=self.logged_in_headers()) self.assert_success(rv) json_data = json.loads(rv.get_data(as_text=True)) files = FileModelSchema(many=True).load(json_data, session=session) diff --git a/tests/test_workflow_spec_api.py b/tests/test_workflow_spec_api.py index 85c539bf..08faf16f 100644 --- a/tests/test_workflow_spec_api.py +++ b/tests/test_workflow_spec_api.py @@ -27,7 +27,9 @@ class TestWorkflowSpec(BaseTest): num_before = session.query(WorkflowSpecModel).count() spec = WorkflowSpecModel(id='make_cookies', display_name='Cooooookies', description='Om nom nom delicious cookies') - rv = self.app.post('/v1.0/workflow-specification', content_type="application/json", + rv = self.app.post('/v1.0/workflow-specification', + headers=self.logged_in_headers(), + content_type="application/json", data=json.dumps(WorkflowSpecModelSchema().dump(spec))) self.assert_success(rv) db_spec = session.query(WorkflowSpecModel).filter_by(id='make_cookies').first() @@ -38,7 +40,7 @@ class TestWorkflowSpec(BaseTest): def test_get_workflow_specification(self): self.load_example_data() db_spec = session.query(WorkflowSpecModel).first() - rv = self.app.get('/v1.0/workflow-specification/%s' % db_spec.id) + rv = self.app.get('/v1.0/workflow-specification/%s' % db_spec.id, headers=self.logged_in_headers()) self.assert_success(rv) json_data = json.loads(rv.get_data(as_text=True)) api_spec = WorkflowSpecModelSchema().load(json_data, session=session) @@ -55,7 +57,7 @@ class TestWorkflowSpec(BaseTest): num_workflows_before = session.query(WorkflowModel).filter_by(workflow_spec_id=spec_id).count() self.assertGreater(num_files_before + num_workflows_before, 0) - rv = self.app.delete('/v1.0/workflow-specification/' + spec_id) + rv = self.app.delete('/v1.0/workflow-specification/' + spec_id, headers=self.logged_in_headers()) self.assert_success(rv) num_specs_after = session.query(WorkflowSpecModel).filter_by(id=spec_id).count() From f177e33ad6288f5f926fd4c03656ca00c497e29f Mon Sep 17 00:00:00 2001 From: Aaron Louie Date: Wed, 11 Mar 2020 15:31:23 -0400 Subject: [PATCH 7/7] Tiny tweak --- crc/api/stats.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/crc/api/stats.py b/crc/api/stats.py index 8bc63a15..66d88455 100644 --- a/crc/api/stats.py +++ b/crc/api/stats.py @@ -8,9 +8,8 @@ from crc.models.stats import WorkflowStatsModel, WorkflowStatsModelSchema, TaskE @auth.login_required def get_workflow_stats(workflow_id): - schema = WorkflowStatsModelSchema() - workflow_model = session.query(WorkflowStatsModel).filter_by(id=workflow_id).first() - return schema.dump(workflow_model) + workflow_model = session.query(WorkflowStatsModel).filter_by(workflow_id=workflow_id).first() + return WorkflowStatsModelSchema().dump(workflow_model) @auth.login_required