Merge remote-tracking branch 'origin/master' into feature/delete_study

crc/api.yml

@@ -500,6 +500,27 @@ paths:
       responses:
         '204':
           description: The workflow was removed
+  /workflow/{workflow_id}/stats:
+    parameters:
+        - name: workflow_id
+          in: path
+          required: true
+          description: The id of the workflow
+          schema:
+            type: integer
+            format: int32
+    get:
+      operationId: crc.api.stats.get_workflow_stats
+      summary: Stats about a given workflow
+      tags:
+        - Workflows and Tasks
+      responses:
+        '200':
+          description: Returns counts of complete, incomplete, and total number of tasks
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/Workflow"
   # /v1.0/workflow/0/task/0
   /workflow/{workflow_id}/task/{task_id}:
     parameters:
@@ -749,6 +770,12 @@ components:
           type: string
         is_latest_spec:
           type: boolean
+        num_tasks_total:
+          type: integer
+        num_tasks_complete:
+          type: integer
+        num_tasks_incomplete:
+          type: integer
 
       example:
         id: 291234

crc/api/stats.py

@@ -0,0 +1,59 @@
+from datetime import datetime
+
+from flask import g
+
+from crc import session, auth
+from crc.models.stats import WorkflowStatsModel, WorkflowStatsModelSchema, TaskEventModel, TaskEventModelSchema
+
+
+@auth.login_required
+def get_workflow_stats(workflow_id):
+    workflow_model = session.query(WorkflowStatsModel).filter_by(workflow_id=workflow_id).first()
+    return WorkflowStatsModelSchema().dump(workflow_model)
+
+
+@auth.login_required
+def update_workflow_stats(workflow_model, workflow_api_model):
+    stats = session.query(WorkflowStatsModel) \
+        .filter_by(study_id=workflow_model.study_id) \
+        .filter_by(workflow_id=workflow_model.id) \
+        .filter_by(workflow_spec_id=workflow_model.workflow_spec_id) \
+        .filter_by(spec_version=workflow_model.spec_version) \
+        .first()
+
+    if stats is None:
+        stats = WorkflowStatsModel(
+            study_id=workflow_model.study_id,
+            workflow_id=workflow_model.id,
+            workflow_spec_id=workflow_model.workflow_spec_id,
+            spec_version=workflow_model.spec_version,
+        )
+
+    complete_states = ['CANCELLED', 'COMPLETED']
+    incomplete_states = ['MAYBE', 'LIKELY', 'FUTURE', 'WAITING', 'READY']
+    tasks = list(workflow_api_model.user_tasks)
+    stats.num_tasks_total = len(tasks)
+    stats.num_tasks_complete = sum(1 for t in tasks if t.state in complete_states)
+    stats.num_tasks_incomplete = sum(1 for t in tasks if t.state in incomplete_states)
+    stats.last_updated = datetime.now()
+
+    session.add(stats)
+    session.commit()
+    return WorkflowStatsModelSchema().dump(stats)
+
+
+@auth.login_required
+def log_task_complete(workflow_model, task_id):
+    task_event = TaskEventModel(
+        study_id=workflow_model.study_id,
+        user_uid=g.user.uid,
+        workflow_id=workflow_model.id,
+        workflow_spec_id=workflow_model.workflow_spec_id,
+        spec_version=workflow_model.spec_version,
+        task_id=task_id,
+        task_state='COMPLETE',
+        date=datetime.now(),
+    )
+    session.add(task_event)
+    session.commit()
+    return TaskEventModelSchema().dump(task_event)

crc/api/study.py

@@ -33,7 +33,6 @@ def add_study(body):
     return StudyModelSchema().dump(study)
 
 
-
 @auth.login_required
 def update_study(study_id, body):
     if study_id is None:

crc/api/workflow.py

@@ -1,12 +1,13 @@
 import uuid
 
-from crc.api.file import delete_file
+from crc.api.stats import update_workflow_stats, log_task_complete
-from crc import session
+from crc import session, auth
 from crc.api.common import ApiError, ApiErrorSchema
+from crc.api.file import delete_file
 from crc.models.api_models import Task, WorkflowApi, WorkflowApiSchema
+from crc.models.file import FileModel
 from crc.models.workflow import WorkflowModel, WorkflowSpecModelSchema, WorkflowSpecModel
 from crc.services.workflow_processor import WorkflowProcessor
-from crc.models.file import FileModel
 
 
 def all_specifications():
@@ -14,6 +15,7 @@ def all_specifications():
     return schema.dump(session.query(WorkflowSpecModel).all())
 
 
+@auth.login_required
 def add_workflow_specification(body):
     new_spec = WorkflowSpecModelSchema().load(body, session=session)
     session.add(new_spec)
@@ -21,6 +23,7 @@ def add_workflow_specification(body):
     return WorkflowSpecModelSchema().dump(new_spec)
 
 
+@auth.login_required
 def get_workflow_specification(spec_id):
     if spec_id is None:
         error = ApiError('unknown_spec', 'Please provide a valid Workflow Specification ID.')
@@ -35,6 +38,7 @@ def get_workflow_specification(spec_id):
     return WorkflowSpecModelSchema().dump(spec)
 
 
+@auth.login_required
 def update_workflow_specification(spec_id, body):
     spec = WorkflowSpecModelSchema().load(body, session=session)
     spec.id = spec_id
@@ -43,6 +47,7 @@ def update_workflow_specification(spec_id, body):
     return WorkflowSpecModelSchema().dump(spec)
 
 
+@auth.login_required
 def delete_workflow_specification(spec_id):
     if spec_id is None:
         error = ApiError('unknown_spec', 'Please provide a valid Workflow Specification ID.')
@@ -61,13 +66,12 @@ def delete_workflow_specification(spec_id):
 
     session.query(WorkflowModel).filter_by(workflow_spec_id=spec_id).delete()
     session.query(WorkflowSpecModel).filter_by(id=spec_id).delete()
-
     session.commit()
 
 
 def __get_workflow_api_model(processor: WorkflowProcessor):
     spiff_tasks = processor.get_all_user_tasks()
-    user_tasks = map(Task.from_spiff, spiff_tasks)
+    user_tasks = list(map(Task.from_spiff, spiff_tasks))
     workflow_api = WorkflowApi(
         id=processor.get_workflow_id(),
         status=processor.get_status(),
@@ -83,22 +87,29 @@ def __get_workflow_api_model(processor: WorkflowProcessor):
     return workflow_api
 
 
+@auth.login_required
 def get_workflow(workflow_id, soft_reset=False, hard_reset=False):
-    schema = WorkflowApiSchema()
     workflow_model = session.query(WorkflowModel).filter_by(id=workflow_id).first()
     processor = WorkflowProcessor(workflow_model, soft_reset=soft_reset, hard_reset=hard_reset)
-    return schema.dump(__get_workflow_api_model(processor))
+
+    workflow_api_model = __get_workflow_api_model(processor)
+    update_workflow_stats(workflow_model, workflow_api_model)
+    return WorkflowApiSchema().dump(workflow_api_model)
 
 
+@auth.login_required
 def delete(workflow_id):
     session.query(WorkflowModel).filter_by(id=workflow_id).delete()
     session.commit()
 
+
+@auth.login_required
 def get_task(workflow_id, task_id):
     workflow = session.query(WorkflowModel).filter_by(id=workflow_id).first()
     return workflow.bpmn_workflow().get_task(task_id)
 
 
+@auth.login_required
 def update_task(workflow_id, task_id, body):
     workflow_model = session.query(WorkflowModel).filter_by(id=workflow_id).first()
     processor = WorkflowProcessor(workflow_model)
@@ -111,5 +122,8 @@ def update_task(workflow_id, task_id, body):
     workflow_model.bpmn_workflow_json = processor.serialize()
     session.add(workflow_model)
     session.commit()
-    return WorkflowApiSchema().dump(__get_workflow_api_model(processor)
+
-                                    )
+    workflow_api_model = __get_workflow_api_model(processor)
+    update_workflow_stats(workflow_model, workflow_api_model)
+    log_task_complete(workflow_model, task_id)
+    return WorkflowApiSchema().dump(workflow_api_model)

crc/models/api_models.py

@@ -108,10 +108,9 @@ class WorkflowApi(object):
 class WorkflowApiSchema(ma.Schema):
     class Meta:
         model = WorkflowApi
-        fields = ["id", "status",
+        fields = ["id", "status", "user_tasks", "last_task", "next_task",
-                  "user_tasks", "last_task", "next_task",
+                  "workflow_spec_id", "spec_version", "is_latest_spec",
-                  "workflow_spec_id",
+                  "num_tasks_total", "num_tasks_complete", "num_tasks_incomplete"]
-                  "spec_version", "is_latest_spec"]
         unknown = INCLUDE
 
     status = EnumField(WorkflowStatus)
@@ -121,4 +120,15 @@ class WorkflowApiSchema(ma.Schema):
 
     @marshmallow.post_load
     def make_workflow(self, data, **kwargs):
-        return WorkflowApi(**data)
+        keys = [
+            'id',
+            'status',
+            'user_tasks',
+            'last_task',
+            'next_task',
+            'workflow_spec_id',
+            'spec_version',
+            'is_latest_spec'
+        ]
+        filtered_fields = {key: data[key] for key in keys}
+        return WorkflowApi(**filtered_fields)

crc/models/stats.py

@@ -0,0 +1,41 @@
+from marshmallow_sqlalchemy import ModelSchema
+
+from crc import db
+
+
+class WorkflowStatsModel(db.Model):
+    __tablename__ = 'workflow_stats'
+    id = db.Column(db.Integer, primary_key=True)
+    study_id = db.Column(db.Integer, db.ForeignKey('study.id'), nullable=False)
+    workflow_id = db.Column(db.Integer, db.ForeignKey('workflow.id'), nullable=False)
+    workflow_spec_id = db.Column(db.String, db.ForeignKey('workflow_spec.id'))
+    spec_version = db.Column(db.String)
+    num_tasks_total = db.Column(db.Integer)
+    num_tasks_complete = db.Column(db.Integer)
+    num_tasks_incomplete = db.Column(db.Integer)
+    last_updated = db.Column(db.DateTime)
+
+
+class WorkflowStatsModelSchema(ModelSchema):
+    class Meta:
+        model = WorkflowStatsModel
+        include_fk = True  # Includes foreign keys
+
+
+class TaskEventModel(db.Model):
+    __tablename__ = 'task_event'
+    id = db.Column(db.Integer, primary_key=True)
+    study_id = db.Column(db.Integer, db.ForeignKey('study.id'), nullable=False)
+    user_uid = db.Column(db.String, db.ForeignKey('user.uid'), nullable=False)
+    workflow_id = db.Column(db.Integer, db.ForeignKey('workflow.id'), nullable=False)
+    workflow_spec_id = db.Column(db.String, db.ForeignKey('workflow_spec.id'))
+    spec_version = db.Column(db.String)
+    task_id = db.Column(db.String)
+    task_state = db.Column(db.String)
+    date = db.Column(db.DateTime)
+
+
+class TaskEventModelSchema(ModelSchema):
+    class Meta:
+        model = TaskEventModel
+        include_fk = True  # Includes foreign keys

crc/models/study.py

@@ -28,5 +28,3 @@ class StudyModelSchema(ModelSchema):
         include_fk = True  # Includes foreign keys
 
     protocol_builder_status = EnumField(ProtocolBuilderStatus)
-
-

migrations/versions/90dd63672e0a_.py

@@ -0,0 +1,59 @@
+"""empty message
+
+Revision ID: 90dd63672e0a
+Revises: 8856126b6658
+Create Date: 2020-03-10 21:16:38.827156
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '90dd63672e0a'
+down_revision = '8856126b6658'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('task_event',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('study_id', sa.Integer(), nullable=False),
+    sa.Column('user_uid', sa.String(), nullable=False),
+    sa.Column('workflow_id', sa.Integer(), nullable=False),
+    sa.Column('workflow_spec_id', sa.String(), nullable=True),
+    sa.Column('spec_version', sa.String(), nullable=True),
+    sa.Column('task_id', sa.String(), nullable=True),
+    sa.Column('task_state', sa.String(), nullable=True),
+    sa.Column('date', sa.DateTime(), nullable=True),
+    sa.ForeignKeyConstraint(['study_id'], ['study.id'], ),
+    sa.ForeignKeyConstraint(['user_uid'], ['user.uid'], ),
+    sa.ForeignKeyConstraint(['workflow_id'], ['workflow.id'], ),
+    sa.ForeignKeyConstraint(['workflow_spec_id'], ['workflow_spec.id'], ),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_table('workflow_stats',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('study_id', sa.Integer(), nullable=False),
+    sa.Column('workflow_id', sa.Integer(), nullable=False),
+    sa.Column('workflow_spec_id', sa.String(), nullable=True),
+    sa.Column('spec_version', sa.String(), nullable=True),
+    sa.Column('num_tasks_total', sa.Integer(), nullable=True),
+    sa.Column('num_tasks_complete', sa.Integer(), nullable=True),
+    sa.Column('num_tasks_incomplete', sa.Integer(), nullable=True),
+    sa.Column('last_updated', sa.DateTime(), nullable=True),
+    sa.ForeignKeyConstraint(['study_id'], ['study.id'], ),
+    sa.ForeignKeyConstraint(['workflow_id'], ['workflow.id'], ),
+    sa.ForeignKeyConstraint(['workflow_spec_id'], ['workflow_spec.id'], ),
+    sa.PrimaryKeyConstraint('id')
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('workflow_stats')
+    op.drop_table('task_event')
+    # ### end Alembic commands ###

tests/test_study_api.py

@@ -174,7 +174,7 @@ class TestStudyApi(BaseTest):
         self.assertEqual(1, session.query(WorkflowModel).count())
         json_data = json.loads(rv.get_data(as_text=True))
         workflow = WorkflowApiSchema().load(json_data)
-        rv = self.app.delete('/v1.0/workflow/%i' % workflow.id)
+        rv = self.app.delete('/v1.0/workflow/%i' % workflow.id, headers=self.logged_in_headers())
         self.assert_success(rv)
         self.assertEqual(0, session.query(WorkflowModel).count())
 

tests/test_tasks_api.py

@@ -4,6 +4,7 @@ import os
 from crc import session, app
 from crc.models.api_models import WorkflowApiSchema, Task
 from crc.models.file import FileModelSchema
+from crc.models.stats import WorkflowStatsModel, TaskEventModel
 from crc.models.study import StudyModel
 from crc.models.workflow import WorkflowSpecModelSchema, WorkflowModel, WorkflowStatus
 from crc.services.workflow_processor import WorkflowProcessor
@@ -28,6 +29,7 @@ class TestTasksApi(BaseTest):
     def get_workflow_api(self, workflow, soft_reset=False, hard_reset=False):
         rv = self.app.get('/v1.0/workflow/%i?soft_reset=%s&hard_reset=%s' %
                           (workflow.id, str(soft_reset), str(hard_reset)),
+                          headers=self.logged_in_headers(),
                           content_type="application/json")
         self.assert_success(rv)
         json_data = json.loads(rv.get_data(as_text=True))
@@ -37,10 +39,24 @@ class TestTasksApi(BaseTest):
 
     def complete_form(self, workflow, task, dict_data):
         rv = self.app.put('/v1.0/workflow/%i/task/%s/data' % (workflow.id, task.id),
+                          headers=self.logged_in_headers(),
                           content_type="application/json",
                           data=json.dumps(dict_data))
         self.assert_success(rv)
         json_data = json.loads(rv.get_data(as_text=True))
+
+        num_stats = session.query(WorkflowStatsModel) \
+            .filter_by(workflow_id=workflow.id) \
+            .filter_by(workflow_spec_id=workflow.workflow_spec_id) \
+            .count()
+        self.assertGreater(num_stats, 0)
+
+        num_task_events = session.query(TaskEventModel) \
+            .filter_by(workflow_id=workflow.id) \
+            .filter_by(task_id=task.id) \
+            .count()
+        self.assertGreater(num_task_events, 0)
+
         workflow = WorkflowApiSchema().load(json_data)
         return workflow
 
@@ -145,7 +161,7 @@ class TestTasksApi(BaseTest):
         self.assertIsNotNone(workflow_api.next_task)
         self.assertEquals("EndEvent_0evb22x", workflow_api.next_task['name'])
         self.assertTrue(workflow_api.status == WorkflowStatus.complete)
-        rv = self.app.get('/v1.0/file?workflow_id=%i' % workflow.id)
+        rv = self.app.get('/v1.0/file?workflow_id=%i' % workflow.id, headers=self.logged_in_headers())
         self.assert_success(rv)
         json_data = json.loads(rv.get_data(as_text=True))
         files = FileModelSchema(many=True).load(json_data, session=session)
@@ -252,3 +268,30 @@ class TestTasksApi(BaseTest):
         workflow_api = self.get_workflow_api(workflow)
         self.assertTrue(workflow_api.spec_version.startswith("v1 "))
         self.assertFalse(workflow_api.is_latest_spec)
+
+    def test_get_workflow_stats(self):
+        self.load_example_data()
+        workflow = self.create_workflow('exclusive_gateway')
+
+        response_before = self.app.get('/v1.0/workflow/%i/stats' % workflow.id,
+                                       content_type="application/json",
+                                       headers=self.logged_in_headers())
+        self.assert_success(response_before)
+        db_stats_before = session.query(WorkflowStatsModel).filter_by(workflow_id=workflow.id).first()
+        self.assertIsNone(db_stats_before)
+
+        # Start the workflow.
+        tasks = self.get_workflow_api(workflow).user_tasks
+        self.complete_form(workflow, tasks[0], {"has_bananas": True})
+
+        response_after = self.app.get('/v1.0/workflow/%i/stats' % workflow.id,
+                                      content_type="application/json",
+                                      headers=self.logged_in_headers())
+        self.assert_success(response_after)
+        db_stats_after = session.query(WorkflowStatsModel).filter_by(workflow_id=workflow.id).first()
+        self.assertIsNotNone(db_stats_after)
+        self.assertGreater(db_stats_after.num_tasks_complete, 0)
+        self.assertGreater(db_stats_after.num_tasks_incomplete, 0)
+        self.assertGreater(db_stats_after.num_tasks_total, 0)
+        self.assertEqual(db_stats_after.num_tasks_total,
+                         db_stats_after.num_tasks_complete + db_stats_after.num_tasks_incomplete)

tests/test_workflow_spec_api.py

@@ -27,7 +27,9 @@ class TestWorkflowSpec(BaseTest):
         num_before = session.query(WorkflowSpecModel).count()
         spec = WorkflowSpecModel(id='make_cookies', display_name='Cooooookies',
                                  description='Om nom nom delicious cookies')
-        rv = self.app.post('/v1.0/workflow-specification', content_type="application/json",
+        rv = self.app.post('/v1.0/workflow-specification',
+                           headers=self.logged_in_headers(),
+                           content_type="application/json",
                            data=json.dumps(WorkflowSpecModelSchema().dump(spec)))
         self.assert_success(rv)
         db_spec = session.query(WorkflowSpecModel).filter_by(id='make_cookies').first()
@@ -38,7 +40,7 @@ class TestWorkflowSpec(BaseTest):
     def test_get_workflow_specification(self):
         self.load_example_data()
         db_spec = session.query(WorkflowSpecModel).first()
-        rv = self.app.get('/v1.0/workflow-specification/%s' % db_spec.id)
+        rv = self.app.get('/v1.0/workflow-specification/%s' % db_spec.id, headers=self.logged_in_headers())
         self.assert_success(rv)
         json_data = json.loads(rv.get_data(as_text=True))
         api_spec = WorkflowSpecModelSchema().load(json_data, session=session)
@@ -55,7 +57,7 @@ class TestWorkflowSpec(BaseTest):
         num_workflows_before = session.query(WorkflowModel).filter_by(workflow_spec_id=spec_id).count()
         self.assertGreater(num_files_before + num_workflows_before, 0)
 
-        rv = self.app.delete('/v1.0/workflow-specification/' + spec_id)
+        rv = self.app.delete('/v1.0/workflow-specification/' + spec_id, headers=self.logged_in_headers())
         self.assert_success(rv)
 
         num_specs_after = session.query(WorkflowSpecModel).filter_by(id=spec_id).count()