sartography
/
cr-connect-workflow
mirror of https://github.com/sartography/cr-connect-workflow.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into feature/delete_study

This commit is contained in:
Aaron Louie 2020-03-16 08:49:44 -04:00
parent cd6a70b747 bb636d363c
commit 3885bc7624
11 changed files with 274 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
crc/api.yml
View File

@ -500,6 +500,27 @@ paths:
responses:
'204':
description: The workflow was removed
/workflow/{workflow_id}/stats:
parameters:
- name: workflow_id
in: path
required: true
description: The id of the workflow
schema:
type: integer
format: int32
get:
operationId: crc.api.stats.get_workflow_stats
summary: Stats about a given workflow
tags:
- Workflows and Tasks
responses:
'200':
description: Returns counts of complete, incomplete, and total number of tasks
content:
application/json:
schema:
$ref: "#/components/schemas/Workflow"
# /v1.0/workflow/0/task/0
/workflow/{workflow_id}/task/{task_id}:
parameters:
@ -749,6 +770,12 @@ components:
type: string
is_latest_spec:
type: boolean
num_tasks_total:
type: integer
num_tasks_complete:
type: integer
num_tasks_incomplete:
type: integer
example:
id: 291234

59
crc/api/stats.py Normal file
View File

@ -0,0 +1,59 @@
from datetime import datetime
from flask import g
from crc import session, auth
from crc.models.stats import WorkflowStatsModel, WorkflowStatsModelSchema, TaskEventModel, TaskEventModelSchema
@auth.login_required
def get_workflow_stats(workflow_id):
workflow_model = session.query(WorkflowStatsModel).filter_by(workflow_id=workflow_id).first()
return WorkflowStatsModelSchema().dump(workflow_model)
@auth.login_required
def update_workflow_stats(workflow_model, workflow_api_model):
stats = session.query(WorkflowStatsModel) \
.filter_by(study_id=workflow_model.study_id) \
.filter_by(workflow_id=workflow_model.id) \
.filter_by(workflow_spec_id=workflow_model.workflow_spec_id) \
.filter_by(spec_version=workflow_model.spec_version) \
.first()
if stats is None:
stats = WorkflowStatsModel(
study_id=workflow_model.study_id,
workflow_id=workflow_model.id,
workflow_spec_id=workflow_model.workflow_spec_id,
spec_version=workflow_model.spec_version,
)
complete_states = ['CANCELLED', 'COMPLETED']
incomplete_states = ['MAYBE', 'LIKELY', 'FUTURE', 'WAITING', 'READY']
tasks = list(workflow_api_model.user_tasks)
stats.num_tasks_total = len(tasks)
stats.num_tasks_complete = sum(1 for t in tasks if t.state in complete_states)
stats.num_tasks_incomplete = sum(1 for t in tasks if t.state in incomplete_states)
stats.last_updated = datetime.now()
session.add(stats)
session.commit()
return WorkflowStatsModelSchema().dump(stats)
@auth.login_required
def log_task_complete(workflow_model, task_id):
task_event = TaskEventModel(
study_id=workflow_model.study_id,
user_uid=g.user.uid,
workflow_id=workflow_model.id,
workflow_spec_id=workflow_model.workflow_spec_id,
spec_version=workflow_model.spec_version,
task_id=task_id,
task_state='COMPLETE',
date=datetime.now(),
)
session.add(task_event)
session.commit()
return TaskEventModelSchema().dump(task_event)

1
crc/api/study.py
View File

@ -33,7 +33,6 @@ def add_study(body):
return StudyModelSchema().dump(study)
@auth.login_required
def update_study(study_id, body):
if study_id is None:

32
crc/api/workflow.py
View File

@ -1,12 +1,13 @@
import uuid
from crc.api.file import delete_file
from crc import session
from crc.api.stats import update_workflow_stats, log_task_complete
from crc import session, auth
from crc.api.common import ApiError, ApiErrorSchema
from crc.api.file import delete_file
from crc.models.api_models import Task, WorkflowApi, WorkflowApiSchema
from crc.models.file import FileModel
from crc.models.workflow import WorkflowModel, WorkflowSpecModelSchema, WorkflowSpecModel
from crc.services.workflow_processor import WorkflowProcessor
from crc.models.file import FileModel
def all_specifications():
@ -14,6 +15,7 @@ def all_specifications():
return schema.dump(session.query(WorkflowSpecModel).all())
@auth.login_required
def add_workflow_specification(body):
new_spec = WorkflowSpecModelSchema().load(body, session=session)
session.add(new_spec)
@ -21,6 +23,7 @@ def add_workflow_specification(body):
return WorkflowSpecModelSchema().dump(new_spec)
@auth.login_required
def get_workflow_specification(spec_id):
if spec_id is None:
error = ApiError('unknown_spec', 'Please provide a valid Workflow Specification ID.')
@ -35,6 +38,7 @@ def get_workflow_specification(spec_id):
return WorkflowSpecModelSchema().dump(spec)
@auth.login_required
def update_workflow_specification(spec_id, body):
spec = WorkflowSpecModelSchema().load(body, session=session)
spec.id = spec_id
@ -43,6 +47,7 @@ def update_workflow_specification(spec_id, body):
return WorkflowSpecModelSchema().dump(spec)
@auth.login_required
def delete_workflow_specification(spec_id):
if spec_id is None:
error = ApiError('unknown_spec', 'Please provide a valid Workflow Specification ID.')
@ -61,13 +66,12 @@ def delete_workflow_specification(spec_id):
session.query(WorkflowModel).filter_by(workflow_spec_id=spec_id).delete()
session.query(WorkflowSpecModel).filter_by(id=spec_id).delete()
session.commit()
def __get_workflow_api_model(processor: WorkflowProcessor):
spiff_tasks = processor.get_all_user_tasks()
user_tasks = map(Task.from_spiff, spiff_tasks)
user_tasks = list(map(Task.from_spiff, spiff_tasks))
workflow_api = WorkflowApi(
id=processor.get_workflow_id(),
status=processor.get_status(),
@ -83,22 +87,29 @@ def __get_workflow_api_model(processor: WorkflowProcessor):
return workflow_api
@auth.login_required
def get_workflow(workflow_id, soft_reset=False, hard_reset=False):
schema = WorkflowApiSchema()
workflow_model = session.query(WorkflowModel).filter_by(id=workflow_id).first()
processor = WorkflowProcessor(workflow_model, soft_reset=soft_reset, hard_reset=hard_reset)
return schema.dump(__get_workflow_api_model(processor))
workflow_api_model = __get_workflow_api_model(processor)
update_workflow_stats(workflow_model, workflow_api_model)
return WorkflowApiSchema().dump(workflow_api_model)
@auth.login_required
def delete(workflow_id):
session.query(WorkflowModel).filter_by(id=workflow_id).delete()
session.commit()
@auth.login_required
def get_task(workflow_id, task_id):
workflow = session.query(WorkflowModel).filter_by(id=workflow_id).first()
return workflow.bpmn_workflow().get_task(task_id)
@auth.login_required
def update_task(workflow_id, task_id, body):
workflow_model = session.query(WorkflowModel).filter_by(id=workflow_id).first()
processor = WorkflowProcessor(workflow_model)
@ -111,5 +122,8 @@ def update_task(workflow_id, task_id, body):
workflow_model.bpmn_workflow_json = processor.serialize()
session.add(workflow_model)
session.commit()
return WorkflowApiSchema().dump(__get_workflow_api_model(processor)
)
workflow_api_model = __get_workflow_api_model(processor)
update_workflow_stats(workflow_model, workflow_api_model)
log_task_complete(workflow_model, task_id)
return WorkflowApiSchema().dump(workflow_api_model)

20
crc/models/api_models.py
View File

@ -108,10 +108,9 @@ class WorkflowApi(object):
class WorkflowApiSchema(ma.Schema):
class Meta:
model = WorkflowApi
fields = ["id", "status",
"user_tasks", "last_task", "next_task",
"workflow_spec_id",
"spec_version", "is_latest_spec"]
fields = ["id", "status", "user_tasks", "last_task", "next_task",
"workflow_spec_id", "spec_version", "is_latest_spec",
"num_tasks_total", "num_tasks_complete", "num_tasks_incomplete"]
unknown = INCLUDE
status = EnumField(WorkflowStatus)
@ -121,4 +120,15 @@ class WorkflowApiSchema(ma.Schema):
@marshmallow.post_load
def make_workflow(self, data, **kwargs):
return WorkflowApi(**data)
keys = [
'id',
'status',
'user_tasks',
'last_task',
'next_task',
'workflow_spec_id',
'spec_version',
'is_latest_spec'
]
filtered_fields = {key: data[key] for key in keys}
return WorkflowApi(**filtered_fields)

41
crc/models/stats.py Normal file
View File

@ -0,0 +1,41 @@
from marshmallow_sqlalchemy import ModelSchema
from crc import db
class WorkflowStatsModel(db.Model):
__tablename__ = 'workflow_stats'
id = db.Column(db.Integer, primary_key=True)
study_id = db.Column(db.Integer, db.ForeignKey('study.id'), nullable=False)
workflow_id = db.Column(db.Integer, db.ForeignKey('workflow.id'), nullable=False)
workflow_spec_id = db.Column(db.String, db.ForeignKey('workflow_spec.id'))
spec_version = db.Column(db.String)
num_tasks_total = db.Column(db.Integer)
num_tasks_complete = db.Column(db.Integer)
num_tasks_incomplete = db.Column(db.Integer)
last_updated = db.Column(db.DateTime)
class WorkflowStatsModelSchema(ModelSchema):
class Meta:
model = WorkflowStatsModel
include_fk = True # Includes foreign keys
class TaskEventModel(db.Model):
__tablename__ = 'task_event'
id = db.Column(db.Integer, primary_key=True)
study_id = db.Column(db.Integer, db.ForeignKey('study.id'), nullable=False)
user_uid = db.Column(db.String, db.ForeignKey('user.uid'), nullable=False)
workflow_id = db.Column(db.Integer, db.ForeignKey('workflow.id'), nullable=False)
workflow_spec_id = db.Column(db.String, db.ForeignKey('workflow_spec.id'))
spec_version = db.Column(db.String)
task_id = db.Column(db.String)
task_state = db.Column(db.String)
date = db.Column(db.DateTime)
class TaskEventModelSchema(ModelSchema):
class Meta:
model = TaskEventModel
include_fk = True # Includes foreign keys

2
crc/models/study.py
View File

@ -28,5 +28,3 @@ class StudyModelSchema(ModelSchema):
include_fk = True # Includes foreign keys
protocol_builder_status = EnumField(ProtocolBuilderStatus)

59
migrations/versions/90dd63672e0a_.py Normal file
View File

@ -0,0 +1,59 @@
"""empty message
Revision ID: 90dd63672e0a
Revises: 8856126b6658
Create Date: 2020-03-10 21:16:38.827156
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '90dd63672e0a'
down_revision = '8856126b6658'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('task_event',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('study_id', sa.Integer(), nullable=False),
sa.Column('user_uid', sa.String(), nullable=False),
sa.Column('workflow_id', sa.Integer(), nullable=False),
sa.Column('workflow_spec_id', sa.String(), nullable=True),
sa.Column('spec_version', sa.String(), nullable=True),
sa.Column('task_id', sa.String(), nullable=True),
sa.Column('task_state', sa.String(), nullable=True),
sa.Column('date', sa.DateTime(), nullable=True),
sa.ForeignKeyConstraint(['study_id'], ['study.id'], ),
sa.ForeignKeyConstraint(['user_uid'], ['user.uid'], ),
sa.ForeignKeyConstraint(['workflow_id'], ['workflow.id'], ),
sa.ForeignKeyConstraint(['workflow_spec_id'], ['workflow_spec.id'], ),
sa.PrimaryKeyConstraint('id')
)
op.create_table('workflow_stats',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('study_id', sa.Integer(), nullable=False),
sa.Column('workflow_id', sa.Integer(), nullable=False),
sa.Column('workflow_spec_id', sa.String(), nullable=True),
sa.Column('spec_version', sa.String(), nullable=True),
sa.Column('num_tasks_total', sa.Integer(), nullable=True),
sa.Column('num_tasks_complete', sa.Integer(), nullable=True),
sa.Column('num_tasks_incomplete', sa.Integer(), nullable=True),
sa.Column('last_updated', sa.DateTime(), nullable=True),
sa.ForeignKeyConstraint(['study_id'], ['study.id'], ),
sa.ForeignKeyConstraint(['workflow_id'], ['workflow.id'], ),
sa.ForeignKeyConstraint(['workflow_spec_id'], ['workflow_spec.id'], ),
sa.PrimaryKeyConstraint('id')
)
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table('workflow_stats')
op.drop_table('task_event')
# ### end Alembic commands ###

2
tests/test_study_api.py
View File

@ -174,7 +174,7 @@ class TestStudyApi(BaseTest):
self.assertEqual(1, session.query(WorkflowModel).count())
json_data = json.loads(rv.get_data(as_text=True))
workflow = WorkflowApiSchema().load(json_data)
rv = self.app.delete('/v1.0/workflow/%i' % workflow.id)
rv = self.app.delete('/v1.0/workflow/%i' % workflow.id, headers=self.logged_in_headers())
self.assert_success(rv)
self.assertEqual(0, session.query(WorkflowModel).count())

45
tests/test_tasks_api.py
View File

@ -4,6 +4,7 @@ import os
from crc import session, app
from crc.models.api_models import WorkflowApiSchema, Task
from crc.models.file import FileModelSchema
from crc.models.stats import WorkflowStatsModel, TaskEventModel
from crc.models.study import StudyModel
from crc.models.workflow import WorkflowSpecModelSchema, WorkflowModel, WorkflowStatus
from crc.services.workflow_processor import WorkflowProcessor
@ -28,6 +29,7 @@ class TestTasksApi(BaseTest):
def get_workflow_api(self, workflow, soft_reset=False, hard_reset=False):
rv = self.app.get('/v1.0/workflow/%i?soft_reset=%s&hard_reset=%s' %
(workflow.id, str(soft_reset), str(hard_reset)),
headers=self.logged_in_headers(),
content_type="application/json")
self.assert_success(rv)
json_data = json.loads(rv.get_data(as_text=True))
@ -37,10 +39,24 @@ class TestTasksApi(BaseTest):
def complete_form(self, workflow, task, dict_data):
rv = self.app.put('/v1.0/workflow/%i/task/%s/data' % (workflow.id, task.id),
headers=self.logged_in_headers(),
content_type="application/json",
data=json.dumps(dict_data))
self.assert_success(rv)
json_data = json.loads(rv.get_data(as_text=True))
num_stats = session.query(WorkflowStatsModel) \
.filter_by(workflow_id=workflow.id) \
.filter_by(workflow_spec_id=workflow.workflow_spec_id) \
.count()
self.assertGreater(num_stats, 0)
num_task_events = session.query(TaskEventModel) \
.filter_by(workflow_id=workflow.id) \
.filter_by(task_id=task.id) \
.count()
self.assertGreater(num_task_events, 0)
workflow = WorkflowApiSchema().load(json_data)
return workflow
@ -145,7 +161,7 @@ class TestTasksApi(BaseTest):
self.assertIsNotNone(workflow_api.next_task)
self.assertEquals("EndEvent_0evb22x", workflow_api.next_task['name'])
self.assertTrue(workflow_api.status == WorkflowStatus.complete)
rv = self.app.get('/v1.0/file?workflow_id=%i' % workflow.id)
rv = self.app.get('/v1.0/file?workflow_id=%i' % workflow.id, headers=self.logged_in_headers())
self.assert_success(rv)
json_data = json.loads(rv.get_data(as_text=True))
files = FileModelSchema(many=True).load(json_data, session=session)
@ -252,3 +268,30 @@ class TestTasksApi(BaseTest):
workflow_api = self.get_workflow_api(workflow)
self.assertTrue(workflow_api.spec_version.startswith("v1 "))
self.assertFalse(workflow_api.is_latest_spec)
def test_get_workflow_stats(self):
self.load_example_data()
workflow = self.create_workflow('exclusive_gateway')
response_before = self.app.get('/v1.0/workflow/%i/stats' % workflow.id,
content_type="application/json",
headers=self.logged_in_headers())
self.assert_success(response_before)
db_stats_before = session.query(WorkflowStatsModel).filter_by(workflow_id=workflow.id).first()
self.assertIsNone(db_stats_before)
# Start the workflow.
tasks = self.get_workflow_api(workflow).user_tasks
self.complete_form(workflow, tasks[0], {"has_bananas": True})
response_after = self.app.get('/v1.0/workflow/%i/stats' % workflow.id,
content_type="application/json",
headers=self.logged_in_headers())
self.assert_success(response_after)
db_stats_after = session.query(WorkflowStatsModel).filter_by(workflow_id=workflow.id).first()
self.assertIsNotNone(db_stats_after)
self.assertGreater(db_stats_after.num_tasks_complete, 0)
self.assertGreater(db_stats_after.num_tasks_incomplete, 0)
self.assertGreater(db_stats_after.num_tasks_total, 0)
self.assertEqual(db_stats_after.num_tasks_total,
db_stats_after.num_tasks_complete + db_stats_after.num_tasks_incomplete)

8
tests/test_workflow_spec_api.py
View File

@ -27,7 +27,9 @@ class TestWorkflowSpec(BaseTest):
num_before = session.query(WorkflowSpecModel).count()
spec = WorkflowSpecModel(id='make_cookies', display_name='Cooooookies',
description='Om nom nom delicious cookies')
rv = self.app.post('/v1.0/workflow-specification', content_type="application/json",
rv = self.app.post('/v1.0/workflow-specification',
headers=self.logged_in_headers(),
content_type="application/json",
data=json.dumps(WorkflowSpecModelSchema().dump(spec)))
self.assert_success(rv)
db_spec = session.query(WorkflowSpecModel).filter_by(id='make_cookies').first()
@ -38,7 +40,7 @@ class TestWorkflowSpec(BaseTest):
def test_get_workflow_specification(self):
self.load_example_data()
db_spec = session.query(WorkflowSpecModel).first()
rv = self.app.get('/v1.0/workflow-specification/%s' % db_spec.id)
rv = self.app.get('/v1.0/workflow-specification/%s' % db_spec.id, headers=self.logged_in_headers())
self.assert_success(rv)
json_data = json.loads(rv.get_data(as_text=True))
api_spec = WorkflowSpecModelSchema().load(json_data, session=session)
@ -55,7 +57,7 @@ class TestWorkflowSpec(BaseTest):
num_workflows_before = session.query(WorkflowModel).filter_by(workflow_spec_id=spec_id).count()
self.assertGreater(num_files_before + num_workflows_before, 0)
rv = self.app.delete('/v1.0/workflow-specification/' + spec_id)
rv = self.app.delete('/v1.0/workflow-specification/' + spec_id, headers=self.logged_in_headers())
self.assert_success(rv)
num_specs_after = session.query(WorkflowSpecModel).filter_by(id=spec_id).count()