sartography
/
cr-connect-workflow
mirror of https://github.com/sartography/cr-connect-workflow.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

As we now have an approval_service.py, I moved all the business logic into this service and out of the request_approval.py script. And moved all tests for these features into a test file for the service. Will make it easier to cross reference what is happening, as everything all happens in one file. 

As many of the scripts need to know the workflow, and it's down in a weird parameter, moved this so it is passed in each time.
This commit is contained in:
Dan Funk 2020-05-24 16:13:15 -04:00
parent e9bd19b112
commit 971d9a58e9
11 changed files with 250 additions and 160 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
crc/scripts/complete_template.py
View File

@ -27,12 +27,11 @@ Takes two arguments:
2. The 'code' of the IRB Document as set in the irb_documents.xlsx file."
"""
def do_task_validate_only(self, task, study_id, *args, **kwargs):
def do_task_validate_only(self, task, study_id, workflow_id, *args, **kwargs):
"""For validation only, process the template, but do not store it in the database."""
self.process_template(task, study_id, None, *args, **kwargs)
def do_task(self, task, study_id, *args, **kwargs):
workflow_id = task.workflow.data[WorkflowProcessor.WORKFLOW_ID_KEY]
def do_task(self, task, study_id, workflow_id, *args, **kwargs):
workflow = session.query(WorkflowModel).filter(WorkflowModel.id == workflow_id).first()
final_document_stream = self.process_template(task, study_id, workflow, *args, **kwargs)
file_name = args[0]

6
crc/scripts/fact_service.py
View File

@ -20,10 +20,10 @@ class FactService(Script):
response = requests.get('https://api.chucknorris.io/jokes/random')
return response.json()['value']
def do_task_validate_only(self, task, study_id, **kwargs):
self.do_task(task, study_id, **kwargs)
def do_task_validate_only(self, task, study_id, workflow_id, **kwargs):
self.do_task(task, study_id, workflow_id, **kwargs)
def do_task(self, task, study_id, **kwargs):
def do_task(self, task, study_id, workflow_id, **kwargs):
print(task.data)
if "type" not in task.data:

48
crc/scripts/request_approval.py Normal file
View File

@ -0,0 +1,48 @@
from crc.api.common import ApiError
from crc.scripts.script import Script
from crc.services.approval_service import ApprovalService
class RequestApproval(Script):
"""This still needs to be fully wired up as a Script task callable from the workflow
But the basic logic is here just to get the tests passing and logic sound. """
def get_description(self):
return """
Creates an approval request on this workflow, by the given approver_uid(s),"
Takes one argument, which should point to data located in current task.
Example:
RequestApproval required_approvals.uids
"""
def do_task_validate_only(self, task, study_id, workflow_id, *args, **kwargs):
self.get_uids(task, args)
def do_task(self, task, study_id, workflow_id, *args, **kwargs):
uids = self.get_uids(task, args)
if isinstance(uids, str):
ApprovalService.add_approval(study_id, workflow_id, args)
elif isinstance(uids, list):
for id in uids:
ApprovalService.add_approval(study_id, workflow_id, id)
def get_uids(self, task, args):
if len(args) != 1:
raise ApiError(code="missing_argument",
message="The RequestApproval script requires 1 argument. The "
"the name of the variable in the task data that contains user"
"ids to process.")
uids = task.workflow.script_engine.evaluate(task, args[0])
if not isinstance(uids, str) and not isinstance(uids, list):
raise ApiError(code="invalid_argument",
message="The RequestApproval script requires 1 argument. The "
"the name of the variable in the task data that contains user"
"ids to process. This must point to an array or a string, but "
"it currently points to a %s " % uids.__class__.__name__)
return uids

71
crc/scripts/request_review.py
View File

@ -1,71 +0,0 @@
from datetime import datetime
from sqlalchemy import desc
from crc import db
from crc.models.approval import ApprovalModel, ApprovalStatus, ApprovalFile
from crc.scripts.script import Script
from crc.services.file_service import FileService
class RequestApproval(Script):
"""This still needs to be fully wired up as a Script task callable from the workflow
But the basic logic is here just to get the tests passing and logic sound. """
def get_description(self):
return "Creates an approval request on this workflow, by the given approver_uid"
def add_approval(self, study_id, workflow_id, approver_uid):
"""we might have multiple approvals for a workflow, so I would expect this
method to get called many times."""
# Find any existing approvals for this workflow and approver.
latest_approval_request = db.session.query(ApprovalModel).\
filter(ApprovalModel.workflow_id == workflow_id). \
filter(ApprovalModel.approver_uid == approver_uid). \
order_by(desc(ApprovalModel.version)).first()
# Construct as hash of the latest files to see if things have changed since
# the last approval.
latest_files = FileService.get_workflow_files(workflow_id)
current_workflow_hash = self.generate_workflow_hash(latest_files)
# If an existing approval request exists and no changes were made, do nothing.
# If there is an existing approval request for a previous version of the workflow
# then add a new request, and cancel any waiting/pending requests.
if latest_approval_request:
# We could just compare the ApprovalFile lists here and do away with this hash.
if latest_approval_request.workflow_hash == current_workflow_hash:
return # This approval already exists.
else:
latest_approval_request.status = ApprovalStatus.CANCELED.value
db.session.add(latest_approval_request)
version = latest_approval_request.version + 1
else:
version = 1
model = ApprovalModel(study_id=study_id, workflow_id=workflow_id,
approver_uid=approver_uid, status=ApprovalStatus.WAITING.value,
message="", date_created=datetime.now(),
version=version, workflow_hash=current_workflow_hash)
approval_files = self.create_approval_files(latest_files, model)
db.session.add(model)
db.session.add_all(approval_files)
db.session.commit()
def create_approval_files(self, files, approval):
"""Currently based exclusively on the status of files associated with a workflow."""
file_approval_models = []
for file in files:
file_approval_models.append(ApprovalFile(file_id=file.id,
approval=approval,
file_version=file.latest_version))
return file_approval_models
def generate_workflow_hash(self, files):
"""Currently based exclusively on the status of files associated with a workflow."""
version_array = []
for file in files:
version_array.append(str(file.id) + "[" + str(file.latest_version) + "]")
full_version = "-".join(version_array)
return full_version

4
crc/scripts/script.py
View File

@ -13,12 +13,12 @@ class Script(object):
raise ApiError("invalid_script",
"This script does not supply a description.")
def do_task(self, task, study_id, **kwargs):
def do_task(self, task, study_id, workflow_id, **kwargs):
raise ApiError("invalid_script",
"This is an internal error. The script you are trying to execute '%s' " % self.__class__.__name__ +
"does not properly implement the do_task function.")
def do_task_validate_only(self, task, study_id, **kwargs):
def do_task_validate_only(self, task, study_id, workflow_id, **kwargs):
raise ApiError("invalid_script",
"This is an internal error. The script you are trying to execute '%s' " % self.__class__.__name__ +
"does must provide a validate_only option that mimics the do_task, " +

4
crc/scripts/study_info.py
View File

@ -138,7 +138,7 @@ Returns information specific to the protocol.
documents_example=self.example_to_string("documents"),
)
def do_task_validate_only(self, task, study_id, *args, **kwargs):
def do_task_validate_only(self, task, study_id, workflow_id, *args, **kwargs):
"""For validation only, pretend no results come back from pb"""
self.check_args(args)
# Assure the reference file exists (a bit hacky, but we want to raise this error early, and cleanly.)
@ -184,7 +184,7 @@ Returns information specific to the protocol.
self.add_data_to_task(task=task, data=data["study"])
self.add_data_to_task(task, {"documents": StudyService().get_documents_status(study_id)})
def do_task(self, task, study_id, *args, **kwargs):
def do_task(self, task, study_id, workflow_id, *args, **kwargs):
self.check_args(args)
cmd = args[0]

71
crc/services/approval_service.py
View File

@ -1,6 +1,11 @@
from crc import db, session, app
from datetime import datetime
from crc.models.approval import ApprovalModel
from sqlalchemy import desc
from crc import db, session
from crc.models.approval import ApprovalModel, ApprovalStatus, ApprovalFile
from crc.services.file_service import FileService
class ApprovalService(object):
@ -28,3 +33,65 @@ class ApprovalService(object):
session.commit()
# TODO: Log update action by approver_uid - maybe ?
return db_approval
@staticmethod
def add_approval(study_id, workflow_id, approver_uid):
"""we might have multiple approvals for a workflow, so I would expect this
method to get called multiple times for the same workflow. This will
only add a new approval if no approval already exists for the approver_uid,
unless the workflow has changed, at which point, it will CANCEL any
pending approvals and create a new approval for the latest version
of the workflow."""
# Find any existing approvals for this workflow and approver.
latest_approval_request = db.session.query(ApprovalModel). \
filter(ApprovalModel.workflow_id == workflow_id). \
filter(ApprovalModel.approver_uid == approver_uid). \
order_by(desc(ApprovalModel.version)).first()
# Construct as hash of the latest files to see if things have changed since
# the last approval.
latest_files = FileService.get_workflow_files(workflow_id)
current_workflow_hash = ApprovalService._generate_workflow_hash(latest_files)
# If an existing approval request exists and no changes were made, do nothing.
# If there is an existing approval request for a previous version of the workflow
# then add a new request, and cancel any waiting/pending requests.
if latest_approval_request:
# We could just compare the ApprovalFile lists here and do away with this hash.
if latest_approval_request.workflow_hash == current_workflow_hash:
return # This approval already exists.
else:
latest_approval_request.status = ApprovalStatus.CANCELED.value
db.session.add(latest_approval_request)
version = latest_approval_request.version + 1
else:
version = 1
model = ApprovalModel(study_id=study_id, workflow_id=workflow_id,
approver_uid=approver_uid, status=ApprovalStatus.WAITING.value,
message="", date_created=datetime.now(),
version=version, workflow_hash=current_workflow_hash)
approval_files = ApprovalService._create_approval_files(latest_files, model)
db.session.add(model)
db.session.add_all(approval_files)
db.session.commit()
@staticmethod
def _create_approval_files(files, approval):
"""Currently based exclusively on the status of files associated with a workflow."""
file_approval_models = []
for file in files:
file_approval_models.append(ApprovalFile(file_id=file.id,
approval=approval,
file_version=file.latest_version))
return file_approval_models
@staticmethod
def _generate_workflow_hash(files):
"""Currently based exclusively on the status of files associated with a workflow."""
version_array = []
for file in files:
version_array.append(str(file.id) + "[" + str(file.latest_version) + "]")
full_version = "-".join(version_array)
return full_version

9
crc/services/workflow_processor.py
View File

@ -48,6 +48,11 @@ class CustomBpmnScriptEngine(BpmnScriptEngine):
mod = __import__(module_name, fromlist=[class_name])
klass = getattr(mod, class_name)
study_id = task.workflow.data[WorkflowProcessor.STUDY_ID_KEY]
if(WorkflowProcessor.WORKFLOW_ID_KEY in task.workflow.data):
workflow_id = task.workflow.data[WorkflowProcessor.WORKFLOW_ID_KEY]
else:
workflow_id = None
if not isinstance(klass(), Script):
raise ApiError.from_task("invalid_script",
"This is an internal error. The script '%s:%s' you called " %
@ -57,9 +62,9 @@ class CustomBpmnScriptEngine(BpmnScriptEngine):
if task.workflow.data[WorkflowProcessor.VALIDATION_PROCESS_KEY]:
"""If this is running a validation, and not a normal process, then we want to
mimic running the script, but not make any external calls or database changes."""
klass().do_task_validate_only(task, study_id, *commands[1:])
klass().do_task_validate_only(task, study_id, workflow_id, *commands[1:])
else:
klass().do_task(task, study_id, *commands[1:])
klass().do_task(task, study_id, workflow_id, *commands[1:])
except ModuleNotFoundError:
raise ApiError.from_task("invalid_script",
"Unable to locate Script: '%s:%s'" % (module_name, class_name),

94
tests/test_approvals_service.py Normal file
View File

@ -0,0 +1,94 @@
from tests.base_test import BaseTest
from crc import db
from crc.models.approval import ApprovalModel
from crc.services.approval_service import ApprovalService
from crc.services.file_service import FileService
from crc.services.workflow_processor import WorkflowProcessor
class TestApprovalsService(BaseTest):
def test_create_approval_record(self):
workflow = self.create_workflow("empty_workflow")
ApprovalService.add_approval(study_id=workflow.study_id, workflow_id=workflow.id, approver_uid="dhf8r")
self.assertEquals(1, db.session.query(ApprovalModel).count())
model = db.session.query(ApprovalModel).first()
self.assertEquals(workflow.study_id, model.study_id)
self.assertEquals(workflow.id, model.workflow_id)
self.assertEquals("dhf8r", model.approver_uid)
self.assertEquals(1, model.version)
self.assertIsNotNone(model.workflow_hash)
def test_new_requests_dont_add_if_approval_exists_for_current_workflow(self):
workflow = self.create_workflow("empty_workflow")
ApprovalService.add_approval(study_id=workflow.study_id, workflow_id=workflow.id, approver_uid="dhf8r")
ApprovalService.add_approval(study_id=workflow.study_id, workflow_id=workflow.id, approver_uid="dhf8r")
self.assertEquals(1, db.session.query(ApprovalModel).count())
model = db.session.query(ApprovalModel).first()
self.assertEquals(1, model.version)
def test_new_approval_requests_after_file_modification_create_new_requests(self):
self.load_example_data()
self.create_reference_document()
workflow = self.create_workflow('empty_workflow')
processor = WorkflowProcessor(workflow)
task = processor.next_task()
ApprovalService.add_approval(study_id=workflow.study_id, workflow_id=workflow.id, approver_uid="dhf8r")
irb_code_1 = "UVACompl_PRCAppr" # The first file referenced in pb required docs.
FileService.add_task_file(study_id=workflow.study_id, workflow_id=workflow.id,
workflow_spec_id=workflow.workflow_spec_id,
task_id=task.id,
name="anything.png", content_type="text",
binary_data=b'5678', irb_doc_code=irb_code_1)
ApprovalService.add_approval(study_id=workflow.study_id, workflow_id=workflow.id, approver_uid="dhf8r")
self.assertEquals(2, db.session.query(ApprovalModel).count())
models = db.session.query(ApprovalModel).order_by(ApprovalModel.version).all()
self.assertEquals(1, models[0].version)
self.assertEquals(2, models[1].version)
def test_generate_workflow_hash_and_version(self):
self.load_example_data()
self.create_reference_document()
workflow = self.create_workflow('empty_workflow')
processor = WorkflowProcessor(workflow)
task = processor.next_task()
irb_code_1 = "UVACompl_PRCAppr" # The first file referenced in pb required docs.
irb_code_2 = "NonUVAIRB_AssuranceForm" # The second file in above.
# Add a task file to the workflow.
FileService.add_task_file(study_id=workflow.study_id, workflow_id=workflow.id,
workflow_spec_id=workflow.workflow_spec_id,
task_id=task.id,
name="anything.png", content_type="text",
binary_data=b'5678', irb_doc_code=irb_code_1)
# Add a two form field files with the same irb_code, but
FileService.add_form_field_file(study_id=workflow.study_id, workflow_id=workflow.id,
task_id=task.id,
form_field_key=irb_code_2,
name="anything.png", content_type="text",
binary_data=b'1234')
FileService.add_form_field_file(study_id=workflow.study_id, workflow_id=workflow.id,
form_field_key=irb_code_2,
task_id=task.id,
name="another_anything.png", content_type="text",
binary_data=b'5678')
# Workflow hash should look be id[1]-id[1]-id[1]
# Sould be three files, each with a version of 1.
# where id is the file id, which we don't know, thus the regex.
latest_files = FileService.get_workflow_files(workflow.id)
self.assertRegexpMatches(ApprovalService._generate_workflow_hash(latest_files), "\d+\[1\]-\d+\[1\]-\d+\[1\]")
# Replace last file
# should now be id[1]-id[1]-id[2]
FileService.add_form_field_file(study_id=workflow.study_id, workflow_id=workflow.id,
form_field_key=irb_code_2,
task_id=task.id,
name="another_anything.png", content_type="text",
binary_data=b'9999')
self.assertRegexpMatches(ApprovalService._generate_workflow_hash(latest_files), "\d+\[1\]-\d+\[1\]-\d+\[2\]")

94
tests/test_request_approval_script.py
View File

@ -1,100 +1,48 @@
from tests.base_test import BaseTest
from crc.services.file_service import FileService
from crc.scripts.request_approval import RequestApproval
from crc.services.workflow_processor import WorkflowProcessor
from crc.api.common import ApiError
from crc.scripts.request_review import RequestApproval
from crc import db
from crc.models.approval import ApprovalModel
class TestRequestApprovalScript(BaseTest):
def test_create_approval_record(self):
workflow = self.create_workflow("empty_workflow")
script = RequestApproval()
script.add_approval(study_id=workflow.study_id, workflow_id=workflow.id, approver_uid="dhf8r")
self.assertEquals(1, db.session.query(ApprovalModel).count())
model = db.session.query(ApprovalModel).first()
self.assertEquals(workflow.study_id, model.study_id)
self.assertEquals(workflow.id, model.workflow_id)
self.assertEquals("dhf8r", model.approver_uid)
self.assertEquals(1, model.version)
self.assertIsNotNone(model.workflow_hash)
def test_new_requests_dont_add_if_approval_exists_for_current_workflow(self):
workflow = self.create_workflow("empty_workflow")
script = RequestApproval()
script.add_approval(study_id=workflow.study_id, workflow_id=workflow.id, approver_uid="dhf8r")
script.add_approval(study_id=workflow.study_id, workflow_id=workflow.id, approver_uid="dhf8r")
self.assertEquals(1, db.session.query(ApprovalModel).count())
model = db.session.query(ApprovalModel).first()
self.assertEquals(1, model.version)
def test_new_approval_requests_after_file_modification_create_new_requests(self):
def test_do_task(self):
self.load_example_data()
self.create_reference_document()
workflow = self.create_workflow('empty_workflow')
processor = WorkflowProcessor(workflow)
task = processor.next_task()
task.data = {"approvals": ['dhf8r', 'lb3dp']}
script = RequestApproval()
script.add_approval(study_id=workflow.study_id, workflow_id=workflow.id, approver_uid="dhf8r")
irb_code_1 = "UVACompl_PRCAppr" # The first file referenced in pb required docs.
FileService.add_task_file(study_id=workflow.study_id, workflow_id=workflow.id,
workflow_spec_id=workflow.workflow_spec_id,
task_id=task.id,
name="anything.png", content_type="text",
binary_data=b'5678', irb_doc_code=irb_code_1)
script.add_approval(study_id=workflow.study_id, workflow_id=workflow.id, approver_uid="dhf8r")
script.do_task(task, workflow.study_id, workflow.id, "approvals")
self.assertEquals(2, db.session.query(ApprovalModel).count())
models = db.session.query(ApprovalModel).order_by(ApprovalModel.version).all()
self.assertEquals(1, models[0].version)
self.assertEquals(2, models[1].version)
def test_generate_workflow_hash_and_version(self):
def test_do_task_with_incorrect_argument(self):
"""This script should raise an error if it can't figure out the approvers."""
self.load_example_data()
self.create_reference_document()
workflow = self.create_workflow('empty_workflow')
processor = WorkflowProcessor(workflow)
task = processor.next_task()
irb_code_1 = "UVACompl_PRCAppr" # The first file referenced in pb required docs.
irb_code_2 = "NonUVAIRB_AssuranceForm" # The second file in above.
# Add a task file to the workflow.
FileService.add_task_file(study_id=workflow.study_id, workflow_id=workflow.id,
workflow_spec_id=workflow.workflow_spec_id,
task_id=task.id,
name="anything.png", content_type="text",
binary_data=b'5678', irb_doc_code=irb_code_1)
# Add a two form field files with the same irb_code, but
FileService.add_form_field_file(study_id=workflow.study_id, workflow_id=workflow.id,
task_id=task.id,
form_field_key=irb_code_2,
name="anything.png", content_type="text",
binary_data=b'1234')
FileService.add_form_field_file(study_id=workflow.study_id, workflow_id=workflow.id,
form_field_key=irb_code_2,
task_id=task.id,
name="another_anything.png", content_type="text",
binary_data=b'5678')
task.data = {"approvals": {'dhf8r':"invalid", 'lb3dp':"invalid"}}
script = RequestApproval()
with self.assertRaises(ApiError):
script.do_task(task, workflow.study_id, workflow.id, "approvals")
def test_do_task_validate_only(self):
self.load_example_data()
self.create_reference_document()
workflow = self.create_workflow('empty_workflow')
processor = WorkflowProcessor(workflow)
task = processor.next_task()
task.data = {"approvals": ['dhf8r', 'lb3dp']}
script = RequestApproval()
# Workflow hash should look be id[1]-id[1]-id[1]
# Sould be three files, each with a version of 1.
# where id is the file id, which we don't know, thus the regex.
latest_files = FileService.get_workflow_files(workflow.id)
self.assertRegexpMatches(script.generate_workflow_hash(latest_files), "\d+\[1\]-\d+\[1\]-\d+\[1\]")
# Replace last file
# should now be id[1]-id[1]-id[2]
FileService.add_form_field_file(study_id=workflow.study_id, workflow_id=workflow.id,
form_field_key=irb_code_2,
task_id=task.id,
name="another_anything.png", content_type="text",
binary_data=b'9999')
self.assertRegexpMatches(script.generate_workflow_hash(latest_files), "\d+\[1\]-\d+\[1\]-\d+\[2\]")
script.do_task_validate_only(task, workflow.study_id, workflow.id, "approvals")
self.assertEquals(0, db.session.query(ApprovalModel).count())

4
tests/test_study_details_documents.py
View File

@ -1,4 +1,5 @@
import json
from tests.base_test import BaseTest
from unittest.mock import patch
from crc import db, session
@ -10,7 +11,6 @@ from crc.scripts.study_info import StudyInfo
from crc.services.file_service import FileService
from crc.services.study_service import StudyService
from crc.services.workflow_processor import WorkflowProcessor
from tests.base_test import BaseTest
class TestStudyDetailsDocumentsScript(BaseTest):
@ -63,7 +63,7 @@ class TestStudyDetailsDocumentsScript(BaseTest):
workflow_model = StudyService._create_workflow_model(study, workflow_spec_model)
processor = WorkflowProcessor(workflow_model)
task = processor.next_task()
StudyInfo().do_task_validate_only(task, study.id, "documents")
StudyInfo().do_task_validate_only(task, study.id, workflow_model.id, "documents")
def test_load_lookup_data(self):
self.create_reference_document()