Merge branch 'dev' into chore/unlock-admin-sandbox-739

README.md

@@ -128,6 +128,7 @@ define the Docker Container, as /config/default.py offers a good example of the
       - PB_ENABLED=true   # Generally true, we should connect to Protocol Builder
       - PREFERRED_URL_SCHEME=https   # Generally you want to run on SSL, should be https 
       - SERVER_NAME=testing.crconnect.uvadcos.io  # The url used to access this app.
+      - INSTANCE_NAME=testing  # This is the informal name of the server, used in BPMN documents
       - TOKEN_AUTH_SECRET_KEY=-0-0-0- TESTING SUPER SECURE -0-0-0- # Some random characters that seed our key gen.
       - APPLICATION_ROOT=/api # Appended to SERVER_NAME, is the full path to this service
       - ADMIN_UIDS=dhf8r,cah3us  # A comma delimited list of people who can preform administrative tasks.

config/default.py

@@ -16,6 +16,7 @@ API_TOKEN = environ.get('API_TOKEN', default = 'af95596f327c9ecc007b60414fc84b61
 
 NAME = "CR Connect Workflow"
 SERVER_NAME = environ.get('SERVER_NAME', default="localhost:5000")
+INSTANCE_NAME = environ.get('INSTANCE_NAME', default='development')
 DEFAULT_PORT = "5000"
 FLASK_PORT = environ.get('PORT0') or environ.get('FLASK_PORT', default=DEFAULT_PORT)
 FRONTEND = environ.get('FRONTEND', default="localhost:4200")

crc/api/admin.py

@@ -43,6 +43,9 @@ class UserView(AdminModelView):
 class StudyView(AdminModelView):
     column_filters = ['id']
     column_searchable_list = ['title']
+    can_create = True
+    can_edit = True
+    can_delete = True
 
 
 class ApprovalView(AdminModelView):
@@ -56,6 +59,9 @@ class WorkflowView(AdminModelView):
 class FileView(AdminModelView):
     column_filters = ['workflow_id', 'type']
     column_exclude_list = ['data']
+    can_create = True
+    can_edit = True
+    can_delete = True
 
     @action('publish', 'Publish', 'Are you sure you want to publish this file(s)?')
     def action_publish(self, ids):
@@ -77,6 +83,8 @@ class EmailView(AdminModelView):
 
 class TaskLogView(AdminModelView):
     column_exclude_list = ['id']
+    column_searchable_list = ['code', 'message', 'task']
+    column_filters = ['level', 'code', 'study_id', 'workflow_id', 'workflow_spec_id']
     can_create = True
     can_edit = True
     can_delete = True
@@ -87,6 +95,7 @@ def json_formatter(view, context, model, name):
     json_value = json.dumps(value, ensure_ascii=False, indent=2)
     return markupsafe.Markup(f'<pre>{json_value}</pre>')
 
+
 class TaskEventView(AdminModelView):
     column_filters = ['workflow_id', 'action']
     column_list = ['study_id', 'user_id', 'workflow_id', 'action', 'task_title', 'form_data', 'date']

crc/models/file.py

@@ -80,37 +80,6 @@ class FileModel(db.Model):
     archived = db.Column(db.Boolean, default=False)
 
 
-# class DocumentModel(FileModel):
-#     ...
-
-
-class FileDataModel(db.Model):
-    # TODO: remove when the file refactor is finished
-    __tablename__ = 'file_data'
-    id = db.Column(db.Integer, primary_key=True)
-    md5_hash = db.Column(UUID(as_uuid=True), unique=False, nullable=False)
-    data = deferred(db.Column(db.LargeBinary))  # Don't load it unless you have to.
-    version = db.Column(db.Integer, default=0)
-    size = db.Column(db.Integer, default=0)
-    date_created = db.Column(db.DateTime(timezone=True), server_default=func.now())
-    file_model_id = db.Column(db.Integer, db.ForeignKey('file.id'))
-    file_model = db.relationship("FileModel", foreign_keys=[file_model_id])
-    user_uid = db.Column(db.String, db.ForeignKey('user.uid'), nullable=True)
-
-
-class OldFileModel(db.Model):
-    # TODO: remove when the file refactor is finished
-    __tablename__ = 'old_file'
-    id = db.Column(db.Integer, primary_key=True)
-    name = db.Column(db.String)
-    type = db.Column(db.Enum(FileType))
-    content_type = db.Column(db.String)
-    workflow_id = db.Column(db.Integer, db.ForeignKey('workflow.id'), nullable=True)
-    task_spec = db.Column(db.String, nullable=True)
-    irb_doc_code = db.Column(db.String, nullable=True)  # Code reference to the documents.xlsx reference file.
-    # data_stores = relationship(DataStoreModel, cascade="all,delete", backref="file")
-
-
 class File(object):
     def __init__(self):
         self.content_type = None
@@ -165,15 +134,6 @@ class File(object):
         return instance
 
 
-# class DocumentModelSchema(SQLAlchemyAutoSchema):
-#     class Meta:
-#         model = DocumentModel
-#         load_instance = True
-#         include_relationships = True
-#         include_fk = True  # Includes foreign keys
-#         unknown = EXCLUDE
-
-
 class FileModelSchema(SQLAlchemyAutoSchema):
     class Meta:
         model = FileModel
@@ -193,7 +153,8 @@ class FileSchema(Schema):
         unknown = INCLUDE
     url = Method("get_url")
 
-    def get_url(self, obj):
+    @staticmethod
+    def get_url(obj):
         token = 'not_available'
         if hasattr(obj, 'id') and obj.id is not None:
             file_url = url_for("/v1_0.crc_api_file_get_file_data_link", file_id=obj.id, _external=True)

crc/scripts/complete_template.py

@@ -22,9 +22,13 @@ class CompleteTemplate(Script):
         return """Using the Jinja template engine, takes data available in the current task, and uses it to populate 
 a word document that contains Jinja markup.  Please see https://docxtpl.readthedocs.io/en/latest/ 
 for more information on exact syntax.
-Takes two arguments:
+
+Takes two required arguments:
 1. The name of a MS Word docx file to use as a template.
-2. The 'code' of the IRB Document as set in the documents.xlsx file."
+2. The 'code' of the IRB Document as set in the documents.xlsx file.
+
+And one optional argument:
+1. The name for the generated file. Otherwise, we use the first argument.
 """
 
     def do_task_validate_only(self, task, study_id, workflow_id, *args, **kwargs):
@@ -33,14 +37,16 @@ Takes two arguments:
         self.process_template(task, study_id, workflow, *args, **kwargs)
 
     def do_task(self, task, study_id, workflow_id, *args, **kwargs):
-        workflow_spec_service = WorkflowSpecService()
         workflow = session.query(WorkflowModel).filter(WorkflowModel.id == workflow_id).first()
         final_document_stream = self.process_template(task, study_id, workflow, *args, **kwargs)
-        file_name = args[0]
+        template_file_name = args[0]
         irb_doc_code = args[1]
+        file_name = None
+        if len(args) > 2:
+            file_name = args[2]
         UserFileService.add_workflow_file(workflow_id=workflow_id,
                                           task_spec_name=task.get_name(),
-                                          name=file_name,
+                                          name=file_name if file_name else template_file_name,
                                           content_type=CONTENT_TYPES['docx'],
                                           binary_data=final_document_stream.read(),
                                           irb_doc_code=irb_doc_code)
@@ -77,7 +83,8 @@ Takes two arguments:
             raise WorkflowTaskExecException(task, ae.message, exception=ae, line_number=ae.line_number,
                                             error_line=ae.error_line)
 
-    def get_image_file_data(self, fields_str, task):
+    @staticmethod
+    def get_image_file_data(fields_str, task):
         image_file_data = []
         images_field_str = re.sub(r'[\[\]]', '', fields_str)
         images_field_keys = [v.strip() for v in images_field_str.strip().split(',')]

crc/scripts/get_instance.py

@@ -0,0 +1,18 @@
+from crc import app
+from crc.scripts.script import Script
+
+
+class GetInstance(Script):
+
+    def get_description(self):
+        return """Get the name of the current instance, using the INSTANCE_NAME environment variable."""
+
+    def do_task_validate_only(self, task, study_id, workflow_id, *args, **kwargs):
+        return self.do_task(task, study_id, workflow_id, *args, **kwargs)
+
+    def do_task(self, task, study_id, workflow_id, *args, **kwargs):
+        if 'INSTANCE_NAME' in app.config:
+            return app.config['INSTANCE_NAME']
+        # TODO: Not sure what we should do here
+        app.logger.info('no_instance_name: INSTANCE_NAME not configured for this server.')
+        return ''

crc/scripts/get_zipped_files.py

@@ -44,7 +44,6 @@ class GetZippedFiles(Script):
                         for file in files:
                             zip_key_words = doc_info[file.irb_doc_code]['zip_key_words']
                             file_name = f'{study_id} {zip_key_words} {file.name}'
-                            # file_data = session.query(FileDataModel).filter(FileDataModel.file_model_id == file.id).first()
                             zfw.writestr(file_name, file.data)
 
                     with open(temp_file.name, mode='rb') as handle:

crc/services/email_service.py

@@ -8,7 +8,6 @@ from jinja2 import Template
 from crc import app, db, mail, session
 
 from crc.models.email import EmailModel
-from crc.models.file import FileDataModel
 from crc.models.study import StudyModel
 
 from crc.services.jinja_service import JinjaService

crc/services/user_file_service.py

@@ -5,7 +5,6 @@ import random
 import string
 
 import pandas as pd
-from github import Github, GithubObject, UnknownObjectException
 from uuid import UUID
 from lxml import etree
 
@@ -15,7 +14,7 @@ from sqlalchemy.exc import IntegrityError
 from crc import session, app
 from crc.api.common import ApiError
 from crc.models.data_store import DataStoreModel
-from crc.models.file import FileType, FileDataModel, FileModel, FileModel
+from crc.models.file import FileType, FileModel
 from crc.models.workflow import WorkflowModel
 from crc.services.cache_service import cache
 from crc.services.user_service import UserService
@@ -136,23 +135,12 @@ class UserFileService(object):
 
     @staticmethod
     def get_workflow_data_files(workflow_id=None):
-        """Returns all the FileDataModels related to a running workflow -
+        """Returns all the FileModels related to a running workflow -
         So these are the latest data files that were uploaded or generated
         that go along with this workflow.  Not related to the spec in any way"""
         file_models = UserFileService.get_files(workflow_id=workflow_id)
         return file_models
 
-    @staticmethod
-    def get_file_data(file_id: int, version: int = None):
-        """Returns the file data with the given version, or the lastest file, if version isn't provided."""
-        query = session.query(FileDataModel) \
-            .filter(FileDataModel.file_model_id == file_id)
-        if version:
-            query = query.filter(FileDataModel.version == version)
-        else:
-            query = query.order_by(desc(FileDataModel.date_created))
-        return query.first()
-
     @staticmethod
     def delete_file_data_stores(file_id):
         try:

migrations/versions/3489d5a6a2c0_migrate_file_data_to_document_table.py

@@ -9,7 +9,7 @@ from alembic import op
 import sqlalchemy as sa
 
 from crc.models.data_store import DataStoreModel
-from crc.models.file import OldFileModel, FileModel, FileDataModel
+from crc.models.file import FileModel  # OldFileModel, , FileDataModel
 
 
 

migrations/versions/546575fa21a8_file_refactor_cleanup.py

@@ -0,0 +1,28 @@
+"""file refactor cleanup
+
+Revision ID: 546575fa21a8
+Revises: ea1cd0f3d603
+Create Date: 2022-05-20 08:11:10.540804
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '546575fa21a8'
+down_revision = 'ea1cd0f3d603'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.drop_constraint('document_id_key', 'data_store', type_='foreignkey')
+    op.drop_table('document')
+    op.drop_table('file_data')
+    op.drop_table('old_file')
+
+
+def downgrade():
+    # This is cleanup from file refactor. There is no downgrade.
+    pass

migrations/versions/7225d990740e_move_files_to_filesystem.py

@@ -11,7 +11,7 @@ import sqlalchemy as sa
 
 # import crc
 from crc import app, session
-from crc.models.file import FileModel, FileModelSchema, FileDataModel, LookupFileModel, CONTENT_TYPES
+from crc.models.file import FileModel, FileModelSchema, LookupFileModel, CONTENT_TYPES  # , FileDataModel
 from crc.services.spec_file_service import SpecFileService
 from crc.services.reference_file_service import ReferenceFileService
 from crc.services.workflow_service import WorkflowService

tests/data/docx_embedded/docx_embedded.bpmn

@@ -8,13 +8,14 @@
     <bpmn:userTask id="Activity_GetData" name="Get Data" camunda:formKey="DataForm">
       <bpmn:extensionElements>
         <camunda:formData>
-          <camunda:formField id="file_name" label="'File Name'"  type="string" />
-          <camunda:formField id="irb_doc_code" label="'IRB Doc Code'"  type="enum">
+          <camunda:formField id="template_file_name" label="&#39;File Name&#39;" type="string" />
+          <camunda:formField id="irb_doc_code" label="&#39;IRB Doc Code&#39;" type="enum">
             <camunda:value id="Study_App_Doc" name="Study_App_Doc" />
             <camunda:value id="Study_Protocol" name="Study_Protocol" />
           </camunda:formField>
-          <camunda:formField id="name" label="'Name'"  type="string" defaultValue="World" />
+          <camunda:formField id="name" label="&#39;Name&#39;" type="string" defaultValue="World" />
           <camunda:formField id="include_me" type="string" />
+          <camunda:formField id="file_name" label="&#39;File Name&#39;" type="string" />
         </camunda:formData>
       </bpmn:extensionElements>
       <bpmn:incoming>Flow_1lthj06</bpmn:incoming>
@@ -26,7 +27,10 @@
       <bpmn:outgoing>Flow_0ltznd4</bpmn:outgoing>
       <bpmn:script>print(f'name is {name}.')
 print(f'include_me is {include_me}')
-result = complete_template(file_name, irb_doc_code)</bpmn:script>
+if 'file_name' in globals():
+    result = complete_template(template_file_name, irb_doc_code, file_name)
+else:
+    result = complete_template(template_file_name, irb_doc_code)</bpmn:script>
     </bpmn:scriptTask>
     <bpmn:sequenceFlow id="Flow_0ltznd4" sourceRef="Activity_CompleteTemplate" targetRef="Activity_DisplayData" />
     <bpmn:manualTask id="Activity_DisplayData" name="Display Data">
@@ -61,18 +65,18 @@ result = complete_template(file_name, irb_doc_code)</bpmn:script>
       <bpmndi:BPMNShape id="_BPMNShape_StartEvent_2" bpmnElement="StartEvent_1">
         <dc:Bounds x="179" y="99" width="36" height="36" />
       </bpmndi:BPMNShape>
+      <bpmndi:BPMNShape id="Activity_10trg2t_di" bpmnElement="Activity_GetData">
+        <dc:Bounds x="273" y="77" width="100" height="80" />
+      </bpmndi:BPMNShape>
+      <bpmndi:BPMNShape id="Activity_1u6kbns_di" bpmnElement="Activity_CompleteTemplate">
+        <dc:Bounds x="432" y="77" width="100" height="80" />
+      </bpmndi:BPMNShape>
       <bpmndi:BPMNShape id="Activity_14dp1gz_di" bpmnElement="Activity_DisplayData">
         <dc:Bounds x="590" y="77" width="100" height="80" />
       </bpmndi:BPMNShape>
       <bpmndi:BPMNShape id="Event_0edltir_di" bpmnElement="Event_0edltir">
         <dc:Bounds x="752" y="99" width="36" height="36" />
       </bpmndi:BPMNShape>
-      <bpmndi:BPMNShape id="Activity_1u6kbns_di" bpmnElement="Activity_CompleteTemplate">
-        <dc:Bounds x="432" y="77" width="100" height="80" />
-      </bpmndi:BPMNShape>
-      <bpmndi:BPMNShape id="Activity_10trg2t_di" bpmnElement="Activity_GetData">
-        <dc:Bounds x="273" y="77" width="100" height="80" />
-      </bpmndi:BPMNShape>
     </bpmndi:BPMNPlane>
   </bpmndi:BPMNDiagram>
 </bpmn:definitions>

tests/data/get_instance/get_instance.bpmn

@@ -0,0 +1,41 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<bpmn:definitions xmlns:bpmn="http://www.omg.org/spec/BPMN/20100524/MODEL" xmlns:bpmndi="http://www.omg.org/spec/BPMN/20100524/DI" xmlns:dc="http://www.omg.org/spec/DD/20100524/DC" xmlns:di="http://www.omg.org/spec/DD/20100524/DI" id="Definitions_0bhwbua" targetNamespace="http://bpmn.io/schema/bpmn" exporter="Camunda Modeler" exporterVersion="4.2.0">
+  <bpmn:process id="Process_GetInstanceScript" name="Get Instance" isExecutable="true">
+    <bpmn:startEvent id="StartEvent_1">
+      <bpmn:outgoing>Flow_1hony91</bpmn:outgoing>
+    </bpmn:startEvent>
+    <bpmn:sequenceFlow id="Flow_1hony91" sourceRef="StartEvent_1" targetRef="Activity_GetInstance" />
+    <bpmn:scriptTask id="Activity_GetInstance" name="Get Instance Script">
+      <bpmn:incoming>Flow_1hony91</bpmn:incoming>
+      <bpmn:outgoing>Flow_0ojlh77</bpmn:outgoing>
+      <bpmn:script>instance = get_instance()</bpmn:script>
+    </bpmn:scriptTask>
+    <bpmn:sequenceFlow id="Flow_0ojlh77" sourceRef="Activity_GetInstance" targetRef="Event_12sf522" />
+    <bpmn:endEvent id="Event_12sf522">
+      <bpmn:documentation>## Instance
+{{ instance }}</bpmn:documentation>
+      <bpmn:incoming>Flow_0ojlh77</bpmn:incoming>
+    </bpmn:endEvent>
+  </bpmn:process>
+  <bpmndi:BPMNDiagram id="BPMNDiagram_1">
+    <bpmndi:BPMNPlane id="BPMNPlane_1" bpmnElement="Process_GetInstanceScript">
+      <bpmndi:BPMNEdge id="Flow_1hony91_di" bpmnElement="Flow_1hony91">
+        <di:waypoint x="215" y="117" />
+        <di:waypoint x="270" y="117" />
+      </bpmndi:BPMNEdge>
+      <bpmndi:BPMNEdge id="Flow_0ojlh77_di" bpmnElement="Flow_0ojlh77">
+        <di:waypoint x="370" y="117" />
+        <di:waypoint x="592" y="117" />
+      </bpmndi:BPMNEdge>
+      <bpmndi:BPMNShape id="_BPMNShape_StartEvent_2" bpmnElement="StartEvent_1">
+        <dc:Bounds x="179" y="99" width="36" height="36" />
+      </bpmndi:BPMNShape>
+      <bpmndi:BPMNShape id="Activity_1k9au60_di" bpmnElement="Activity_GetInstance">
+        <dc:Bounds x="270" y="77" width="100" height="80" />
+      </bpmndi:BPMNShape>
+      <bpmndi:BPMNShape id="Event_12sf522_di" bpmnElement="Event_12sf522">
+        <dc:Bounds x="592" y="99" width="36" height="36" />
+      </bpmndi:BPMNShape>
+    </bpmndi:BPMNPlane>
+  </bpmndi:BPMNDiagram>
+</bpmn:definitions>

tests/scripts/test_get_instance.py

@@ -0,0 +1,14 @@
+from tests.base_test import BaseTest
+
+from crc import app
+
+
+class TestGetInstance(BaseTest):
+
+    def test_get_instance(self):
+        instance_name = app.config['INSTANCE_NAME']
+        workflow = self.create_workflow('get_instance')
+        workflow_api = self.get_workflow_api(workflow)
+        task = workflow_api.next_task
+        data = task.data
+        self.assertEqual(instance_name, data['instance'])

tests/test_complete_template_script.py

@@ -39,25 +39,45 @@ class TestCompleteTemplate(unittest.TestCase):
 
 class TestEmbeddedTemplate(BaseTest):
 
-    def test_embedded_template(self):
+    def run_docx_embedded_workflow(self, data):
         self.create_reference_document()
         workflow = self.create_workflow('docx_embedded')
         workflow_api = self.get_workflow_api(workflow)
         task = workflow_api.next_task
 
+        workflow_api = self.complete_form(workflow, task, data)
+        return workflow_api
+
+    def test_embedded_template(self):
         data = {'include_me': 'Hello {{ name }}!',
                 'name': 'World',
-                'file_name': 'simple.docx',
+                'template_file_name': 'simple.docx',
                 'irb_doc_code': 'Study_App_Doc'}
-        self.complete_form(workflow, task, data)
+        workflow_api = self.run_docx_embedded_workflow(data)
 
         # Get the file data created for us in the workflow
         file_model = session.query(FileModel).\
-            filter(FileModel.workflow_id == workflow.id).\
+            filter(FileModel.workflow_id == workflow_api.id).\
             filter(FileModel.irb_doc_code == 'Study_App_Doc').\
             first()
+        # If we don't pass file_name, name should be set to template_file_name
+        self.assertEqual(data['template_file_name'], file_model.name)
 
         # read the data as a word document
         document = docx.Document(BytesIO(file_model.data))
         # Make sure 'Hello World!' is there
         self.assertEqual('Hello World!', document.paragraphs[4].text)
+
+        data = {'include_me': 'Hello {{ name }}!',
+                'name': 'World',
+                'template_file_name': 'simple.docx',
+                'irb_doc_code': 'Study_App_Doc',
+                'file_name': 'test_file_name.docx'}
+        workflow_api = self.run_docx_embedded_workflow(data)
+
+        file_model = session.query(FileModel).\
+            filter(FileModel.workflow_id == workflow_api.id).\
+            filter(FileModel.irb_doc_code == 'Study_App_Doc').\
+            first()
+        # If we do pass file_name, name should be set to file_name
+        self.assertEqual(data['file_name'], file_model.name)