diff --git a/README.md b/README.md index 3ab6b22e..3e306fbf 100644 --- a/README.md +++ b/README.md @@ -128,6 +128,7 @@ define the Docker Container, as /config/default.py offers a good example of the - PB_ENABLED=true # Generally true, we should connect to Protocol Builder - PREFERRED_URL_SCHEME=https # Generally you want to run on SSL, should be https - SERVER_NAME=testing.crconnect.uvadcos.io # The url used to access this app. + - INSTANCE_NAME=testing # This is the informal name of the server, used in BPMN documents - TOKEN_AUTH_SECRET_KEY=-0-0-0- TESTING SUPER SECURE -0-0-0- # Some random characters that seed our key gen. - APPLICATION_ROOT=/api # Appended to SERVER_NAME, is the full path to this service - ADMIN_UIDS=dhf8r,cah3us # A comma delimited list of people who can preform administrative tasks. diff --git a/config/default.py b/config/default.py index b04d1791..ad638be7 100644 --- a/config/default.py +++ b/config/default.py @@ -16,6 +16,7 @@ API_TOKEN = environ.get('API_TOKEN', default = 'af95596f327c9ecc007b60414fc84b61 NAME = "CR Connect Workflow" SERVER_NAME = environ.get('SERVER_NAME', default="localhost:5000") +INSTANCE_NAME = environ.get('INSTANCE_NAME', default='development') DEFAULT_PORT = "5000" FLASK_PORT = environ.get('PORT0') or environ.get('FLASK_PORT', default=DEFAULT_PORT) FRONTEND = environ.get('FRONTEND', default="localhost:4200") diff --git a/crc/api/admin.py b/crc/api/admin.py index c3386428..99bada62 100644 --- a/crc/api/admin.py +++ b/crc/api/admin.py @@ -43,6 +43,9 @@ class UserView(AdminModelView): class StudyView(AdminModelView): column_filters = ['id'] column_searchable_list = ['title'] + can_create = True + can_edit = True + can_delete = True class ApprovalView(AdminModelView): @@ -56,6 +59,9 @@ class WorkflowView(AdminModelView): class FileView(AdminModelView): column_filters = ['workflow_id', 'type'] column_exclude_list = ['data'] + can_create = True + can_edit = True + can_delete = True @action('publish', 'Publish', 'Are you sure you want to publish this file(s)?') def action_publish(self, ids): @@ -77,6 +83,8 @@ class EmailView(AdminModelView): class TaskLogView(AdminModelView): column_exclude_list = ['id'] + column_searchable_list = ['code', 'message', 'task'] + column_filters = ['level', 'code', 'study_id', 'workflow_id', 'workflow_spec_id'] can_create = True can_edit = True can_delete = True @@ -87,6 +95,7 @@ def json_formatter(view, context, model, name): json_value = json.dumps(value, ensure_ascii=False, indent=2) return markupsafe.Markup(f'
{json_value}
') + class TaskEventView(AdminModelView): column_filters = ['workflow_id', 'action'] column_list = ['study_id', 'user_id', 'workflow_id', 'action', 'task_title', 'form_data', 'date'] diff --git a/crc/models/file.py b/crc/models/file.py index d4abb8fb..e38a3704 100644 --- a/crc/models/file.py +++ b/crc/models/file.py @@ -80,37 +80,6 @@ class FileModel(db.Model): archived = db.Column(db.Boolean, default=False) -# class DocumentModel(FileModel): -# ... - - -class FileDataModel(db.Model): - # TODO: remove when the file refactor is finished - __tablename__ = 'file_data' - id = db.Column(db.Integer, primary_key=True) - md5_hash = db.Column(UUID(as_uuid=True), unique=False, nullable=False) - data = deferred(db.Column(db.LargeBinary)) # Don't load it unless you have to. - version = db.Column(db.Integer, default=0) - size = db.Column(db.Integer, default=0) - date_created = db.Column(db.DateTime(timezone=True), server_default=func.now()) - file_model_id = db.Column(db.Integer, db.ForeignKey('file.id')) - file_model = db.relationship("FileModel", foreign_keys=[file_model_id]) - user_uid = db.Column(db.String, db.ForeignKey('user.uid'), nullable=True) - - -class OldFileModel(db.Model): - # TODO: remove when the file refactor is finished - __tablename__ = 'old_file' - id = db.Column(db.Integer, primary_key=True) - name = db.Column(db.String) - type = db.Column(db.Enum(FileType)) - content_type = db.Column(db.String) - workflow_id = db.Column(db.Integer, db.ForeignKey('workflow.id'), nullable=True) - task_spec = db.Column(db.String, nullable=True) - irb_doc_code = db.Column(db.String, nullable=True) # Code reference to the documents.xlsx reference file. - # data_stores = relationship(DataStoreModel, cascade="all,delete", backref="file") - - class File(object): def __init__(self): self.content_type = None @@ -165,15 +134,6 @@ class File(object): return instance -# class DocumentModelSchema(SQLAlchemyAutoSchema): -# class Meta: -# model = DocumentModel -# load_instance = True -# include_relationships = True -# include_fk = True # Includes foreign keys -# unknown = EXCLUDE - - class FileModelSchema(SQLAlchemyAutoSchema): class Meta: model = FileModel @@ -193,7 +153,8 @@ class FileSchema(Schema): unknown = INCLUDE url = Method("get_url") - def get_url(self, obj): + @staticmethod + def get_url(obj): token = 'not_available' if hasattr(obj, 'id') and obj.id is not None: file_url = url_for("/v1_0.crc_api_file_get_file_data_link", file_id=obj.id, _external=True) diff --git a/crc/scripts/complete_template.py b/crc/scripts/complete_template.py index c8ef9283..bc7e6d87 100644 --- a/crc/scripts/complete_template.py +++ b/crc/scripts/complete_template.py @@ -22,9 +22,13 @@ class CompleteTemplate(Script): return """Using the Jinja template engine, takes data available in the current task, and uses it to populate a word document that contains Jinja markup. Please see https://docxtpl.readthedocs.io/en/latest/ for more information on exact syntax. -Takes two arguments: + +Takes two required arguments: 1. The name of a MS Word docx file to use as a template. -2. The 'code' of the IRB Document as set in the documents.xlsx file." +2. The 'code' of the IRB Document as set in the documents.xlsx file. + +And one optional argument: +1. The name for the generated file. Otherwise, we use the first argument. """ def do_task_validate_only(self, task, study_id, workflow_id, *args, **kwargs): @@ -33,14 +37,16 @@ Takes two arguments: self.process_template(task, study_id, workflow, *args, **kwargs) def do_task(self, task, study_id, workflow_id, *args, **kwargs): - workflow_spec_service = WorkflowSpecService() workflow = session.query(WorkflowModel).filter(WorkflowModel.id == workflow_id).first() final_document_stream = self.process_template(task, study_id, workflow, *args, **kwargs) - file_name = args[0] + template_file_name = args[0] irb_doc_code = args[1] + file_name = None + if len(args) > 2: + file_name = args[2] UserFileService.add_workflow_file(workflow_id=workflow_id, task_spec_name=task.get_name(), - name=file_name, + name=file_name if file_name else template_file_name, content_type=CONTENT_TYPES['docx'], binary_data=final_document_stream.read(), irb_doc_code=irb_doc_code) @@ -77,7 +83,8 @@ Takes two arguments: raise WorkflowTaskExecException(task, ae.message, exception=ae, line_number=ae.line_number, error_line=ae.error_line) - def get_image_file_data(self, fields_str, task): + @staticmethod + def get_image_file_data(fields_str, task): image_file_data = [] images_field_str = re.sub(r'[\[\]]', '', fields_str) images_field_keys = [v.strip() for v in images_field_str.strip().split(',')] diff --git a/crc/scripts/get_instance.py b/crc/scripts/get_instance.py new file mode 100644 index 00000000..dfbad2c3 --- /dev/null +++ b/crc/scripts/get_instance.py @@ -0,0 +1,18 @@ +from crc import app +from crc.scripts.script import Script + + +class GetInstance(Script): + + def get_description(self): + return """Get the name of the current instance, using the INSTANCE_NAME environment variable.""" + + def do_task_validate_only(self, task, study_id, workflow_id, *args, **kwargs): + return self.do_task(task, study_id, workflow_id, *args, **kwargs) + + def do_task(self, task, study_id, workflow_id, *args, **kwargs): + if 'INSTANCE_NAME' in app.config: + return app.config['INSTANCE_NAME'] + # TODO: Not sure what we should do here + app.logger.info('no_instance_name: INSTANCE_NAME not configured for this server.') + return '' diff --git a/crc/scripts/get_zipped_files.py b/crc/scripts/get_zipped_files.py index 577c5903..de595c85 100644 --- a/crc/scripts/get_zipped_files.py +++ b/crc/scripts/get_zipped_files.py @@ -44,7 +44,6 @@ class GetZippedFiles(Script): for file in files: zip_key_words = doc_info[file.irb_doc_code]['zip_key_words'] file_name = f'{study_id} {zip_key_words} {file.name}' - # file_data = session.query(FileDataModel).filter(FileDataModel.file_model_id == file.id).first() zfw.writestr(file_name, file.data) with open(temp_file.name, mode='rb') as handle: diff --git a/crc/services/email_service.py b/crc/services/email_service.py index 43e9428a..49ab69ef 100644 --- a/crc/services/email_service.py +++ b/crc/services/email_service.py @@ -8,7 +8,6 @@ from jinja2 import Template from crc import app, db, mail, session from crc.models.email import EmailModel -from crc.models.file import FileDataModel from crc.models.study import StudyModel from crc.services.jinja_service import JinjaService diff --git a/crc/services/user_file_service.py b/crc/services/user_file_service.py index d409a727..b476a362 100644 --- a/crc/services/user_file_service.py +++ b/crc/services/user_file_service.py @@ -5,7 +5,6 @@ import random import string import pandas as pd -from github import Github, GithubObject, UnknownObjectException from uuid import UUID from lxml import etree @@ -15,7 +14,7 @@ from sqlalchemy.exc import IntegrityError from crc import session, app from crc.api.common import ApiError from crc.models.data_store import DataStoreModel -from crc.models.file import FileType, FileDataModel, FileModel, FileModel +from crc.models.file import FileType, FileModel from crc.models.workflow import WorkflowModel from crc.services.cache_service import cache from crc.services.user_service import UserService @@ -136,23 +135,12 @@ class UserFileService(object): @staticmethod def get_workflow_data_files(workflow_id=None): - """Returns all the FileDataModels related to a running workflow - + """Returns all the FileModels related to a running workflow - So these are the latest data files that were uploaded or generated that go along with this workflow. Not related to the spec in any way""" file_models = UserFileService.get_files(workflow_id=workflow_id) return file_models - @staticmethod - def get_file_data(file_id: int, version: int = None): - """Returns the file data with the given version, or the lastest file, if version isn't provided.""" - query = session.query(FileDataModel) \ - .filter(FileDataModel.file_model_id == file_id) - if version: - query = query.filter(FileDataModel.version == version) - else: - query = query.order_by(desc(FileDataModel.date_created)) - return query.first() - @staticmethod def delete_file_data_stores(file_id): try: diff --git a/migrations/versions/3489d5a6a2c0_migrate_file_data_to_document_table.py b/migrations/versions/3489d5a6a2c0_migrate_file_data_to_document_table.py index 6b748163..fa7a94f3 100644 --- a/migrations/versions/3489d5a6a2c0_migrate_file_data_to_document_table.py +++ b/migrations/versions/3489d5a6a2c0_migrate_file_data_to_document_table.py @@ -9,7 +9,7 @@ from alembic import op import sqlalchemy as sa from crc.models.data_store import DataStoreModel -from crc.models.file import OldFileModel, FileModel, FileDataModel +from crc.models.file import FileModel # OldFileModel, , FileDataModel diff --git a/migrations/versions/546575fa21a8_file_refactor_cleanup.py b/migrations/versions/546575fa21a8_file_refactor_cleanup.py new file mode 100644 index 00000000..51d5ba84 --- /dev/null +++ b/migrations/versions/546575fa21a8_file_refactor_cleanup.py @@ -0,0 +1,28 @@ +"""file refactor cleanup + +Revision ID: 546575fa21a8 +Revises: ea1cd0f3d603 +Create Date: 2022-05-20 08:11:10.540804 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '546575fa21a8' +down_revision = 'ea1cd0f3d603' +branch_labels = None +depends_on = None + + +def upgrade(): + op.drop_constraint('document_id_key', 'data_store', type_='foreignkey') + op.drop_table('document') + op.drop_table('file_data') + op.drop_table('old_file') + + +def downgrade(): + # This is cleanup from file refactor. There is no downgrade. + pass diff --git a/migrations/versions/7225d990740e_move_files_to_filesystem.py b/migrations/versions/7225d990740e_move_files_to_filesystem.py index d3641763..3628be32 100644 --- a/migrations/versions/7225d990740e_move_files_to_filesystem.py +++ b/migrations/versions/7225d990740e_move_files_to_filesystem.py @@ -11,7 +11,7 @@ import sqlalchemy as sa # import crc from crc import app, session -from crc.models.file import FileModel, FileModelSchema, FileDataModel, LookupFileModel, CONTENT_TYPES +from crc.models.file import FileModel, FileModelSchema, LookupFileModel, CONTENT_TYPES # , FileDataModel from crc.services.spec_file_service import SpecFileService from crc.services.reference_file_service import ReferenceFileService from crc.services.workflow_service import WorkflowService diff --git a/tests/data/docx_embedded/docx_embedded.bpmn b/tests/data/docx_embedded/docx_embedded.bpmn index 8992534a..8647c228 100644 --- a/tests/data/docx_embedded/docx_embedded.bpmn +++ b/tests/data/docx_embedded/docx_embedded.bpmn @@ -8,13 +8,14 @@ - - + + - + + Flow_1lthj06 @@ -26,7 +27,10 @@ Flow_0ltznd4 print(f'name is {name}.') print(f'include_me is {include_me}') -result = complete_template(file_name, irb_doc_code) +if 'file_name' in globals(): + result = complete_template(template_file_name, irb_doc_code, file_name) +else: + result = complete_template(template_file_name, irb_doc_code) @@ -61,18 +65,18 @@ result = complete_template(file_name, irb_doc_code) + + + + + + - - - - - - diff --git a/tests/data/get_instance/get_instance.bpmn b/tests/data/get_instance/get_instance.bpmn new file mode 100644 index 00000000..b0bae1fc --- /dev/null +++ b/tests/data/get_instance/get_instance.bpmn @@ -0,0 +1,41 @@ + + + + + Flow_1hony91 + + + + Flow_1hony91 + Flow_0ojlh77 + instance = get_instance() + + + + ## Instance +{{ instance }} + Flow_0ojlh77 + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/scripts/test_get_instance.py b/tests/scripts/test_get_instance.py new file mode 100644 index 00000000..aa6ef2b0 --- /dev/null +++ b/tests/scripts/test_get_instance.py @@ -0,0 +1,14 @@ +from tests.base_test import BaseTest + +from crc import app + + +class TestGetInstance(BaseTest): + + def test_get_instance(self): + instance_name = app.config['INSTANCE_NAME'] + workflow = self.create_workflow('get_instance') + workflow_api = self.get_workflow_api(workflow) + task = workflow_api.next_task + data = task.data + self.assertEqual(instance_name, data['instance']) diff --git a/tests/test_complete_template_script.py b/tests/test_complete_template_script.py index bb4dac9c..c18fc8b5 100644 --- a/tests/test_complete_template_script.py +++ b/tests/test_complete_template_script.py @@ -39,25 +39,45 @@ class TestCompleteTemplate(unittest.TestCase): class TestEmbeddedTemplate(BaseTest): - def test_embedded_template(self): + def run_docx_embedded_workflow(self, data): self.create_reference_document() workflow = self.create_workflow('docx_embedded') workflow_api = self.get_workflow_api(workflow) task = workflow_api.next_task + workflow_api = self.complete_form(workflow, task, data) + return workflow_api + + def test_embedded_template(self): data = {'include_me': 'Hello {{ name }}!', 'name': 'World', - 'file_name': 'simple.docx', + 'template_file_name': 'simple.docx', 'irb_doc_code': 'Study_App_Doc'} - self.complete_form(workflow, task, data) + workflow_api = self.run_docx_embedded_workflow(data) # Get the file data created for us in the workflow file_model = session.query(FileModel).\ - filter(FileModel.workflow_id == workflow.id).\ + filter(FileModel.workflow_id == workflow_api.id).\ filter(FileModel.irb_doc_code == 'Study_App_Doc').\ first() + # If we don't pass file_name, name should be set to template_file_name + self.assertEqual(data['template_file_name'], file_model.name) # read the data as a word document document = docx.Document(BytesIO(file_model.data)) # Make sure 'Hello World!' is there self.assertEqual('Hello World!', document.paragraphs[4].text) + + data = {'include_me': 'Hello {{ name }}!', + 'name': 'World', + 'template_file_name': 'simple.docx', + 'irb_doc_code': 'Study_App_Doc', + 'file_name': 'test_file_name.docx'} + workflow_api = self.run_docx_embedded_workflow(data) + + file_model = session.query(FileModel).\ + filter(FileModel.workflow_id == workflow_api.id).\ + filter(FileModel.irb_doc_code == 'Study_App_Doc').\ + first() + # If we do pass file_name, name should be set to file_name + self.assertEqual(data['file_name'], file_model.name)