*** WIP ***

Moved reference files to their own service

crc/api/file.py

@@ -13,6 +13,7 @@ from crc.models.file import FileSchema, FileModel, File, FileModelSchema, FileDa
 from crc.models.workflow import WorkflowSpecModel
 from crc.services.document_service import DocumentService
 from crc.services.file_service import FileService
+from crc.services.reference_file_service import ReferenceFileService
 from crc.services.spec_file_service import SpecFileService
 
 
@@ -21,7 +22,7 @@ def to_file_api(file_model):
     if file_model.workflow_spec_id is not None:
         file_data_model = SpecFileService().get_spec_file_data(file_model.id)
     elif file_model.is_reference:
-        file_data_model = SpecFileService().get_reference_file_data(file_model.name)
+        file_data_model = ReferenceFileService().get_reference_file_data(file_model.name)
     else:
         file_data_model = FileService.get_file_data(file_model.id)
     return File.from_models(file_model, file_data_model,
@@ -54,7 +55,7 @@ def get_spec_files(workflow_spec_id, include_libraries=False):
 
 
 def get_reference_files():
-    results = SpecFileService.get_reference_files()
+    results = ReferenceFileService.get_reference_files()
     files = (to_file_api(model) for model in results)
     return FileSchema(many=True).dump(files)
 
@@ -100,7 +101,7 @@ def add_spec_file(workflow_spec_id):
 def get_reference_file(name):
     file_extension = FileService.get_extension(name)
     content_type = CONTENT_TYPES[file_extension]
-    file_data = SpecFileService.get_reference_file_data(name)
+    file_data = ReferenceFileService.get_reference_file_data(name)
     return send_file(
         io.BytesIO(file_data.data),
         attachment_filename=name,
@@ -129,14 +130,14 @@ def update_reference_file_data(name):
         raise ApiError(code='file_does_not_exist',
                        message=f"The reference file {name} does not exist.")
     else:
-        SpecFileService().update_reference_file(file_model, file.stream.read())
+        ReferenceFileService().update_reference_file(file_model, file.stream.read())
 
     return FileSchema().dump(to_file_api(file_model))
 
 
 def add_reference_file():
     file = connexion.request.files['file']
-    file_model = SpecFileService.add_reference_file(name=file.filename,
+    file_model = ReferenceFileService.add_reference_file(name=file.filename,
                                                     content_type=file.content_type,
                                                     binary_data=file.stream.read())
     return FileSchema().dump(to_file_api(file_model))
@@ -216,7 +217,7 @@ def get_file_data_link(file_id, auth_token, version=None):
     if file_model.workflow_spec_id is not None:
         file_data = SpecFileService().get_spec_file_data(file_id)
     elif file_model.is_reference:
-        file_data = SpecFileService().get_reference_file_data(file_id)
+        file_data = ReferenceFileService().get_reference_file_data(file_id)
     else:
         file_data = FileService.get_file_data(file_id, version)
     if file_data is None:
@@ -292,6 +293,7 @@ def delete_spec_file(file_id):
     pass
 
 
+# TODO: Finish this
 def delete_reference_file(file_id):
     pass
 

crc/services/lookup_service.py

@@ -18,6 +18,7 @@ from crc.models.ldap import LdapSchema
 from crc.models.workflow import WorkflowModel, WorkflowSpecDependencyFile
 from crc.services.file_service import FileService
 from crc.services.spec_file_service import SpecFileService
+from crc.services.reference_file_service import ReferenceFileService
 from crc.services.ldap_service import LdapService
 from crc.services.workflow_processor import WorkflowProcessor
 
@@ -52,7 +53,7 @@ class LookupService(object):
 
     @staticmethod
     def get_lookup_model_for_file_data(file_id, file_name, value_column, label_column):
-        file_data = SpecFileService.get_reference_file_data(file_name)
+        file_data = ReferenceFileService.get_reference_file_data(file_name)
         lookup_model = db.session.query(LookupFileModel).filter(LookupFileModel.file_model_id == file_id).first()
         if not lookup_model:
             logging.warning("!!!! Making a very expensive call to update the lookup model.")

crc/services/reference_file_service.py

@@ -0,0 +1,93 @@
+import datetime
+import hashlib
+import os
+
+from crc import session
+from crc.api.common import ApiError
+from crc.models.file import FileModel, FileDataModel
+from crc.services.file_service import FileService, FileType
+from crc.services.spec_file_service import SpecFileService
+
+from uuid import UUID
+
+
+class ReferenceFileService(object):
+
+    @staticmethod
+    def add_reference_file(name, content_type, binary_data):
+        """Create a file with the given name, but not associated with a spec or workflow.
+           Only one file with the given reference name can exist."""
+        file_model = session.query(FileModel). \
+            filter(FileModel.is_reference == True). \
+            filter(FileModel.name == name).first()
+        if not file_model:
+            file_extension = FileService.get_extension(name)
+            file_type = FileType[file_extension].value
+
+            file_model = FileModel(
+                name=name,
+                is_reference=True,
+                type=file_type,
+                content_type=content_type
+            )
+            session.add(file_model)
+            session.commit()
+        else:
+            raise ApiError(code='file_already_exists',
+                           message=f"The reference file {name} already exists.")
+        return ReferenceFileService().update_reference_file(file_model, binary_data)
+
+    def update_reference_file(self, file_model, binary_data):
+        self.write_reference_file_to_system(file_model, binary_data)
+        print('update_reference_file')
+        return file_model
+
+    @staticmethod
+    def get_reference_file_data(file_name):
+        file_model = session.query(FileModel).filter(FileModel.name == file_name).filter(
+            FileModel.is_reference == True).first()
+        if file_model is not None:
+            sync_file_root = SpecFileService().get_sync_file_root()
+            file_path = os.path.join(sync_file_root, 'Reference', file_name)
+            if os.path.exists(file_path):
+                mtime = os.path.getmtime(file_path)
+                with open(file_path, 'rb') as f_open:
+                    reference_file_data = f_open.read()
+                    size = len(reference_file_data)
+                    md5_checksum = UUID(hashlib.md5(reference_file_data).hexdigest())
+
+                    reference_file_data_model = FileDataModel(data=reference_file_data,
+                                                              md5_hash=md5_checksum,
+                                                              size=size,
+                                                              date_created=datetime.datetime.fromtimestamp(mtime),
+                                                              file_model_id=file_model.id
+                                                              )
+                    return reference_file_data_model
+        else:
+            raise ApiError("file_not_found", "There is no reference file with the name '%s'" % file_name)
+
+    def write_reference_file_to_system(self, file_model, file_data):
+        file_path = self.write_reference_file_data_to_system(file_model, file_data)
+        self.write_reference_file_info_to_system(file_path, file_model)
+
+    @staticmethod
+    def write_reference_file_data_to_system(file_model, file_data):
+        category_name = 'Reference'
+        sync_file_root = SpecFileService.get_sync_file_root()
+        file_path = os.path.join(sync_file_root,
+                                 category_name,
+                                 file_model.name)
+        SpecFileService.write_file_data_to_system(file_path, file_data)
+        return file_path
+
+    @staticmethod
+    def write_reference_file_info_to_system(file_path, file_model):
+        SpecFileService.write_file_info_to_system(file_path, file_model)
+
+    @staticmethod
+    def get_reference_files():
+        reference_files = session.query(FileModel). \
+            filter_by(is_reference=True). \
+            filter(FileModel.archived == False). \
+            all()
+        return reference_files

crc/services/spec_file_service.py

@@ -317,83 +317,3 @@ class SpecFileService(object):
             return SpecFileService.find_spec_model_in_db(workflow.outer_workflow)
 
         return workflow_model
-
-    #
-    # Reference File Methods
-    #
-
-    @staticmethod
-    def add_reference_file(name, content_type, binary_data):
-        """Create a file with the given name, but not associated with a spec or workflow.
-           Only one file with the given reference name can exist."""
-        file_model = session.query(FileModel). \
-            filter(FileModel.is_reference == True). \
-            filter(FileModel.name == name).first()
-        if not file_model:
-            file_extension = FileService.get_extension(name)
-            file_type = FileType[file_extension].value
-
-            file_model = FileModel(
-                name=name,
-                is_reference=True,
-                type=file_type,
-                content_type=content_type
-            )
-            session.add(file_model)
-            session.commit()
-        else:
-            raise ApiError(code='file_already_exists',
-                           message=f"The reference file {name} already exists.")
-        return SpecFileService().update_reference_file(file_model, binary_data)
-
-    def update_reference_file(self, file_model, binary_data):
-        self.write_reference_file_to_system(file_model, binary_data)
-        print('update_reference_file')
-        return file_model
-
-    @staticmethod
-    def get_reference_file_data(file_name):
-        file_model = session.query(FileModel).filter(FileModel.name==file_name).filter(FileModel.is_reference==True).first()
-        if file_model is not None:
-            sync_file_root = SpecFileService().get_sync_file_root()
-            file_path = os.path.join(sync_file_root, 'Reference', file_name)
-            if os.path.exists(file_path):
-                mtime = os.path.getmtime(file_path)
-                with open(file_path, 'rb') as f_open:
-                    reference_file_data = f_open.read()
-                    size = len(reference_file_data)
-                    md5_checksum = UUID(hashlib.md5(reference_file_data).hexdigest())
-
-                    reference_file_data_model = FileDataModel(data=reference_file_data,
-                                                              md5_hash=md5_checksum,
-                                                              size=size,
-                                                              date_created=datetime.datetime.fromtimestamp(mtime),
-                                                              file_model_id=file_model.id
-                                                              )
-                    return reference_file_data_model
-        else:
-            raise ApiError("file_not_found", "There is no reference file with the name '%s'" % file_name)
-
-    def write_reference_file_to_system(self, file_model, file_data):
-        file_path = self.write_reference_file_data_to_system(file_model, file_data)
-        self.write_reference_file_info_to_system(file_path, file_model)
-
-    def write_reference_file_data_to_system(self, file_model, file_data):
-        category_name = 'Reference'
-        sync_file_root = self.get_sync_file_root()
-        file_path = os.path.join(sync_file_root,
-                                 category_name,
-                                 file_model.name)
-        self.write_file_data_to_system(file_path, file_data)
-        return file_path
-
-    def write_reference_file_info_to_system(self, file_path, file_model):
-        self.write_file_info_to_system(file_path, file_model)
-
-    @staticmethod
-    def get_reference_files():
-        reference_files = session.query(FileModel).\
-            filter_by(is_reference=True).\
-            filter(FileModel.archived == False).\
-            all()
-        return reference_files

example_data.py

@@ -9,6 +9,7 @@ from crc.models.user import UserModel
 from crc.models.workflow import WorkflowSpecModel, WorkflowSpecCategoryModel
 from crc.services.document_service import DocumentService
 from crc.services.file_service import FileService
+from crc.services.reference_file_service import ReferenceFileService
 from crc.services.spec_file_service import SpecFileService
 from crc.services.study_service import StudyService
 
@@ -188,7 +189,7 @@ class ExampleDataLoader:
     def load_rrt(self):
         file_path = os.path.join(app.root_path, 'static', 'reference', 'rrt_documents.xlsx')
         file = open(file_path, "rb")
-        SpecFileService.add_reference_file(FileService.DOCUMENT_LIST,
+        ReferenceFileService.add_reference_file(FileService.DOCUMENT_LIST,
                                        binary_data=file.read(),
                                        content_type=CONTENT_TYPES['xls'])
         file.close()
@@ -290,14 +291,14 @@ class ExampleDataLoader:
     def load_reference_documents(self):
         file_path = os.path.join(app.root_path, 'static', 'reference', 'irb_documents.xlsx')
         file = open(file_path, "rb")
-        SpecFileService.add_reference_file(DocumentService.DOCUMENT_LIST,
+        ReferenceFileService.add_reference_file(DocumentService.DOCUMENT_LIST,
                                        binary_data=file.read(),
                                        content_type=CONTENT_TYPES['xlsx'])
         file.close()
 
         file_path = os.path.join(app.root_path, 'static', 'reference', 'investigators.xlsx')
         file = open(file_path, "rb")
-        SpecFileService.add_reference_file(StudyService.INVESTIGATOR_LIST,
+        ReferenceFileService.add_reference_file(StudyService.INVESTIGATOR_LIST,
                                        binary_data=file.read(),
                                        content_type=CONTENT_TYPES['xlsx'])
         file.close()

tests/base_test.py

@@ -22,6 +22,7 @@ from crc.models.user import UserModel
 from crc.models.workflow import WorkflowSpecModel, WorkflowSpecCategoryModel
 from crc.services.ldap_service import LdapService
 from crc.services.file_service import FileService
+from crc.services.reference_file_service import ReferenceFileService
 from crc.services.spec_file_service import SpecFileService
 from crc.services.study_service import StudyService
 from crc.services.user_service import UserService
@@ -301,7 +302,7 @@ class BaseTest(unittest.TestCase):
     def create_reference_document(self):
         file_path = os.path.join(app.root_path, 'static', 'reference', 'irb_documents.xlsx')
         with open(file_path, "rb") as file:
-            SpecFileService.add_reference_file(DocumentService.DOCUMENT_LIST,
+            ReferenceFileService.add_reference_file(DocumentService.DOCUMENT_LIST,
                                                content_type=CONTENT_TYPES['xlsx'],
                                                binary_data=file.read())