Merge branch 'feature/spec_files_wthout_db' of https://github.com/sartography/cr-connect-workflow into feature/spec_files_wthout_db

crc/__init__.py

@@ -55,10 +55,8 @@ from crc import models
 from crc import api
 from crc.api import admin
 from crc.services.workflow_service import WorkflowService
-from crc.services.workflow_spec_service import WorkflowSpecService
 connexion_app.add_api('api.yml', base_path='/v1.0')
 
-
 # needed function to avoid circular import
 def process_waiting_tasks():
     with app.app_context():

crc/api/spec_file.py

@@ -8,12 +8,15 @@ from flask import send_file
 import io
 import connexion
 
+from crc.services.workflow_spec_service import WorkflowSpecService
+
+workflow_spec_service = WorkflowSpecService()
 
 def get_files(spec_id, include_libraries=False):
     if spec_id is None:
         raise ApiError(code='missing_spec_id',
                        message='Please specify the workflow_spec_id.')
-    workflow_spec = session.query(WorkflowSpecModel).filter_by(id=spec_id).first()
+    workflow_spec = workflow_spec_service.get_spec(spec_id)
     if workflow_spec is None:
         raise ApiError(code='unknown_spec',
                        message=f'Unknown Spec: {spec_id}')
@@ -23,7 +26,7 @@ def get_files(spec_id, include_libraries=False):
 
 
 def get_file(spec_id, file_name):
-    workflow_spec = session.query(WorkflowSpecModel).filter_by(id=spec_id).first()
+    workflow_spec = workflow_spec_service.get_spec(spec_id)
     files = SpecFileService.get_files(workflow_spec, file_name)
     if len(files) == 0:
         raise ApiError(code='unknown file',
@@ -33,16 +36,16 @@ def get_file(spec_id, file_name):
 
 
 def add_file(spec_id):
-    workflow_spec = session.query(WorkflowSpecModel).filter_by(id=spec_id).first()
+    workflow_spec = workflow_spec_service.get_spec(spec_id)
     file = connexion.request.files['file']
     file = SpecFileService.add_file(workflow_spec, file.filename, file.stream.read())
-    if not WorkflowSpecModel.primary_process_id and file.type == FileType.bpmn:
+    if not workflow_spec.primary_process_name and file.type == FileType.bpmn:
         SpecFileService.set_primary_bpmn(workflow_spec, file.name)
     return FileSchema().dump(file)
 
 
 def update(spec_id, file_name, is_primary):
-    workflow_spec = session.query(WorkflowSpecModel).filter_by(id=spec_id).first()
+    workflow_spec = workflow_spec_service.get_spec(spec_id)
     files = SpecFileService.get_files(workflow_spec, file_name)
     if len(files) < 1:
         raise ApiError(code='unknown file',
@@ -55,7 +58,7 @@ def update(spec_id, file_name, is_primary):
 
 
 def update_data(spec_id, file_name):
-    workflow_spec_model = session.query(WorkflowSpecModel).filter_by(id=spec_id).first()
+    workflow_spec_model = workflow_spec_service.get_spec(spec_id)
     if workflow_spec_model is None:
         raise ApiError(code='missing_spec',
                        message=f'The workflow spec for id {spec_id} does not exist.')
@@ -65,10 +68,10 @@ def update_data(spec_id, file_name):
 
 
 def get_data(spec_id, file_name):
-    workflow_spec_model = session.query(WorkflowSpecModel).filter_by(id=spec_id).first()
+    workflow_spec = workflow_spec_service.get_spec(spec_id)
-    file_data = SpecFileService.get_data(workflow_spec_model, file_name)
+    file_data = SpecFileService.get_data(workflow_spec, file_name)
     if file_data is not None:
-        file_info = SpecFileService.get_files(workflow_spec_model, file_name)[0]
+        file_info = SpecFileService.get_files(workflow_spec, file_name)[0]
         return send_file(
             io.BytesIO(file_data),
             attachment_filename=file_name,
@@ -82,5 +85,5 @@ def get_data(spec_id, file_name):
 
 
 def delete(spec_id, file_name):
-    workflow_spec_model = session.query(WorkflowSpecModel).filter_by(id=spec_id).first()
+    workflow_spec = workflow_spec_service.get_spec(spec_id)
-    SpecFileService.delete_file(workflow_spec_model, file_name)
+    SpecFileService.delete_file(workflow_spec, file_name)

crc/api/workflow.py

@@ -2,7 +2,7 @@ import uuid
 
 from flask import g
 
-from crc import session, WorkflowSpecService
+from crc import session
 from crc.api.common import ApiError, ApiErrorSchema
 from crc.models.api_models import WorkflowApiSchema
 from crc.models.study import StudyModel, WorkflowMetadata, StudyStatus
@@ -18,46 +18,45 @@ from crc.services.task_logging_service import TaskLoggingService
 from crc.services.user_service import UserService
 from crc.services.workflow_processor import WorkflowProcessor
 from crc.services.workflow_service import WorkflowService
+from crc.services.workflow_spec_service import WorkflowSpecService
 
+workflow_spec_service = WorkflowSpecService()
 
 def all_specifications(libraries=False,standalone=False):
     if libraries and standalone:
         raise ApiError('inconceivable!', 'You should specify libraries or standalone, but not both')
-    schema = WorkflowSpecModelSchema(many=True)
+
     if libraries:
-        return schema.dump(session.query(WorkflowSpecModel)\
+        return workflow_spec_service.get_libraries()
-              .filter(WorkflowSpecModel.library==True).all())
 
     if standalone:
-        return schema.dump(session.query(WorkflowSpecModel)\
+        return workflow_spec_service.get_standalones()
-              .filter(WorkflowSpecModel.standalone==True).all())
 
     # return standard workflows (not library, not standalone)
-    return schema.dump(session.query(WorkflowSpecModel)
+    categories = workflow_spec_service.get_categories()
-                       .filter((WorkflowSpecModel.library==False) | (
+    workflows = []
-                                WorkflowSpecModel.library==None))
+    for cat in categories:
-                       .filter((WorkflowSpecModel.standalone==False) | (
+        workflows.extend(cat.workflows)
-                                WorkflowSpecModel.standalone==None))
+    return workflows
-                       .all())
 
 
 def add_workflow_specification(body):
-    category_id = body['display_name']
+    category_name = body['display_name']
     # TODO: set this spec to the last one in the spec list
-    # WorkflowService.cleanup_workflow_spec_display_order(category_id)
+    WorkflowService.cleanup_workflow_spec_display_order(category_name)
 
     # Libraries and standalone workflows don't get a category_id
     if body['library'] is True or body['standalone'] is True:
         body['category_id'] = None
 
-    new_spec = WorkflowSpecService.add_spec(body)
+    new_spec = workflow_spec_service.add_spec(body)
     return new_spec
 
 
 def get_workflow_specification(spec_id):
     if spec_id is None:
         raise ApiError('unknown_spec', 'Please provide a valid Workflow Specification ID.')
-    spec = WorkflowSpecService.get_spec(spec_id)
+    spec = workflow_spec_service.get_spec(spec_id)
 
     if spec is None:
         raise ApiError('unknown_spec', 'The Workflow Specification "' + spec_id + '" is not recognized.')
@@ -70,8 +69,8 @@ def validate_spec_and_library(spec_id,library_id):
     if library_id is None:
         raise ApiError('unknown_spec', 'Please provide a valid Library Specification ID.')
 
-    spec = WorkflowSpecService.get_spec(spec_id)
+    spec = workflow_spec_service.get_spec(spec_id)
-    library = WorkflowSpecService.get_library(library_id);
+    library = workflow_spec_service.get_library(library_id);
     if spec is None:
         raise ApiError('unknown_spec', 'The Workflow Specification "' + spec_id + '" is not recognized.')
     if library is None:
@@ -85,17 +84,25 @@ def add_workflow_spec_library(spec_id, library_id):
     libraries: workflow_spec_service.get_libraries()
     libraryids = [x.library_spec_id for x in libraries]
     if library_id in libraryids:
-        raise ApiError('unknown_spec', 'The Library Specification "' + spec_id + '" is already attached.')
+        raise ApiError('unknown_spec', 'The Library Specification "' + library_id + '" is already attached.')
 
-    # TODO: this used to return all libraries. maybe we still want that here
+    spec = workflow_spec_service.get_spec(spec_id)
-    newlib = WorkflowSpecService.add_library(spec_id, library_id)
+    library = workflow_spec_service.get_spec(library_id)
-    return newlib
+    spec.libraries.push(library)
+    workflow_spec_service.update_spec(spec_id)
+    return spec
 
-def drop_workflow_spec_library(spec_id,library_id):
+
+def drop_workflow_spec_library(spec_id, library_id):
     validate_spec_and_library(spec_id, library_id)
-    WorkflowSpecService.delete_library(library_id)
+    spec = workflow_spec_service.get_spec(spec_id)
-    # TODO: this used to return all libraries. maybe we still want that here
+
-    return WorkflowSpecService.get_libraries()
+    # heres a piece of code that certainly wont work
+    library = workflow_spec_service.get_spec(library_id)
+    if library in spec.libraries:
+        spec.libraries.pop(library)
+    workflow_spec_service.update_spec(spec_id)
+    return spec
 
 
 def validate_workflow_specification(spec_id, study_id=None, test_until=None):
@@ -113,7 +120,7 @@ def validate_workflow_specification(spec_id, study_id=None, test_until=None):
 def update_workflow_specification(spec_id, body):
     if spec_id is None:
         raise ApiError('unknown_spec', 'Please provide a valid Workflow Spec ID.')
-    spec = WorkflowSpecService.get_spec(spec_id)
+    spec = workflow_spec_service.get_spec(spec_id)
 
     if spec is None:
         raise ApiError('unknown_study', 'The spec "' + spec_id + '" is not recognized.')
@@ -126,7 +133,7 @@ def update_workflow_specification(spec_id, body):
     if body['library'] is True or body['standalone'] is True:
         body['category_id'] = None
 
-    spec = WorkflowSpecService.update_spec(spec_id, body)
+    spec = workflow_spec_service.update_spec(spec_id, body)
     return spec
 
 
@@ -134,7 +141,7 @@ def delete_workflow_specification(spec_id):
     if spec_id is None:
         raise ApiError('unknown_spec', 'Please provide a valid Workflow Specification ID.')
 
-    spec = WorkflowSpecService.get_spec(spec_id)
+    spec = workflow_spec_service.get_spec(spec_id)
 
     if spec is None:
         raise ApiError('unknown_spec', 'The Workflow Specification "' + spec_id + '" is not recognized.')
@@ -150,9 +157,8 @@ def delete_workflow_specification(spec_id):
     WorkflowService.delete_workflow_spec_task_events(spec_id)
 
     # .delete() doesn't work when we need a cascade. Must grab the record, and explicitly delete
-    WorkflowSpecService.delete_spec(spec_id)
+    workflow_spec_service.delete_spec(spec_id)
 
-    # TODO: fix reordering
     # Reorder the remaining specs
     WorkflowService.cleanup_workflow_spec_display_order(spec.category_id)
 
@@ -161,9 +167,9 @@ def reorder_workflow_specification(spec_id, direction):
     if direction not in ('up', 'down'):
         raise ApiError(code='bad_direction',
                        message='The direction must be `up` or `down`.')
-    spec = WorkflowSpecService.get_spec(spec_id)
+    spec = workflow_spec_service.get_spec(spec_id)
     if spec:
-        ordered_specs = WorkflowSpecService.reorder_spec(spec, direction)
+        ordered_specs = workflow_spec_service.reorder_spec(spec, direction)
     else:
         raise ApiError(code='bad_spec_id',
                        message=f'The spec_id {spec_id} did not return a specification. Please check that it is valid.')
@@ -318,22 +324,22 @@ def __update_task(processor, task, data, user):
 
 
 def list_workflow_spec_categories():
-    return WorkflowSpecService.get_workflow_categories()
+    return workflow_spec_service.get_workflow_categories()
 
 
 def get_workflow_spec_category(cat_id):
-    return WorkflowSpecService.get_workflow_category(cat_id)
+    return workflow_spec_service.get_workflow_category(cat_id)
 
 
 def add_workflow_spec_category(body):
-    return WorkflowSpecService.add_category(body)
+    return workflow_spec_service.add_category(body)
 
 
 def update_workflow_spec_category(cat_id, body):
     if cat_id is None:
         raise ApiError('unknown_category', 'Please provide a valid Workflow Spec Category ID.')
 
-    category = WorkflowSpecService.update_category(cat_id, body)
+    category = workflow_spec_service.update_category(cat_id, body)
 
     if category is None:
         raise ApiError('unknown_category', 'The category "' + cat_id + '" is not recognized.')
@@ -348,19 +354,16 @@ def update_workflow_spec_category(cat_id, body):
 def delete_workflow_spec_category(cat_id):
     workflow_spec_service.delete_category(cat_id)
 
+
 def reorder_workflow_spec_category(cat_id, direction):
-    # TODO: fix reordering here too
     if direction not in ('up', 'down'):
         raise ApiError(code='bad_direction',
                        message='The direction must be `up` or `down`.')
     WorkflowService.cleanup_workflow_spec_category_display_order()
-    # TODO: modify this to get_workflow_category(category)
+    category = workflow_spec_service.get_category(cat_id)
-    category = session.query(WorkflowSpecCategoryModel).\
-        filter(WorkflowSpecCategoryModel.id == cat_id).first()
     if category:
-        ordered_categories = WorkflowService.reorder_workflow_spec_category(category, direction)
+        ordered_categories = workflow_spec_service.reorder_workflow_spec_category(category, direction)
-        schema = WorkflowSpecCategoryModelSchema(many=True)
+        return ordered_categories
-        return schema.dump(ordered_categories)
     else:
         raise ApiError(code='bad_category_id',
                        message=f'The category id {cat_id} did not return a Workflow Spec Category. Make sure it is a valid ID.')

crc/models/task_event.py

@@ -14,7 +14,7 @@ class TaskEventModel(db.Model):
     study_id = db.Column(db.Integer, db.ForeignKey('study.id'))
     user_uid = db.Column(db.String, nullable=False) # In some cases the unique user id may not exist in the db yet.
     workflow_id = db.Column(db.Integer, db.ForeignKey('workflow.id'), nullable=False)
-    workflow_spec_id = db.Column(db.String, db.ForeignKey('workflow_spec.id'))
+    workflow_spec_id = db.Column(db.String)
     spec_version = db.Column(db.String)
     action = db.Column(db.String)
     task_id = db.Column(db.String)

crc/models/workflow.py

@@ -1,6 +1,6 @@
 import enum
 
-from marshmallow import EXCLUDE
+from marshmallow import EXCLUDE, post_load
 from sqlalchemy import func
 
 from crc import db, ma
@@ -19,12 +19,15 @@ class WorkflowSpecCategorySchema(ma.Schema):
         model = WorkflowSpecCategory
         fields = ["id", "display_name", "display_order", "admin"]
 
+    @post_load
+    def make_cat(self, data, **kwargs):
+        return WorkflowSpecCategory(**data)
+
 
 class WorkflowSpecInfo(object):
-    def __init__(self, id, display_name, description, category_name,
+    def __init__(self, id, display_name, description,  is_master_spec,
-                 display_order, is_master_spec,
+                 standalone, library, primary_file_name, primary_process_id,
-                 standalone, library, primary_file_name, primary_process_id, is_review,
+                 libraries, category_name=None, display_order=0, is_review=False):
-                 libraries):
         self.id = id  # Sting unique id
         self.display_name = display_name
         self.description = description
@@ -45,9 +48,12 @@ class WorkflowSpecInfoSchema(ma.Schema):
         model = WorkflowSpecInfo
         fields = ["id", "display_name", "description", "category_id", "is_master_spec,",
                   "standalone", "library", "primary_file_name", "primary_process_id", "is_review",
-                  "libraries"]
+                  "libraries", "category_name", "display_order", "is_master_spec", "is_review"]
         unknown = EXCLUDE
 
+    @post_load
+    def make_spec(self, data, **kwargs):
+        return WorkflowSpecInfo(**data)
 
 class WorkflowState(enum.Enum):
     hidden = "hidden"
@@ -79,7 +85,7 @@ class WorkflowModel(db.Model):
     status = db.Column(db.Enum(WorkflowStatus))
     study_id = db.Column(db.Integer, db.ForeignKey('study.id'))
     study = db.relationship("StudyModel", backref='workflow')
-    workflow_spec_id = db.Column(db.String, db.ForeignKey('workflow_spec.id'))
+    workflow_spec_id = db.Column(db.String)
     total_tasks = db.Column(db.Integer, default=0)
     completed_tasks = db.Column(db.Integer, default=0)
     last_updated = db.Column(db.DateTime(timezone=True), server_default=func.now())

crc/services/workflow_spec_service.py

@@ -29,7 +29,6 @@ class WorkflowSpecService(FileSystemService):
         self.master_spec = None
         self.libraries = {}
         self.standalone = {}
-        self.scan_file_system()
 
     def add_spec(self, spec: WorkflowSpecInfo):
         self.update_spec(spec)
@@ -54,6 +53,9 @@ class WorkflowSpecService(FileSystemService):
             raise ApiError('unknown spec', 'unable to find a spec with id:' + spec_id)
         return self.specs[spec_id]
 
+    def get_specs(self):
+        return list(self.specs.values())
+
     def reorder_spec(self, spec:WorkflowSpecInfo, direction):
         workflows = spec.category.workflows
         workflows.sort(key=lambda w: w.display_order)
@@ -70,23 +72,45 @@ class WorkflowSpecService(FileSystemService):
         return workflows
 
     def get_libraries(self) -> List[WorkflowSpecInfo]:
-        # fixme
+        spec_list = list(self.libraries.values())
+        spec_list.sort(key=lambda w: w.display_order)
+        return spec_list
+
+    def get_standalones(self) -> List[WorkflowSpecInfo]:
+        spec_list = list(self.standalone.values())
+        spec_list.sort(key=lambda w: w.display_order)
+        return spec_list
+
+    def get_standalones(self) -> List[WorkflowSpecInfo]:
         pass
 
     def get_categories(self) -> List[WorkflowSpecCategory]:
-        pass
+        """Returns the categories as a list in display order"""
+        cat_list = list(self.categories.values())
+        cat_list.sort(key=lambda w: w.display_order)
+        return cat_list
 
     def get_category(self, category_id) -> WorkflowSpecCategory:
-        pass
+        if category_id not in self.categories:
+            raise ApiError('unknown category', 'unable to find a category with id:' + category_id)
+        return self.categories[category_id]
 
     def add_category(self, category: WorkflowSpecCategory):
-        pass
+        self.update_category(category)
 
     def update_category(self, category: WorkflowSpecCategory):
-        pass
+        cat_path = self.category_path(category.display_name)
+        os.makedirs(os.path.dirname(cat_path), exist_ok=True)
+        json_path = os.path.join(cat_path, self.CAT_JSON_FILE)
+        with open(json_path, "w") as cat_json:
+            json.dump(self.CAT_SCHEMA.dump(category), cat_json, indent=4)
+        self.scan_file_system()
 
-    def delete_category(self, category: WorkflowSpecCategory):
+    def delete_category(self, category_id: str):
-        pass
+        if category_id in self.specs:
+            path = self.category_path(category_id)
+            shutil.rmtree(path)
+            self.scan_file_system()
 
     def reorder_workflow_spec_category(self, spec:WorkflowSpecInfo, direction):
         # Fixme:  Resort Workflow categories
@@ -101,18 +125,20 @@ class WorkflowSpecService(FileSystemService):
         self.master_spec = None
         self.libraries = {}
         if not os.path.exists(FileSystemService.root_path()):
-            raise ApiError('missing_specs', 'The path for workflow specifications does not exist.')
+            return # Nothing to scan yet.  There are no files.
         directory_items = os.scandir(FileSystemService.root_path())
         for item in directory_items:
             if item.is_dir():
                 if item.name == self.LIBRARY_SPECS:
                     self.scan_category(item, is_library=True)
+                if item.name == self.STAND_ALONE_SPECS:
+                    self.scan_category(item, is_standalone=True)
                 elif item.name == self.MASTER_SPECIFICATION:
                     self.scan_spec(item, is_master=True)
                 else:
                     self.scan_category(item)
 
-    def scan_category(self, dir_item: os.DirEntry, is_library=False):
+    def scan_category(self, dir_item: os.DirEntry, is_library=False, is_standalone=False):
         """Reads the category.json file, and any workflow directories """
         cat_path = os.path.join(dir_item.path, self.CAT_JSON_FILE)
         if os.path.exists(cat_path):
@@ -125,6 +151,8 @@ class WorkflowSpecService(FileSystemService):
                 json.dump(self.CAT_SCHEMA.dump(cat), wf_json, indent=4)
         if is_library:
             self.libraries = cat
+        elif is_standalone:
+            self.standalone = cat
         else:
             self.categories[cat.id] = cat
         workflow_dirs = os.scandir(FileSystemService.root_path())
@@ -141,7 +169,6 @@ class WorkflowSpecService(FileSystemService):
 #            workflow_metas.append(WorkflowMetadata.from_workflow(workflow))
         return workflow_metas
 
-
     def scan_spec(self, dir_item: os.DirEntry, is_master=False, category=None):
         if not is_master and not category:
             raise ApiError("invalid_spec_dir", "Please specify what category this workflow belongs to.")
@@ -153,15 +180,16 @@ class WorkflowSpecService(FileSystemService):
         else:
             spec = WorkflowSpecInfo(id=dir_item.name, library=False, standalone=False, is_master_spec=is_master,
                                     display_name=dir_item.name, description="", primary_process_id="",
-                                    primary_file_name="")
+                                    primary_file_name="", category_name="", display_order=0, is_review=False,
+                                    libraries=[])
             with open(spec_path, "w") as wf_json:
-                json.dump(WorkflowSpecInfoSchema.dump(spec), wf_json, indent=4)
+                json.dump(self.WF_SCHEMA.dump(spec), wf_json, indent=4)
         if is_master:
             self.master_spec = spec
         elif category:
             self.specs[spec.id] = spec
             spec.category = category
-            category.workflow_specs.append(spec)
+            category.workflows.append(spec)
 
     def set_primary_bpmn(self, workflow_spec: WorkflowSpecInfo, file_name: str, binary_data=None):
         # If this is a BPMN, extract the process id, and determine if it is contains swim lanes.

tests/base_test.py

@@ -12,7 +12,7 @@ import datetime
 import shutil
 from flask import g
 
-from crc import app, db, session, WorkflowSpecService
+from crc import app, db, session
 from crc.models.api_models import WorkflowApiSchema, MultiInstanceType
 from crc.models.file import FileModel, CONTENT_TYPES
 from crc.models.task_event import TaskEventModel
@@ -26,7 +26,7 @@ from crc.services.user_service import UserService
 from crc.services.workflow_service import WorkflowService
 from crc.services.document_service import DocumentService
 from example_data import ExampleDataLoader
-from crc.services.user_file_service import UserFileService
+from crc.services.workflow_spec_service import WorkflowSpecService
 
 # UNCOMMENT THIS FOR DEBUGGING SQL ALCHEMY QUERIES
 import logging

tests/data/IMPORT_TEST/Category Number Two/category.json

@@ -1,5 +1,6 @@
 {
   "id": 2,
   "admin": false,
-  "display_order": 1
+  "display_order": 1,
+  "display_name": "Category Number Two"
 }

tests/workflow/cr_connect.log

@@ -1,3 +1,5 @@
 2022-01-26 15:05:47,955 - connexion.apis.abstract - ERROR - Failed to add operation for GET /v1.0/workflow_sync/pullall
 2022-01-26 15:06:36,303 - connexion.apis.abstract - ERROR - Failed to add operation for GET /v1.0/workflow_sync/pullall
 2022-01-26 15:08:01,978 - connexion.apis.abstract - ERROR - Failed to add operation for GET /v1.0/file/{md5_hash}/hash_data
+2022-02-07 12:46:21,875 - connexion.apis.abstract - ERROR - Failed to add operation for GET /v1.0/ldap
+2022-02-07 12:49:13,793 - connexion.apis.abstract - ERROR - Failed to add operation for GET /v1.0/ldap

tests/workflow/test_workflow_spec_service.py

@@ -4,10 +4,10 @@ import shutil
 
 from tests.base_test import BaseTest
 
-from crc.services.workflow_sync import WorkflowSyncService
 from crc.models.workflow import WorkflowSpecInfo, WorkflowSpecCategory
 from crc.services.file_system_service import FileSystemService
 from crc.services.spec_file_service import SpecFileService
+from crc.services.workflow_spec_service import WorkflowSpecService
 from crc import db, app
 
 
@@ -15,13 +15,14 @@ class TestWorkflowSync(BaseTest):
 
     spec_path = FileSystemService.root_path()
     import_spec_path = os.path.join(app.root_path, '..', 'tests', 'data', 'IMPORT_TEST')
+    service = WorkflowSpecService()
 
-    def set_up_file_system(self):
+    def copy_files_to_file_system(self):
-        """Some tests rely on a well populated file system and an empty database to start"""
+        """Some tests rely on a well populated file system """
         shutil.copytree(self.import_spec_path, self.spec_path)
 
-    def set_up_database(self):
+    def build_file_system_from_models(self):
-        """Some tests rely on a well populated database and an empty file system to start"""
+        """Some tests check to see what happens when we write data to an empty file system."""
 
         # Construct Two Categories, with one workflow in first category, two in the second.
         # assure that the data in categories.json is correct
@@ -35,77 +36,24 @@ class TestWorkflowSync(BaseTest):
         self.load_test_spec('email', category_id=c1.id, library=True)
 
     def test_from_file_system_blank_slate(self):
-        self.assertEquals(0, len(db.session.query(WorkflowSpecModel).all()))
+        self.assertEquals(0, len(self.service.get_categories()))
-        self.assertEquals(0, len(db.session.query(WorkflowSpecCategoryModel).all()))
+        self.assertEquals(0, len(self.service.get_specs()))
-        self.set_up_file_system()
+        self.copy_files_to_file_system()
-        WorkflowSyncService().from_file_system()
+        self.service.scan_file_system()
-        self.assertEquals(2, len(db.session.query(WorkflowSpecCategoryModel).all()))
+        self.assertEquals(2, len(self.service.get_categories()))
-        self.assertEquals(5, len(db.session.query(WorkflowSpecModel).all()))
+        self.assertEquals(5, len(self.service.get_specs()))
-        self.assertEquals(1, len(db.session.query(WorkflowSpecModel).filter(WorkflowSpecModel.category_id == 1).all()))
+        self.assertEquals(1, len(self.service.get_category('Category Number One').workflows))
-        self.assertEquals(2, len(db.session.query(WorkflowSpecModel).filter(WorkflowSpecModel.category_id == 2).all()))
+        self.assertEquals(2, len(self.service.get_category('Category Number Two').workflows))
-        self.assertEquals(1, len(db.session.query(WorkflowSpecModel).filter(WorkflowSpecModel.is_master_spec).all()))
+        self.assertIsNotNone(self.service.master_spec)
-        self.assertEquals(1, len(db.session.query(WorkflowSpecModel).filter(WorkflowSpecModel.library).all()))
+        self.assertEquals(1, len(self.service.get_libraries()))
-        # The top level workflow, has a library
+        self.assertEquals(1, len(self.service.master_spec.libraries))
-        tlw = db.session.query(WorkflowSpecModel).filter(WorkflowSpecModel.is_master_spec).first()
-        self.assertEquals(1, len(tlw.libraries))
-
-    def test_repeated_imports(self):
-        self.set_up_file_system()
-        WorkflowSyncService().from_file_system()
-        WorkflowSyncService().from_file_system()
-        WorkflowSyncService().from_file_system()
-        self.assertEquals(2, len(db.session.query(WorkflowSpecCategoryModel).all()))
-        self.assertEquals(5, len(db.session.query(WorkflowSpecModel).all()))
 
     def test_delete_category_and_workflows(self):
-        self.set_up_file_system()
+        self.copy_files_to_file_system()
-        WorkflowSyncService().from_file_system()
         cat_path = SpecFileService().category_path('Category Number One')
-        shutil.rmtree(cat_path)
+        shutil.rmtree(cat_path) # Remove the path, as if from a git pull and the path was removed.
-        WorkflowSyncService().from_file_system()
+        self.assertEquals(3, len(self.service.get_categories()))
-        self.assertEquals(1, len(db.session.query(WorkflowSpecCategoryModel).all()))
+        self.assertEquals(4, len(self.service.get_specs()))
-        self.assertEquals(4, len(db.session.query(WorkflowSpecModel).all()))
+
-        self.assertEquals(0, len(db.session.query(WorkflowSpecModel).filter(WorkflowSpecModel.category_id == 1).all()))
-
-        WorkflowSyncService().to_file_system()
-        json_path = os.path.join(FileSystemService.root_path(), "categories.json")
-        with open(json_path) as json_file:
-            data = json.load(json_file)
-            self.assertEquals(1, len(data['categories']), "When the json file is written back to disk, there is only one category now.")
-
-    def test_to_file_system(self):
-        """Assure we get the basic paths on the file system that we would expect."""
-        self.assertFalse(os.path.exists(self.spec_path))
-        self.set_up_database()
-        self.assertEqual(4, len(os.listdir(self.spec_path)), "Adding workflows should create dir structure")
-        WorkflowSyncService().to_file_system()
-        self.assertEqual(5, len(os.listdir(self.spec_path)), "Sync service should create categories.json file")
-
-    def test_to_file_system_correct_categories(self):
-        """Assure we have two categories in the json file, and that these directories exist, and contain
-           workflow.json files for each workflow."""
-        self.set_up_database()
-        WorkflowSyncService().to_file_system()
-        json_path = os.path.join(self.spec_path, 'categories.json')
-
-        with open(json_path) as json_file:
-            data = json.load(json_file)
-        self.assertTrue('categories' in data)
-        self.assertEqual(2, len(data['categories']))
-        counter = 0
-        for c in data['categories']:
-            cat_path = os.path.join(self.spec_path, c['display_name'])
-            self.assertTrue(os.path.exists(cat_path), "The category directories exist.")
-            self.assertEqual(data['categories'][counter]['display_order'], counter, "Order is correct")
-            counter += 1
-            workflow_dirs = os.listdir(cat_path)
-            for wd in workflow_dirs:
-                wf_json_path = os.path.join(cat_path, wd, 'workflow.json')
-                self.assertTrue(os.path.exists(wf_json_path), "A workflow.json file should exist.")
-        # Fixme: Assure the master workflow spec, and Libraries are also exported to file system.
 
 
-    # Todo:
-    #  * What if category json files, and directories don't match?
-    #  * Test renaming a category
-    #  * Test moving a workflow to a different category