From 2a6578b785badf831cb691c865d1afed298ebb01 Mon Sep 17 00:00:00 2001 From: alicia pritchett Date: Mon, 7 Feb 2022 12:10:57 -0500 Subject: [PATCH 1/7] fixed workflow.py --- crc/api/workflow.py | 61 ++++++++++++++------------- crc/services/workflow_spec_service.py | 3 ++ 2 files changed, 34 insertions(+), 30 deletions(-) diff --git a/crc/api/workflow.py b/crc/api/workflow.py index 640c1cad..ad411c30 100644 --- a/crc/api/workflow.py +++ b/crc/api/workflow.py @@ -23,28 +23,25 @@ from crc.services.workflow_service import WorkflowService def all_specifications(libraries=False,standalone=False): if libraries and standalone: raise ApiError('inconceivable!', 'You should specify libraries or standalone, but not both') - schema = WorkflowSpecModelSchema(many=True) + if libraries: - return schema.dump(session.query(WorkflowSpecModel)\ - .filter(WorkflowSpecModel.library==True).all()) + return WorkflowSpecService.get_libraries() if standalone: - return schema.dump(session.query(WorkflowSpecModel)\ - .filter(WorkflowSpecModel.standalone==True).all()) + return WorkflowSpecService.get_standalones() # return standard workflows (not library, not standalone) - return schema.dump(session.query(WorkflowSpecModel) - .filter((WorkflowSpecModel.library==False) | ( - WorkflowSpecModel.library==None)) - .filter((WorkflowSpecModel.standalone==False) | ( - WorkflowSpecModel.standalone==None)) - .all()) + categories = WorkflowSpecService.get_categories() + workflows = [] + for cat in categories: + workflows.extend(cat.workflows) + return workflows def add_workflow_specification(body): - category_id = body['display_name'] + category_name = body['display_name'] # TODO: set this spec to the last one in the spec list - # WorkflowService.cleanup_workflow_spec_display_order(category_id) + WorkflowService.cleanup_workflow_spec_display_order(category_name) # Libraries and standalone workflows don't get a category_id if body['library'] is True or body['standalone'] is True: @@ -82,20 +79,28 @@ def validate_spec_and_library(spec_id,library_id): def add_workflow_spec_library(spec_id, library_id): validate_spec_and_library(spec_id, library_id) - libraries: workflow_spec_service.get_libraries() + libraries: WorkflowSpecService.get_libraries() libraryids = [x.library_spec_id for x in libraries] if library_id in libraryids: - raise ApiError('unknown_spec', 'The Library Specification "' + spec_id + '" is already attached.') + raise ApiError('unknown_spec', 'The Library Specification "' + library_id + '" is already attached.') - # TODO: this used to return all libraries. maybe we still want that here - newlib = WorkflowSpecService.add_library(spec_id, library_id) - return newlib + spec = WorkflowSpecService.get_spec(spec_id) + library = WorkflowSpecService.get_spec(library_id) + spec.libraries.push(library) + WorkflowSpecService.update_spec(spec_id) + return spec -def drop_workflow_spec_library(spec_id,library_id): + +def drop_workflow_spec_library(spec_id, library_id): validate_spec_and_library(spec_id, library_id) - WorkflowSpecService.delete_library(library_id) - # TODO: this used to return all libraries. maybe we still want that here - return WorkflowSpecService.get_libraries() + spec = WorkflowSpecService.get_spec(spec_id) + + # heres a piece of code that certainly wont work + library = WorkflowSpecService.get_spec(library_id) + if library in spec.libraries: + spec.libraries.pop(library) + WorkflowSpecService.update_spec(spec_id) + return spec def validate_workflow_specification(spec_id, study_id=None, test_until=None): @@ -152,7 +157,6 @@ def delete_workflow_specification(spec_id): # .delete() doesn't work when we need a cascade. Must grab the record, and explicitly delete WorkflowSpecService.delete_spec(spec_id) - # TODO: fix reordering # Reorder the remaining specs WorkflowService.cleanup_workflow_spec_display_order(spec.category_id) @@ -348,19 +352,16 @@ def update_workflow_spec_category(cat_id, body): def delete_workflow_spec_category(cat_id): workflow_spec_service.delete_category(cat_id) + def reorder_workflow_spec_category(cat_id, direction): - # TODO: fix reordering here too if direction not in ('up', 'down'): raise ApiError(code='bad_direction', message='The direction must be `up` or `down`.') WorkflowService.cleanup_workflow_spec_category_display_order() - # TODO: modify this to get_workflow_category(category) - category = session.query(WorkflowSpecCategoryModel).\ - filter(WorkflowSpecCategoryModel.id == cat_id).first() + category = WorkflowSpecService.get_category(cat_id) if category: - ordered_categories = WorkflowService.reorder_workflow_spec_category(category, direction) - schema = WorkflowSpecCategoryModelSchema(many=True) - return schema.dump(ordered_categories) + ordered_categories = WorkflowSpecService.reorder_workflow_spec_category(category, direction) + return ordered_categories else: raise ApiError(code='bad_category_id', message=f'The category id {cat_id} did not return a Workflow Spec Category. Make sure it is a valid ID.') diff --git a/crc/services/workflow_spec_service.py b/crc/services/workflow_spec_service.py index 30cd7746..814a161c 100644 --- a/crc/services/workflow_spec_service.py +++ b/crc/services/workflow_spec_service.py @@ -73,6 +73,9 @@ class WorkflowSpecService(FileSystemService): # fixme pass + def get_standalones(self) -> List[WorkflowSpecInfo]: + pass + def get_categories(self) -> List[WorkflowSpecCategory]: pass From c78140e60ccba27f53d0204c32ac935b8b3650fc Mon Sep 17 00:00:00 2001 From: alicia pritchett Date: Mon, 7 Feb 2022 12:17:49 -0500 Subject: [PATCH 2/7] fixed spec_file.py --- crc/api/spec_file.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/crc/api/spec_file.py b/crc/api/spec_file.py index 8d0b21e8..0d951a43 100644 --- a/crc/api/spec_file.py +++ b/crc/api/spec_file.py @@ -1,4 +1,4 @@ -from crc import session +from crc import session, WorkflowSpecService from crc.api.common import ApiError from crc.models.file import FileSchema, FileType from crc.services.spec_file_service import SpecFileService @@ -13,7 +13,7 @@ def get_files(spec_id, include_libraries=False): if spec_id is None: raise ApiError(code='missing_spec_id', message='Please specify the workflow_spec_id.') - workflow_spec = session.query(WorkflowSpecModel).filter_by(id=spec_id).first() + workflow_spec = WorkflowSpecService.get_spec(spec_id) if workflow_spec is None: raise ApiError(code='unknown_spec', message=f'Unknown Spec: {spec_id}') @@ -23,7 +23,7 @@ def get_files(spec_id, include_libraries=False): def get_file(spec_id, file_name): - workflow_spec = session.query(WorkflowSpecModel).filter_by(id=spec_id).first() + workflow_spec = WorkflowSpecService.get_spec(spec_id) files = SpecFileService.get_files(workflow_spec, file_name) if len(files) == 0: raise ApiError(code='unknown file', @@ -33,16 +33,16 @@ def get_file(spec_id, file_name): def add_file(spec_id): - workflow_spec = session.query(WorkflowSpecModel).filter_by(id=spec_id).first() + workflow_spec = WorkflowSpecService.get_spec(spec_id) file = connexion.request.files['file'] file = SpecFileService.add_file(workflow_spec, file.filename, file.stream.read()) - if not WorkflowSpecModel.primary_process_id and file.type == FileType.bpmn: + if not workflow_spec.primary_process_name and file.type == FileType.bpmn: SpecFileService.set_primary_bpmn(workflow_spec, file.name) return FileSchema().dump(file) def update(spec_id, file_name, is_primary): - workflow_spec = session.query(WorkflowSpecModel).filter_by(id=spec_id).first() + workflow_spec = WorkflowSpecService.get_spec(spec_id) files = SpecFileService.get_files(workflow_spec, file_name) if len(files) < 1: raise ApiError(code='unknown file', @@ -55,7 +55,7 @@ def update(spec_id, file_name, is_primary): def update_data(spec_id, file_name): - workflow_spec_model = session.query(WorkflowSpecModel).filter_by(id=spec_id).first() + workflow_spec_model = WorkflowSpecService.get_spec(spec_id) if workflow_spec_model is None: raise ApiError(code='missing_spec', message=f'The workflow spec for id {spec_id} does not exist.') @@ -65,10 +65,10 @@ def update_data(spec_id, file_name): def get_data(spec_id, file_name): - workflow_spec_model = session.query(WorkflowSpecModel).filter_by(id=spec_id).first() - file_data = SpecFileService.get_data(workflow_spec_model, file_name) + workflow_spec = WorkflowSpecService.get_spec(spec_id) + file_data = SpecFileService.get_data(workflow_spec, file_name) if file_data is not None: - file_info = SpecFileService.get_files(workflow_spec_model, file_name)[0] + file_info = SpecFileService.get_files(workflow_spec, file_name)[0] return send_file( io.BytesIO(file_data), attachment_filename=file_name, @@ -82,5 +82,5 @@ def get_data(spec_id, file_name): def delete(spec_id, file_name): - workflow_spec_model = session.query(WorkflowSpecModel).filter_by(id=spec_id).first() - SpecFileService.delete_file(workflow_spec_model, file_name) + workflow_spec = WorkflowSpecService.get_spec(spec_id) + SpecFileService.delete_file(workflow_spec, file_name) From 1c384a78e9fc49d4c27db6321734fdcee7ffd565 Mon Sep 17 00:00:00 2001 From: Dan Date: Mon, 7 Feb 2022 12:18:32 -0500 Subject: [PATCH 3/7] the tests kind of try and run now ... --- crc/models/task_event.py | 2 +- crc/models/workflow.py | 2 +- crc/services/workflow_spec_service.py | 54 ++++++++--- tests/workflow/test_workflow_spec_service.py | 97 +++++--------------- 4 files changed, 64 insertions(+), 91 deletions(-) diff --git a/crc/models/task_event.py b/crc/models/task_event.py index 2a6fff76..f8583a71 100644 --- a/crc/models/task_event.py +++ b/crc/models/task_event.py @@ -14,7 +14,7 @@ class TaskEventModel(db.Model): study_id = db.Column(db.Integer, db.ForeignKey('study.id')) user_uid = db.Column(db.String, nullable=False) # In some cases the unique user id may not exist in the db yet. workflow_id = db.Column(db.Integer, db.ForeignKey('workflow.id'), nullable=False) - workflow_spec_id = db.Column(db.String, db.ForeignKey('workflow_spec.id')) + workflow_spec_id = db.Column(db.String) spec_version = db.Column(db.String) action = db.Column(db.String) task_id = db.Column(db.String) diff --git a/crc/models/workflow.py b/crc/models/workflow.py index dd8fd8b7..ab12c849 100644 --- a/crc/models/workflow.py +++ b/crc/models/workflow.py @@ -79,7 +79,7 @@ class WorkflowModel(db.Model): status = db.Column(db.Enum(WorkflowStatus)) study_id = db.Column(db.Integer, db.ForeignKey('study.id')) study = db.relationship("StudyModel", backref='workflow') - workflow_spec_id = db.Column(db.String, db.ForeignKey('workflow_spec.id')) + workflow_spec_id = db.Column(db.String) total_tasks = db.Column(db.Integer, default=0) completed_tasks = db.Column(db.Integer, default=0) last_updated = db.Column(db.DateTime(timezone=True), server_default=func.now()) diff --git a/crc/services/workflow_spec_service.py b/crc/services/workflow_spec_service.py index 30cd7746..a9f2d61d 100644 --- a/crc/services/workflow_spec_service.py +++ b/crc/services/workflow_spec_service.py @@ -54,6 +54,9 @@ class WorkflowSpecService(FileSystemService): raise ApiError('unknown spec', 'unable to find a spec with id:' + spec_id) return self.specs[spec_id] + def get_specs(self): + return list(self.specs.values()) + def reorder_spec(self, spec:WorkflowSpecInfo, direction): workflows = spec.category.workflows workflows.sort(key=lambda w: w.display_order) @@ -70,23 +73,42 @@ class WorkflowSpecService(FileSystemService): return workflows def get_libraries(self) -> List[WorkflowSpecInfo]: - # fixme - pass + spec_list = list(self.libraries.values()) + spec_list.sort(key=lambda w: w.display_order) + return spec_list + + def get_standalones(self) -> List[WorkflowSpecInfo]: + spec_list = list(self.standalone.values()) + spec_list.sort(key=lambda w: w.display_order) + return spec_list def get_categories(self) -> List[WorkflowSpecCategory]: - pass + """Returns the categories as a list in display order""" + cat_list = list(self.categories.values()) + cat_list.sort(key=lambda w: w.display_order) + return cat_list def get_category(self, category_id) -> WorkflowSpecCategory: - pass + if category_id not in self.categories: + raise ApiError('unknown category', 'unable to find a category with id:' + category_id) + return self.categories[category_id] def add_category(self, category: WorkflowSpecCategory): - pass + self.update_category(category) def update_category(self, category: WorkflowSpecCategory): - pass + cat_path = self.category_path(category.display_name) + os.makedirs(os.path.dirname(cat_path), exist_ok=True) + json_path = os.path.join(cat_path, self.CAT_JSON_FILE) + with open(json_path, "w") as cat_json: + json.dump(self.CAT_SCHEMA.dump(category), cat_json, indent=4) + self.scan_file_system() - def delete_category(self, category: WorkflowSpecCategory): - pass + def delete_category(self, category_id: str): + if category_id in self.specs: + path = self.category_path(category_id) + shutil.rmtree(path) + self.scan_file_system() def reorder_workflow_spec_category(self, spec:WorkflowSpecInfo, direction): # Fixme: Resort Workflow categories @@ -101,18 +123,20 @@ class WorkflowSpecService(FileSystemService): self.master_spec = None self.libraries = {} if not os.path.exists(FileSystemService.root_path()): - raise ApiError('missing_specs', 'The path for workflow specifications does not exist.') + return # Nothing to scan yet. There are no files. directory_items = os.scandir(FileSystemService.root_path()) for item in directory_items: if item.is_dir(): if item.name == self.LIBRARY_SPECS: self.scan_category(item, is_library=True) + if item.name == self.STAND_ALONE_SPECS: + self.scan_category(item, is_standalone=True) elif item.name == self.MASTER_SPECIFICATION: self.scan_spec(item, is_master=True) else: self.scan_category(item) - def scan_category(self, dir_item: os.DirEntry, is_library=False): + def scan_category(self, dir_item: os.DirEntry, is_library=False, is_standalone=False): """Reads the category.json file, and any workflow directories """ cat_path = os.path.join(dir_item.path, self.CAT_JSON_FILE) if os.path.exists(cat_path): @@ -125,6 +149,8 @@ class WorkflowSpecService(FileSystemService): json.dump(self.CAT_SCHEMA.dump(cat), wf_json, indent=4) if is_library: self.libraries = cat + elif is_standalone: + self.standalone = cat else: self.categories[cat.id] = cat workflow_dirs = os.scandir(FileSystemService.root_path()) @@ -141,7 +167,6 @@ class WorkflowSpecService(FileSystemService): # workflow_metas.append(WorkflowMetadata.from_workflow(workflow)) return workflow_metas - def scan_spec(self, dir_item: os.DirEntry, is_master=False, category=None): if not is_master and not category: raise ApiError("invalid_spec_dir", "Please specify what category this workflow belongs to.") @@ -153,15 +178,16 @@ class WorkflowSpecService(FileSystemService): else: spec = WorkflowSpecInfo(id=dir_item.name, library=False, standalone=False, is_master_spec=is_master, display_name=dir_item.name, description="", primary_process_id="", - primary_file_name="") + primary_file_name="", category_name="", display_order=0, is_review=False, + libraries=[]) with open(spec_path, "w") as wf_json: - json.dump(WorkflowSpecInfoSchema.dump(spec), wf_json, indent=4) + json.dump(self.WF_SCHEMA.dump(spec), wf_json, indent=4) if is_master: self.master_spec = spec elif category: self.specs[spec.id] = spec spec.category = category - category.workflow_specs.append(spec) + category.workflows.append(spec) def set_primary_bpmn(self, workflow_spec: WorkflowSpecInfo, file_name: str, binary_data=None): # If this is a BPMN, extract the process id, and determine if it is contains swim lanes. diff --git a/tests/workflow/test_workflow_spec_service.py b/tests/workflow/test_workflow_spec_service.py index 861fba7a..d6efda6b 100644 --- a/tests/workflow/test_workflow_spec_service.py +++ b/tests/workflow/test_workflow_spec_service.py @@ -4,11 +4,10 @@ import shutil from tests.base_test import BaseTest -from crc.services.workflow_sync import WorkflowSyncService from crc.models.workflow import WorkflowSpecInfo, WorkflowSpecCategory from crc.services.file_system_service import FileSystemService from crc.services.spec_file_service import SpecFileService -from crc import db, app +from crc import db, app, WorkflowSpecService class TestWorkflowSync(BaseTest): @@ -16,12 +15,12 @@ class TestWorkflowSync(BaseTest): spec_path = FileSystemService.root_path() import_spec_path = os.path.join(app.root_path, '..', 'tests', 'data', 'IMPORT_TEST') - def set_up_file_system(self): - """Some tests rely on a well populated file system and an empty database to start""" + def copy_files_to_file_system(self): + """Some tests rely on a well populated file system """ shutil.copytree(self.import_spec_path, self.spec_path) - def set_up_database(self): - """Some tests rely on a well populated database and an empty file system to start""" + def build_file_system_from_models(self): + """Some tests check to see what happens when we write data to an empty file system.""" # Construct Two Categories, with one workflow in first category, two in the second. # assure that the data in categories.json is correct @@ -35,77 +34,25 @@ class TestWorkflowSync(BaseTest): self.load_test_spec('email', category_id=c1.id, library=True) def test_from_file_system_blank_slate(self): - self.assertEquals(0, len(db.session.query(WorkflowSpecModel).all())) - self.assertEquals(0, len(db.session.query(WorkflowSpecCategoryModel).all())) - self.set_up_file_system() - WorkflowSyncService().from_file_system() - self.assertEquals(2, len(db.session.query(WorkflowSpecCategoryModel).all())) - self.assertEquals(5, len(db.session.query(WorkflowSpecModel).all())) - self.assertEquals(1, len(db.session.query(WorkflowSpecModel).filter(WorkflowSpecModel.category_id == 1).all())) - self.assertEquals(2, len(db.session.query(WorkflowSpecModel).filter(WorkflowSpecModel.category_id == 2).all())) - self.assertEquals(1, len(db.session.query(WorkflowSpecModel).filter(WorkflowSpecModel.is_master_spec).all())) - self.assertEquals(1, len(db.session.query(WorkflowSpecModel).filter(WorkflowSpecModel.library).all())) - # The top level workflow, has a library - tlw = db.session.query(WorkflowSpecModel).filter(WorkflowSpecModel.is_master_spec).first() - self.assertEquals(1, len(tlw.libraries)) - - def test_repeated_imports(self): - self.set_up_file_system() - WorkflowSyncService().from_file_system() - WorkflowSyncService().from_file_system() - WorkflowSyncService().from_file_system() - self.assertEquals(2, len(db.session.query(WorkflowSpecCategoryModel).all())) - self.assertEquals(5, len(db.session.query(WorkflowSpecModel).all())) + service = WorkflowSpecService() + self.assertEquals(0, len(service.get_categories())) + self.assertEquals(0, len(service.get_specs())) + self.copy_files_to_file_system() + self.assertEquals(2, len(service.get_categories())) + self.assertEquals(5, len(service.get_specs())) + self.assertEquals(1, len(service.get_category('Category Number One').workflows)) + self.assertEquals(2, len(service.get_category('Category Number Two').workflows)) + self.assertIsNotNone(service.master_spec) + self.assertEquals(1, len(service.get_libraries())) + self.assertEquals(1, len(service.master_spec.libraries)) def test_delete_category_and_workflows(self): - self.set_up_file_system() - WorkflowSyncService().from_file_system() + self.copy_files_to_file_system() + service = WorkflowSpecService() cat_path = SpecFileService().category_path('Category Number One') - shutil.rmtree(cat_path) - WorkflowSyncService().from_file_system() - self.assertEquals(1, len(db.session.query(WorkflowSpecCategoryModel).all())) - self.assertEquals(4, len(db.session.query(WorkflowSpecModel).all())) - self.assertEquals(0, len(db.session.query(WorkflowSpecModel).filter(WorkflowSpecModel.category_id == 1).all())) - - WorkflowSyncService().to_file_system() - json_path = os.path.join(FileSystemService.root_path(), "categories.json") - with open(json_path) as json_file: - data = json.load(json_file) - self.assertEquals(1, len(data['categories']), "When the json file is written back to disk, there is only one category now.") - - def test_to_file_system(self): - """Assure we get the basic paths on the file system that we would expect.""" - self.assertFalse(os.path.exists(self.spec_path)) - self.set_up_database() - self.assertEqual(4, len(os.listdir(self.spec_path)), "Adding workflows should create dir structure") - WorkflowSyncService().to_file_system() - self.assertEqual(5, len(os.listdir(self.spec_path)), "Sync service should create categories.json file") - - def test_to_file_system_correct_categories(self): - """Assure we have two categories in the json file, and that these directories exist, and contain - workflow.json files for each workflow.""" - self.set_up_database() - WorkflowSyncService().to_file_system() - json_path = os.path.join(self.spec_path, 'categories.json') - - with open(json_path) as json_file: - data = json.load(json_file) - self.assertTrue('categories' in data) - self.assertEqual(2, len(data['categories'])) - counter = 0 - for c in data['categories']: - cat_path = os.path.join(self.spec_path, c['display_name']) - self.assertTrue(os.path.exists(cat_path), "The category directories exist.") - self.assertEqual(data['categories'][counter]['display_order'], counter, "Order is correct") - counter += 1 - workflow_dirs = os.listdir(cat_path) - for wd in workflow_dirs: - wf_json_path = os.path.join(cat_path, wd, 'workflow.json') - self.assertTrue(os.path.exists(wf_json_path), "A workflow.json file should exist.") - # Fixme: Assure the master workflow spec, and Libraries are also exported to file system. + shutil.rmtree(cat_path) # Remove the path, as if from a git pull and the path was removed. + self.assertEquals(2, len(service.get_categories())) + self.assertEquals(4, len(service.get_specs())) + - # Todo: - # * What if category json files, and directories don't match? - # * Test renaming a category - # * Test moving a workflow to a different category From abc17a8aff4a04096f6221f5c3b6b865c59d43fb Mon Sep 17 00:00:00 2001 From: Dan Date: Mon, 7 Feb 2022 12:30:20 -0500 Subject: [PATCH 4/7] load the workflow_spec_service from app to avoid frequent file scans. --- crc/__init__.py | 2 +- crc/models/workflow.py | 18 ++++++++----- .../Category Number Two/category.json | 3 ++- tests/workflow/test_workflow_spec_service.py | 26 +++++++++---------- 4 files changed, 27 insertions(+), 22 deletions(-) diff --git a/crc/__init__.py b/crc/__init__.py index 2b5e1968..35ef16be 100644 --- a/crc/__init__.py +++ b/crc/__init__.py @@ -57,7 +57,7 @@ from crc.api import admin from crc.services.workflow_service import WorkflowService from crc.services.workflow_spec_service import WorkflowSpecService connexion_app.add_api('api.yml', base_path='/v1.0') - +workflow_spec_service = WorkflowSpecService() # needed function to avoid circular import def process_waiting_tasks(): diff --git a/crc/models/workflow.py b/crc/models/workflow.py index ab12c849..7c2d9ec9 100644 --- a/crc/models/workflow.py +++ b/crc/models/workflow.py @@ -1,6 +1,6 @@ import enum -from marshmallow import EXCLUDE +from marshmallow import EXCLUDE, post_load from sqlalchemy import func from crc import db, ma @@ -19,12 +19,15 @@ class WorkflowSpecCategorySchema(ma.Schema): model = WorkflowSpecCategory fields = ["id", "display_name", "display_order", "admin"] + @post_load + def make_cat(self, data, **kwargs): + return WorkflowSpecCategory(**data) + class WorkflowSpecInfo(object): - def __init__(self, id, display_name, description, category_name, - display_order, is_master_spec, - standalone, library, primary_file_name, primary_process_id, is_review, - libraries): + def __init__(self, id, display_name, description, is_master_spec, + standalone, library, primary_file_name, primary_process_id, + libraries, category_name=None, display_order=0, is_review=False): self.id = id # Sting unique id self.display_name = display_name self.description = description @@ -45,9 +48,12 @@ class WorkflowSpecInfoSchema(ma.Schema): model = WorkflowSpecInfo fields = ["id", "display_name", "description", "category_id", "is_master_spec,", "standalone", "library", "primary_file_name", "primary_process_id", "is_review", - "libraries"] + "libraries", "category_name", "display_order", "is_master_spec", "is_review"] unknown = EXCLUDE + @post_load + def make_spec(self, data, **kwargs): + return WorkflowSpecInfo(**data) class WorkflowState(enum.Enum): hidden = "hidden" diff --git a/tests/data/IMPORT_TEST/Category Number Two/category.json b/tests/data/IMPORT_TEST/Category Number Two/category.json index 44ec8481..b2f4bba6 100644 --- a/tests/data/IMPORT_TEST/Category Number Two/category.json +++ b/tests/data/IMPORT_TEST/Category Number Two/category.json @@ -1,5 +1,6 @@ { "id": 2, "admin": false, - "display_order": 1 + "display_order": 1, + "display_name": "Category Number Two" } diff --git a/tests/workflow/test_workflow_spec_service.py b/tests/workflow/test_workflow_spec_service.py index d6efda6b..1d3d5f78 100644 --- a/tests/workflow/test_workflow_spec_service.py +++ b/tests/workflow/test_workflow_spec_service.py @@ -7,7 +7,7 @@ from tests.base_test import BaseTest from crc.models.workflow import WorkflowSpecInfo, WorkflowSpecCategory from crc.services.file_system_service import FileSystemService from crc.services.spec_file_service import SpecFileService -from crc import db, app, WorkflowSpecService +from crc import db, app, workflow_spec_service class TestWorkflowSync(BaseTest): @@ -34,25 +34,23 @@ class TestWorkflowSync(BaseTest): self.load_test_spec('email', category_id=c1.id, library=True) def test_from_file_system_blank_slate(self): - service = WorkflowSpecService() - self.assertEquals(0, len(service.get_categories())) - self.assertEquals(0, len(service.get_specs())) + self.assertEquals(0, len(workflow_spec_service.get_categories())) + self.assertEquals(0, len(workflow_spec_service.get_specs())) self.copy_files_to_file_system() - self.assertEquals(2, len(service.get_categories())) - self.assertEquals(5, len(service.get_specs())) - self.assertEquals(1, len(service.get_category('Category Number One').workflows)) - self.assertEquals(2, len(service.get_category('Category Number Two').workflows)) - self.assertIsNotNone(service.master_spec) - self.assertEquals(1, len(service.get_libraries())) - self.assertEquals(1, len(service.master_spec.libraries)) + self.assertEquals(2, len(workflow_spec_service.get_categories())) + self.assertEquals(5, len(workflow_spec_service.get_specs())) + self.assertEquals(1, len(workflow_spec_service.get_category('Category Number One').workflows)) + self.assertEquals(2, len(workflow_spec_service.get_category('Category Number Two').workflows)) + self.assertIsNotNone(workflow_spec_service.master_spec) + self.assertEquals(1, len(workflow_spec_service.get_libraries())) + self.assertEquals(1, len(workflow_spec_service.master_spec.libraries)) def test_delete_category_and_workflows(self): self.copy_files_to_file_system() - service = WorkflowSpecService() cat_path = SpecFileService().category_path('Category Number One') shutil.rmtree(cat_path) # Remove the path, as if from a git pull and the path was removed. - self.assertEquals(2, len(service.get_categories())) - self.assertEquals(4, len(service.get_specs())) + self.assertEquals(3, len(workflow_spec_service.get_categories())) + self.assertEquals(4, len(workflow_spec_service.get_specs())) From ad095aaaf56dc16bff314b5e09686b8049ce2d11 Mon Sep 17 00:00:00 2001 From: Dan Date: Mon, 7 Feb 2022 12:38:00 -0500 Subject: [PATCH 5/7] don't try to load the workflowspecservice too early. --- crc/__init__.py | 2 -- tests/workflow/test_workflow_spec_service.py | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/crc/__init__.py b/crc/__init__.py index 35ef16be..a332326b 100644 --- a/crc/__init__.py +++ b/crc/__init__.py @@ -55,9 +55,7 @@ from crc import models from crc import api from crc.api import admin from crc.services.workflow_service import WorkflowService -from crc.services.workflow_spec_service import WorkflowSpecService connexion_app.add_api('api.yml', base_path='/v1.0') -workflow_spec_service = WorkflowSpecService() # needed function to avoid circular import def process_waiting_tasks(): diff --git a/tests/workflow/test_workflow_spec_service.py b/tests/workflow/test_workflow_spec_service.py index 1d3d5f78..92a61659 100644 --- a/tests/workflow/test_workflow_spec_service.py +++ b/tests/workflow/test_workflow_spec_service.py @@ -37,6 +37,7 @@ class TestWorkflowSync(BaseTest): self.assertEquals(0, len(workflow_spec_service.get_categories())) self.assertEquals(0, len(workflow_spec_service.get_specs())) self.copy_files_to_file_system() + workflow_spec_service.scan_file_system() self.assertEquals(2, len(workflow_spec_service.get_categories())) self.assertEquals(5, len(workflow_spec_service.get_specs())) self.assertEquals(1, len(workflow_spec_service.get_category('Category Number One').workflows)) From 732a8106da90c43be1dacfb12ab951ab6f3a136e Mon Sep 17 00:00:00 2001 From: alicia pritchett Date: Mon, 7 Feb 2022 12:44:39 -0500 Subject: [PATCH 6/7] fix uhh some import statements? --- crc/api/workflow.py | 57 ++++++++++++++++++++++++--------------------- 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/crc/api/workflow.py b/crc/api/workflow.py index ad411c30..77633a92 100644 --- a/crc/api/workflow.py +++ b/crc/api/workflow.py @@ -2,7 +2,7 @@ import uuid from flask import g -from crc import session, WorkflowSpecService +from crc import session from crc.api.common import ApiError, ApiErrorSchema from crc.models.api_models import WorkflowApiSchema from crc.models.study import StudyModel, WorkflowMetadata, StudyStatus @@ -18,6 +18,9 @@ from crc.services.task_logging_service import TaskLoggingService from crc.services.user_service import UserService from crc.services.workflow_processor import WorkflowProcessor from crc.services.workflow_service import WorkflowService +from crc.services.workflow_spec_service import WorkflowSpecService + +workflow_spec_service = WorkflowSpecService() def all_specifications(libraries=False,standalone=False): @@ -25,13 +28,13 @@ def all_specifications(libraries=False,standalone=False): raise ApiError('inconceivable!', 'You should specify libraries or standalone, but not both') if libraries: - return WorkflowSpecService.get_libraries() + return workflow_spec_service.get_libraries() if standalone: - return WorkflowSpecService.get_standalones() + return workflow_spec_service.get_standalones() # return standard workflows (not library, not standalone) - categories = WorkflowSpecService.get_categories() + categories = workflow_spec_service.get_categories() workflows = [] for cat in categories: workflows.extend(cat.workflows) @@ -47,14 +50,14 @@ def add_workflow_specification(body): if body['library'] is True or body['standalone'] is True: body['category_id'] = None - new_spec = WorkflowSpecService.add_spec(body) + new_spec = workflow_spec_service.add_spec(body) return new_spec def get_workflow_specification(spec_id): if spec_id is None: raise ApiError('unknown_spec', 'Please provide a valid Workflow Specification ID.') - spec = WorkflowSpecService.get_spec(spec_id) + spec = workflow_spec_service.get_spec(spec_id) if spec is None: raise ApiError('unknown_spec', 'The Workflow Specification "' + spec_id + '" is not recognized.') @@ -67,8 +70,8 @@ def validate_spec_and_library(spec_id,library_id): if library_id is None: raise ApiError('unknown_spec', 'Please provide a valid Library Specification ID.') - spec = WorkflowSpecService.get_spec(spec_id) - library = WorkflowSpecService.get_library(library_id); + spec = workflow_spec_service.get_spec(spec_id) + library = workflow_spec_service.get_library(library_id); if spec is None: raise ApiError('unknown_spec', 'The Workflow Specification "' + spec_id + '" is not recognized.') if library is None: @@ -79,27 +82,27 @@ def validate_spec_and_library(spec_id,library_id): def add_workflow_spec_library(spec_id, library_id): validate_spec_and_library(spec_id, library_id) - libraries: WorkflowSpecService.get_libraries() + libraries: workflow_spec_service.get_libraries() libraryids = [x.library_spec_id for x in libraries] if library_id in libraryids: raise ApiError('unknown_spec', 'The Library Specification "' + library_id + '" is already attached.') - spec = WorkflowSpecService.get_spec(spec_id) - library = WorkflowSpecService.get_spec(library_id) + spec = workflow_spec_service.get_spec(spec_id) + library = workflow_spec_service.get_spec(library_id) spec.libraries.push(library) - WorkflowSpecService.update_spec(spec_id) + workflow_spec_service.update_spec(spec_id) return spec def drop_workflow_spec_library(spec_id, library_id): validate_spec_and_library(spec_id, library_id) - spec = WorkflowSpecService.get_spec(spec_id) + spec = workflow_spec_service.get_spec(spec_id) # heres a piece of code that certainly wont work - library = WorkflowSpecService.get_spec(library_id) + library = workflow_spec_service.get_spec(library_id) if library in spec.libraries: spec.libraries.pop(library) - WorkflowSpecService.update_spec(spec_id) + workflow_spec_service.update_spec(spec_id) return spec @@ -118,7 +121,7 @@ def validate_workflow_specification(spec_id, study_id=None, test_until=None): def update_workflow_specification(spec_id, body): if spec_id is None: raise ApiError('unknown_spec', 'Please provide a valid Workflow Spec ID.') - spec = WorkflowSpecService.get_spec(spec_id) + spec = workflow_spec_service.get_spec(spec_id) if spec is None: raise ApiError('unknown_study', 'The spec "' + spec_id + '" is not recognized.') @@ -131,7 +134,7 @@ def update_workflow_specification(spec_id, body): if body['library'] is True or body['standalone'] is True: body['category_id'] = None - spec = WorkflowSpecService.update_spec(spec_id, body) + spec = workflow_spec_service.update_spec(spec_id, body) return spec @@ -139,7 +142,7 @@ def delete_workflow_specification(spec_id): if spec_id is None: raise ApiError('unknown_spec', 'Please provide a valid Workflow Specification ID.') - spec = WorkflowSpecService.get_spec(spec_id) + spec = workflow_spec_service.get_spec(spec_id) if spec is None: raise ApiError('unknown_spec', 'The Workflow Specification "' + spec_id + '" is not recognized.') @@ -155,7 +158,7 @@ def delete_workflow_specification(spec_id): WorkflowService.delete_workflow_spec_task_events(spec_id) # .delete() doesn't work when we need a cascade. Must grab the record, and explicitly delete - WorkflowSpecService.delete_spec(spec_id) + workflow_spec_service.delete_spec(spec_id) # Reorder the remaining specs WorkflowService.cleanup_workflow_spec_display_order(spec.category_id) @@ -165,9 +168,9 @@ def reorder_workflow_specification(spec_id, direction): if direction not in ('up', 'down'): raise ApiError(code='bad_direction', message='The direction must be `up` or `down`.') - spec = WorkflowSpecService.get_spec(spec_id) + spec = workflow_spec_service.get_spec(spec_id) if spec: - ordered_specs = WorkflowSpecService.reorder_spec(spec, direction) + ordered_specs = workflow_spec_service.reorder_spec(spec, direction) else: raise ApiError(code='bad_spec_id', message=f'The spec_id {spec_id} did not return a specification. Please check that it is valid.') @@ -322,22 +325,22 @@ def __update_task(processor, task, data, user): def list_workflow_spec_categories(): - return WorkflowSpecService.get_workflow_categories() + return workflow_spec_service.get_workflow_categories() def get_workflow_spec_category(cat_id): - return WorkflowSpecService.get_workflow_category(cat_id) + return workflow_spec_service.get_workflow_category(cat_id) def add_workflow_spec_category(body): - return WorkflowSpecService.add_category(body) + return workflow_spec_service.add_category(body) def update_workflow_spec_category(cat_id, body): if cat_id is None: raise ApiError('unknown_category', 'Please provide a valid Workflow Spec Category ID.') - category = WorkflowSpecService.update_category(cat_id, body) + category = workflow_spec_service.update_category(cat_id, body) if category is None: raise ApiError('unknown_category', 'The category "' + cat_id + '" is not recognized.') @@ -358,9 +361,9 @@ def reorder_workflow_spec_category(cat_id, direction): raise ApiError(code='bad_direction', message='The direction must be `up` or `down`.') WorkflowService.cleanup_workflow_spec_category_display_order() - category = WorkflowSpecService.get_category(cat_id) + category = workflow_spec_service.get_category(cat_id) if category: - ordered_categories = WorkflowSpecService.reorder_workflow_spec_category(category, direction) + ordered_categories = workflow_spec_service.reorder_workflow_spec_category(category, direction) return ordered_categories else: raise ApiError(code='bad_category_id', From 056f901f9ad7daaa851d70940ef62ac2921c11f1 Mon Sep 17 00:00:00 2001 From: Dan Date: Mon, 7 Feb 2022 12:59:48 -0500 Subject: [PATCH 7/7] WIP - its just caos right now. --- crc/api/spec_file.py | 19 ++++--- crc/api/workflow.py | 56 ++++++++++---------- crc/services/workflow_spec_service.py | 1 - tests/base_test.py | 4 +- tests/workflow/cr_connect.log | 2 + tests/workflow/test_workflow_spec_service.py | 28 +++++----- 6 files changed, 59 insertions(+), 51 deletions(-) diff --git a/crc/api/spec_file.py b/crc/api/spec_file.py index 0d951a43..c9104c3d 100644 --- a/crc/api/spec_file.py +++ b/crc/api/spec_file.py @@ -1,4 +1,4 @@ -from crc import session, WorkflowSpecService +from crc import session from crc.api.common import ApiError from crc.models.file import FileSchema, FileType from crc.services.spec_file_service import SpecFileService @@ -8,12 +8,15 @@ from flask import send_file import io import connexion +from crc.services.workflow_spec_service import WorkflowSpecService + +workflow_spec_service = WorkflowSpecService() def get_files(spec_id, include_libraries=False): if spec_id is None: raise ApiError(code='missing_spec_id', message='Please specify the workflow_spec_id.') - workflow_spec = WorkflowSpecService.get_spec(spec_id) + workflow_spec = workflow_spec_service.get_spec(spec_id) if workflow_spec is None: raise ApiError(code='unknown_spec', message=f'Unknown Spec: {spec_id}') @@ -23,7 +26,7 @@ def get_files(spec_id, include_libraries=False): def get_file(spec_id, file_name): - workflow_spec = WorkflowSpecService.get_spec(spec_id) + workflow_spec = workflow_spec_service.get_spec(spec_id) files = SpecFileService.get_files(workflow_spec, file_name) if len(files) == 0: raise ApiError(code='unknown file', @@ -33,7 +36,7 @@ def get_file(spec_id, file_name): def add_file(spec_id): - workflow_spec = WorkflowSpecService.get_spec(spec_id) + workflow_spec = workflow_spec_service.get_spec(spec_id) file = connexion.request.files['file'] file = SpecFileService.add_file(workflow_spec, file.filename, file.stream.read()) if not workflow_spec.primary_process_name and file.type == FileType.bpmn: @@ -42,7 +45,7 @@ def add_file(spec_id): def update(spec_id, file_name, is_primary): - workflow_spec = WorkflowSpecService.get_spec(spec_id) + workflow_spec = workflow_spec_service.get_spec(spec_id) files = SpecFileService.get_files(workflow_spec, file_name) if len(files) < 1: raise ApiError(code='unknown file', @@ -55,7 +58,7 @@ def update(spec_id, file_name, is_primary): def update_data(spec_id, file_name): - workflow_spec_model = WorkflowSpecService.get_spec(spec_id) + workflow_spec_model = workflow_spec_service.get_spec(spec_id) if workflow_spec_model is None: raise ApiError(code='missing_spec', message=f'The workflow spec for id {spec_id} does not exist.') @@ -65,7 +68,7 @@ def update_data(spec_id, file_name): def get_data(spec_id, file_name): - workflow_spec = WorkflowSpecService.get_spec(spec_id) + workflow_spec = workflow_spec_service.get_spec(spec_id) file_data = SpecFileService.get_data(workflow_spec, file_name) if file_data is not None: file_info = SpecFileService.get_files(workflow_spec, file_name)[0] @@ -82,5 +85,5 @@ def get_data(spec_id, file_name): def delete(spec_id, file_name): - workflow_spec = WorkflowSpecService.get_spec(spec_id) + workflow_spec = workflow_spec_service.get_spec(spec_id) SpecFileService.delete_file(workflow_spec, file_name) diff --git a/crc/api/workflow.py b/crc/api/workflow.py index ad411c30..2dc6f4ce 100644 --- a/crc/api/workflow.py +++ b/crc/api/workflow.py @@ -2,7 +2,7 @@ import uuid from flask import g -from crc import session, WorkflowSpecService +from crc import session from crc.api.common import ApiError, ApiErrorSchema from crc.models.api_models import WorkflowApiSchema from crc.models.study import StudyModel, WorkflowMetadata, StudyStatus @@ -18,20 +18,22 @@ from crc.services.task_logging_service import TaskLoggingService from crc.services.user_service import UserService from crc.services.workflow_processor import WorkflowProcessor from crc.services.workflow_service import WorkflowService +from crc.services.workflow_spec_service import WorkflowSpecService +workflow_spec_service = WorkflowSpecService() def all_specifications(libraries=False,standalone=False): if libraries and standalone: raise ApiError('inconceivable!', 'You should specify libraries or standalone, but not both') if libraries: - return WorkflowSpecService.get_libraries() + return workflow_spec_service.get_libraries() if standalone: - return WorkflowSpecService.get_standalones() + return workflow_spec_service.get_standalones() # return standard workflows (not library, not standalone) - categories = WorkflowSpecService.get_categories() + categories = workflow_spec_service.get_categories() workflows = [] for cat in categories: workflows.extend(cat.workflows) @@ -47,14 +49,14 @@ def add_workflow_specification(body): if body['library'] is True or body['standalone'] is True: body['category_id'] = None - new_spec = WorkflowSpecService.add_spec(body) + new_spec = workflow_spec_service.add_spec(body) return new_spec def get_workflow_specification(spec_id): if spec_id is None: raise ApiError('unknown_spec', 'Please provide a valid Workflow Specification ID.') - spec = WorkflowSpecService.get_spec(spec_id) + spec = workflow_spec_service.get_spec(spec_id) if spec is None: raise ApiError('unknown_spec', 'The Workflow Specification "' + spec_id + '" is not recognized.') @@ -67,8 +69,8 @@ def validate_spec_and_library(spec_id,library_id): if library_id is None: raise ApiError('unknown_spec', 'Please provide a valid Library Specification ID.') - spec = WorkflowSpecService.get_spec(spec_id) - library = WorkflowSpecService.get_library(library_id); + spec = workflow_spec_service.get_spec(spec_id) + library = workflow_spec_service.get_library(library_id); if spec is None: raise ApiError('unknown_spec', 'The Workflow Specification "' + spec_id + '" is not recognized.') if library is None: @@ -79,27 +81,27 @@ def validate_spec_and_library(spec_id,library_id): def add_workflow_spec_library(spec_id, library_id): validate_spec_and_library(spec_id, library_id) - libraries: WorkflowSpecService.get_libraries() + libraries: workflow_spec_service.get_libraries() libraryids = [x.library_spec_id for x in libraries] if library_id in libraryids: raise ApiError('unknown_spec', 'The Library Specification "' + library_id + '" is already attached.') - spec = WorkflowSpecService.get_spec(spec_id) - library = WorkflowSpecService.get_spec(library_id) + spec = workflow_spec_service.get_spec(spec_id) + library = workflow_spec_service.get_spec(library_id) spec.libraries.push(library) - WorkflowSpecService.update_spec(spec_id) + workflow_spec_service.update_spec(spec_id) return spec def drop_workflow_spec_library(spec_id, library_id): validate_spec_and_library(spec_id, library_id) - spec = WorkflowSpecService.get_spec(spec_id) + spec = workflow_spec_service.get_spec(spec_id) # heres a piece of code that certainly wont work - library = WorkflowSpecService.get_spec(library_id) + library = workflow_spec_service.get_spec(library_id) if library in spec.libraries: spec.libraries.pop(library) - WorkflowSpecService.update_spec(spec_id) + workflow_spec_service.update_spec(spec_id) return spec @@ -118,7 +120,7 @@ def validate_workflow_specification(spec_id, study_id=None, test_until=None): def update_workflow_specification(spec_id, body): if spec_id is None: raise ApiError('unknown_spec', 'Please provide a valid Workflow Spec ID.') - spec = WorkflowSpecService.get_spec(spec_id) + spec = workflow_spec_service.get_spec(spec_id) if spec is None: raise ApiError('unknown_study', 'The spec "' + spec_id + '" is not recognized.') @@ -131,7 +133,7 @@ def update_workflow_specification(spec_id, body): if body['library'] is True or body['standalone'] is True: body['category_id'] = None - spec = WorkflowSpecService.update_spec(spec_id, body) + spec = workflow_spec_service.update_spec(spec_id, body) return spec @@ -139,7 +141,7 @@ def delete_workflow_specification(spec_id): if spec_id is None: raise ApiError('unknown_spec', 'Please provide a valid Workflow Specification ID.') - spec = WorkflowSpecService.get_spec(spec_id) + spec = workflow_spec_service.get_spec(spec_id) if spec is None: raise ApiError('unknown_spec', 'The Workflow Specification "' + spec_id + '" is not recognized.') @@ -155,7 +157,7 @@ def delete_workflow_specification(spec_id): WorkflowService.delete_workflow_spec_task_events(spec_id) # .delete() doesn't work when we need a cascade. Must grab the record, and explicitly delete - WorkflowSpecService.delete_spec(spec_id) + workflow_spec_service.delete_spec(spec_id) # Reorder the remaining specs WorkflowService.cleanup_workflow_spec_display_order(spec.category_id) @@ -165,9 +167,9 @@ def reorder_workflow_specification(spec_id, direction): if direction not in ('up', 'down'): raise ApiError(code='bad_direction', message='The direction must be `up` or `down`.') - spec = WorkflowSpecService.get_spec(spec_id) + spec = workflow_spec_service.get_spec(spec_id) if spec: - ordered_specs = WorkflowSpecService.reorder_spec(spec, direction) + ordered_specs = workflow_spec_service.reorder_spec(spec, direction) else: raise ApiError(code='bad_spec_id', message=f'The spec_id {spec_id} did not return a specification. Please check that it is valid.') @@ -322,22 +324,22 @@ def __update_task(processor, task, data, user): def list_workflow_spec_categories(): - return WorkflowSpecService.get_workflow_categories() + return workflow_spec_service.get_workflow_categories() def get_workflow_spec_category(cat_id): - return WorkflowSpecService.get_workflow_category(cat_id) + return workflow_spec_service.get_workflow_category(cat_id) def add_workflow_spec_category(body): - return WorkflowSpecService.add_category(body) + return workflow_spec_service.add_category(body) def update_workflow_spec_category(cat_id, body): if cat_id is None: raise ApiError('unknown_category', 'Please provide a valid Workflow Spec Category ID.') - category = WorkflowSpecService.update_category(cat_id, body) + category = workflow_spec_service.update_category(cat_id, body) if category is None: raise ApiError('unknown_category', 'The category "' + cat_id + '" is not recognized.') @@ -358,9 +360,9 @@ def reorder_workflow_spec_category(cat_id, direction): raise ApiError(code='bad_direction', message='The direction must be `up` or `down`.') WorkflowService.cleanup_workflow_spec_category_display_order() - category = WorkflowSpecService.get_category(cat_id) + category = workflow_spec_service.get_category(cat_id) if category: - ordered_categories = WorkflowSpecService.reorder_workflow_spec_category(category, direction) + ordered_categories = workflow_spec_service.reorder_workflow_spec_category(category, direction) return ordered_categories else: raise ApiError(code='bad_category_id', diff --git a/crc/services/workflow_spec_service.py b/crc/services/workflow_spec_service.py index 566b0da2..9a3085b9 100644 --- a/crc/services/workflow_spec_service.py +++ b/crc/services/workflow_spec_service.py @@ -29,7 +29,6 @@ class WorkflowSpecService(FileSystemService): self.master_spec = None self.libraries = {} self.standalone = {} - self.scan_file_system() def add_spec(self, spec: WorkflowSpecInfo): self.update_spec(spec) diff --git a/tests/base_test.py b/tests/base_test.py index 25104103..3478e10c 100644 --- a/tests/base_test.py +++ b/tests/base_test.py @@ -12,7 +12,7 @@ import datetime import shutil from flask import g -from crc import app, db, session, WorkflowSpecService +from crc import app, db, session from crc.models.api_models import WorkflowApiSchema, MultiInstanceType from crc.models.file import FileModel, CONTENT_TYPES from crc.models.task_event import TaskEventModel @@ -26,7 +26,7 @@ from crc.services.user_service import UserService from crc.services.workflow_service import WorkflowService from crc.services.document_service import DocumentService from example_data import ExampleDataLoader -from crc.services.user_file_service import UserFileService +from crc.services.workflow_spec_service import WorkflowSpecService # UNCOMMENT THIS FOR DEBUGGING SQL ALCHEMY QUERIES import logging diff --git a/tests/workflow/cr_connect.log b/tests/workflow/cr_connect.log index 4d6ed08a..62ea31a4 100644 --- a/tests/workflow/cr_connect.log +++ b/tests/workflow/cr_connect.log @@ -1,3 +1,5 @@ 2022-01-26 15:05:47,955 - connexion.apis.abstract - ERROR - Failed to add operation for GET /v1.0/workflow_sync/pullall 2022-01-26 15:06:36,303 - connexion.apis.abstract - ERROR - Failed to add operation for GET /v1.0/workflow_sync/pullall 2022-01-26 15:08:01,978 - connexion.apis.abstract - ERROR - Failed to add operation for GET /v1.0/file/{md5_hash}/hash_data +2022-02-07 12:46:21,875 - connexion.apis.abstract - ERROR - Failed to add operation for GET /v1.0/ldap +2022-02-07 12:49:13,793 - connexion.apis.abstract - ERROR - Failed to add operation for GET /v1.0/ldap diff --git a/tests/workflow/test_workflow_spec_service.py b/tests/workflow/test_workflow_spec_service.py index 92a61659..6c117e4d 100644 --- a/tests/workflow/test_workflow_spec_service.py +++ b/tests/workflow/test_workflow_spec_service.py @@ -7,13 +7,15 @@ from tests.base_test import BaseTest from crc.models.workflow import WorkflowSpecInfo, WorkflowSpecCategory from crc.services.file_system_service import FileSystemService from crc.services.spec_file_service import SpecFileService -from crc import db, app, workflow_spec_service +from crc.services.workflow_spec_service import WorkflowSpecService +from crc import db, app class TestWorkflowSync(BaseTest): spec_path = FileSystemService.root_path() import_spec_path = os.path.join(app.root_path, '..', 'tests', 'data', 'IMPORT_TEST') + service = WorkflowSpecService() def copy_files_to_file_system(self): """Some tests rely on a well populated file system """ @@ -34,24 +36,24 @@ class TestWorkflowSync(BaseTest): self.load_test_spec('email', category_id=c1.id, library=True) def test_from_file_system_blank_slate(self): - self.assertEquals(0, len(workflow_spec_service.get_categories())) - self.assertEquals(0, len(workflow_spec_service.get_specs())) + self.assertEquals(0, len(self.service.get_categories())) + self.assertEquals(0, len(self.service.get_specs())) self.copy_files_to_file_system() - workflow_spec_service.scan_file_system() - self.assertEquals(2, len(workflow_spec_service.get_categories())) - self.assertEquals(5, len(workflow_spec_service.get_specs())) - self.assertEquals(1, len(workflow_spec_service.get_category('Category Number One').workflows)) - self.assertEquals(2, len(workflow_spec_service.get_category('Category Number Two').workflows)) - self.assertIsNotNone(workflow_spec_service.master_spec) - self.assertEquals(1, len(workflow_spec_service.get_libraries())) - self.assertEquals(1, len(workflow_spec_service.master_spec.libraries)) + self.service.scan_file_system() + self.assertEquals(2, len(self.service.get_categories())) + self.assertEquals(5, len(self.service.get_specs())) + self.assertEquals(1, len(self.service.get_category('Category Number One').workflows)) + self.assertEquals(2, len(self.service.get_category('Category Number Two').workflows)) + self.assertIsNotNone(self.service.master_spec) + self.assertEquals(1, len(self.service.get_libraries())) + self.assertEquals(1, len(self.service.master_spec.libraries)) def test_delete_category_and_workflows(self): self.copy_files_to_file_system() cat_path = SpecFileService().category_path('Category Number One') shutil.rmtree(cat_path) # Remove the path, as if from a git pull and the path was removed. - self.assertEquals(3, len(workflow_spec_service.get_categories())) - self.assertEquals(4, len(workflow_spec_service.get_specs())) + self.assertEquals(3, len(self.service.get_categories())) + self.assertEquals(4, len(self.service.get_specs()))