From 056f901f9ad7daaa851d70940ef62ac2921c11f1 Mon Sep 17 00:00:00 2001 From: Dan Date: Mon, 7 Feb 2022 12:59:48 -0500 Subject: [PATCH] WIP - its just caos right now. --- crc/api/spec_file.py | 19 ++++--- crc/api/workflow.py | 56 ++++++++++---------- crc/services/workflow_spec_service.py | 1 - tests/base_test.py | 4 +- tests/workflow/cr_connect.log | 2 + tests/workflow/test_workflow_spec_service.py | 28 +++++----- 6 files changed, 59 insertions(+), 51 deletions(-) diff --git a/crc/api/spec_file.py b/crc/api/spec_file.py index 0d951a43..c9104c3d 100644 --- a/crc/api/spec_file.py +++ b/crc/api/spec_file.py @@ -1,4 +1,4 @@ -from crc import session, WorkflowSpecService +from crc import session from crc.api.common import ApiError from crc.models.file import FileSchema, FileType from crc.services.spec_file_service import SpecFileService @@ -8,12 +8,15 @@ from flask import send_file import io import connexion +from crc.services.workflow_spec_service import WorkflowSpecService + +workflow_spec_service = WorkflowSpecService() def get_files(spec_id, include_libraries=False): if spec_id is None: raise ApiError(code='missing_spec_id', message='Please specify the workflow_spec_id.') - workflow_spec = WorkflowSpecService.get_spec(spec_id) + workflow_spec = workflow_spec_service.get_spec(spec_id) if workflow_spec is None: raise ApiError(code='unknown_spec', message=f'Unknown Spec: {spec_id}') @@ -23,7 +26,7 @@ def get_files(spec_id, include_libraries=False): def get_file(spec_id, file_name): - workflow_spec = WorkflowSpecService.get_spec(spec_id) + workflow_spec = workflow_spec_service.get_spec(spec_id) files = SpecFileService.get_files(workflow_spec, file_name) if len(files) == 0: raise ApiError(code='unknown file', @@ -33,7 +36,7 @@ def get_file(spec_id, file_name): def add_file(spec_id): - workflow_spec = WorkflowSpecService.get_spec(spec_id) + workflow_spec = workflow_spec_service.get_spec(spec_id) file = connexion.request.files['file'] file = SpecFileService.add_file(workflow_spec, file.filename, file.stream.read()) if not workflow_spec.primary_process_name and file.type == FileType.bpmn: @@ -42,7 +45,7 @@ def add_file(spec_id): def update(spec_id, file_name, is_primary): - workflow_spec = WorkflowSpecService.get_spec(spec_id) + workflow_spec = workflow_spec_service.get_spec(spec_id) files = SpecFileService.get_files(workflow_spec, file_name) if len(files) < 1: raise ApiError(code='unknown file', @@ -55,7 +58,7 @@ def update(spec_id, file_name, is_primary): def update_data(spec_id, file_name): - workflow_spec_model = WorkflowSpecService.get_spec(spec_id) + workflow_spec_model = workflow_spec_service.get_spec(spec_id) if workflow_spec_model is None: raise ApiError(code='missing_spec', message=f'The workflow spec for id {spec_id} does not exist.') @@ -65,7 +68,7 @@ def update_data(spec_id, file_name): def get_data(spec_id, file_name): - workflow_spec = WorkflowSpecService.get_spec(spec_id) + workflow_spec = workflow_spec_service.get_spec(spec_id) file_data = SpecFileService.get_data(workflow_spec, file_name) if file_data is not None: file_info = SpecFileService.get_files(workflow_spec, file_name)[0] @@ -82,5 +85,5 @@ def get_data(spec_id, file_name): def delete(spec_id, file_name): - workflow_spec = WorkflowSpecService.get_spec(spec_id) + workflow_spec = workflow_spec_service.get_spec(spec_id) SpecFileService.delete_file(workflow_spec, file_name) diff --git a/crc/api/workflow.py b/crc/api/workflow.py index ad411c30..2dc6f4ce 100644 --- a/crc/api/workflow.py +++ b/crc/api/workflow.py @@ -2,7 +2,7 @@ import uuid from flask import g -from crc import session, WorkflowSpecService +from crc import session from crc.api.common import ApiError, ApiErrorSchema from crc.models.api_models import WorkflowApiSchema from crc.models.study import StudyModel, WorkflowMetadata, StudyStatus @@ -18,20 +18,22 @@ from crc.services.task_logging_service import TaskLoggingService from crc.services.user_service import UserService from crc.services.workflow_processor import WorkflowProcessor from crc.services.workflow_service import WorkflowService +from crc.services.workflow_spec_service import WorkflowSpecService +workflow_spec_service = WorkflowSpecService() def all_specifications(libraries=False,standalone=False): if libraries and standalone: raise ApiError('inconceivable!', 'You should specify libraries or standalone, but not both') if libraries: - return WorkflowSpecService.get_libraries() + return workflow_spec_service.get_libraries() if standalone: - return WorkflowSpecService.get_standalones() + return workflow_spec_service.get_standalones() # return standard workflows (not library, not standalone) - categories = WorkflowSpecService.get_categories() + categories = workflow_spec_service.get_categories() workflows = [] for cat in categories: workflows.extend(cat.workflows) @@ -47,14 +49,14 @@ def add_workflow_specification(body): if body['library'] is True or body['standalone'] is True: body['category_id'] = None - new_spec = WorkflowSpecService.add_spec(body) + new_spec = workflow_spec_service.add_spec(body) return new_spec def get_workflow_specification(spec_id): if spec_id is None: raise ApiError('unknown_spec', 'Please provide a valid Workflow Specification ID.') - spec = WorkflowSpecService.get_spec(spec_id) + spec = workflow_spec_service.get_spec(spec_id) if spec is None: raise ApiError('unknown_spec', 'The Workflow Specification "' + spec_id + '" is not recognized.') @@ -67,8 +69,8 @@ def validate_spec_and_library(spec_id,library_id): if library_id is None: raise ApiError('unknown_spec', 'Please provide a valid Library Specification ID.') - spec = WorkflowSpecService.get_spec(spec_id) - library = WorkflowSpecService.get_library(library_id); + spec = workflow_spec_service.get_spec(spec_id) + library = workflow_spec_service.get_library(library_id); if spec is None: raise ApiError('unknown_spec', 'The Workflow Specification "' + spec_id + '" is not recognized.') if library is None: @@ -79,27 +81,27 @@ def validate_spec_and_library(spec_id,library_id): def add_workflow_spec_library(spec_id, library_id): validate_spec_and_library(spec_id, library_id) - libraries: WorkflowSpecService.get_libraries() + libraries: workflow_spec_service.get_libraries() libraryids = [x.library_spec_id for x in libraries] if library_id in libraryids: raise ApiError('unknown_spec', 'The Library Specification "' + library_id + '" is already attached.') - spec = WorkflowSpecService.get_spec(spec_id) - library = WorkflowSpecService.get_spec(library_id) + spec = workflow_spec_service.get_spec(spec_id) + library = workflow_spec_service.get_spec(library_id) spec.libraries.push(library) - WorkflowSpecService.update_spec(spec_id) + workflow_spec_service.update_spec(spec_id) return spec def drop_workflow_spec_library(spec_id, library_id): validate_spec_and_library(spec_id, library_id) - spec = WorkflowSpecService.get_spec(spec_id) + spec = workflow_spec_service.get_spec(spec_id) # heres a piece of code that certainly wont work - library = WorkflowSpecService.get_spec(library_id) + library = workflow_spec_service.get_spec(library_id) if library in spec.libraries: spec.libraries.pop(library) - WorkflowSpecService.update_spec(spec_id) + workflow_spec_service.update_spec(spec_id) return spec @@ -118,7 +120,7 @@ def validate_workflow_specification(spec_id, study_id=None, test_until=None): def update_workflow_specification(spec_id, body): if spec_id is None: raise ApiError('unknown_spec', 'Please provide a valid Workflow Spec ID.') - spec = WorkflowSpecService.get_spec(spec_id) + spec = workflow_spec_service.get_spec(spec_id) if spec is None: raise ApiError('unknown_study', 'The spec "' + spec_id + '" is not recognized.') @@ -131,7 +133,7 @@ def update_workflow_specification(spec_id, body): if body['library'] is True or body['standalone'] is True: body['category_id'] = None - spec = WorkflowSpecService.update_spec(spec_id, body) + spec = workflow_spec_service.update_spec(spec_id, body) return spec @@ -139,7 +141,7 @@ def delete_workflow_specification(spec_id): if spec_id is None: raise ApiError('unknown_spec', 'Please provide a valid Workflow Specification ID.') - spec = WorkflowSpecService.get_spec(spec_id) + spec = workflow_spec_service.get_spec(spec_id) if spec is None: raise ApiError('unknown_spec', 'The Workflow Specification "' + spec_id + '" is not recognized.') @@ -155,7 +157,7 @@ def delete_workflow_specification(spec_id): WorkflowService.delete_workflow_spec_task_events(spec_id) # .delete() doesn't work when we need a cascade. Must grab the record, and explicitly delete - WorkflowSpecService.delete_spec(spec_id) + workflow_spec_service.delete_spec(spec_id) # Reorder the remaining specs WorkflowService.cleanup_workflow_spec_display_order(spec.category_id) @@ -165,9 +167,9 @@ def reorder_workflow_specification(spec_id, direction): if direction not in ('up', 'down'): raise ApiError(code='bad_direction', message='The direction must be `up` or `down`.') - spec = WorkflowSpecService.get_spec(spec_id) + spec = workflow_spec_service.get_spec(spec_id) if spec: - ordered_specs = WorkflowSpecService.reorder_spec(spec, direction) + ordered_specs = workflow_spec_service.reorder_spec(spec, direction) else: raise ApiError(code='bad_spec_id', message=f'The spec_id {spec_id} did not return a specification. Please check that it is valid.') @@ -322,22 +324,22 @@ def __update_task(processor, task, data, user): def list_workflow_spec_categories(): - return WorkflowSpecService.get_workflow_categories() + return workflow_spec_service.get_workflow_categories() def get_workflow_spec_category(cat_id): - return WorkflowSpecService.get_workflow_category(cat_id) + return workflow_spec_service.get_workflow_category(cat_id) def add_workflow_spec_category(body): - return WorkflowSpecService.add_category(body) + return workflow_spec_service.add_category(body) def update_workflow_spec_category(cat_id, body): if cat_id is None: raise ApiError('unknown_category', 'Please provide a valid Workflow Spec Category ID.') - category = WorkflowSpecService.update_category(cat_id, body) + category = workflow_spec_service.update_category(cat_id, body) if category is None: raise ApiError('unknown_category', 'The category "' + cat_id + '" is not recognized.') @@ -358,9 +360,9 @@ def reorder_workflow_spec_category(cat_id, direction): raise ApiError(code='bad_direction', message='The direction must be `up` or `down`.') WorkflowService.cleanup_workflow_spec_category_display_order() - category = WorkflowSpecService.get_category(cat_id) + category = workflow_spec_service.get_category(cat_id) if category: - ordered_categories = WorkflowSpecService.reorder_workflow_spec_category(category, direction) + ordered_categories = workflow_spec_service.reorder_workflow_spec_category(category, direction) return ordered_categories else: raise ApiError(code='bad_category_id', diff --git a/crc/services/workflow_spec_service.py b/crc/services/workflow_spec_service.py index 566b0da2..9a3085b9 100644 --- a/crc/services/workflow_spec_service.py +++ b/crc/services/workflow_spec_service.py @@ -29,7 +29,6 @@ class WorkflowSpecService(FileSystemService): self.master_spec = None self.libraries = {} self.standalone = {} - self.scan_file_system() def add_spec(self, spec: WorkflowSpecInfo): self.update_spec(spec) diff --git a/tests/base_test.py b/tests/base_test.py index 25104103..3478e10c 100644 --- a/tests/base_test.py +++ b/tests/base_test.py @@ -12,7 +12,7 @@ import datetime import shutil from flask import g -from crc import app, db, session, WorkflowSpecService +from crc import app, db, session from crc.models.api_models import WorkflowApiSchema, MultiInstanceType from crc.models.file import FileModel, CONTENT_TYPES from crc.models.task_event import TaskEventModel @@ -26,7 +26,7 @@ from crc.services.user_service import UserService from crc.services.workflow_service import WorkflowService from crc.services.document_service import DocumentService from example_data import ExampleDataLoader -from crc.services.user_file_service import UserFileService +from crc.services.workflow_spec_service import WorkflowSpecService # UNCOMMENT THIS FOR DEBUGGING SQL ALCHEMY QUERIES import logging diff --git a/tests/workflow/cr_connect.log b/tests/workflow/cr_connect.log index 4d6ed08a..62ea31a4 100644 --- a/tests/workflow/cr_connect.log +++ b/tests/workflow/cr_connect.log @@ -1,3 +1,5 @@ 2022-01-26 15:05:47,955 - connexion.apis.abstract - ERROR - Failed to add operation for GET /v1.0/workflow_sync/pullall 2022-01-26 15:06:36,303 - connexion.apis.abstract - ERROR - Failed to add operation for GET /v1.0/workflow_sync/pullall 2022-01-26 15:08:01,978 - connexion.apis.abstract - ERROR - Failed to add operation for GET /v1.0/file/{md5_hash}/hash_data +2022-02-07 12:46:21,875 - connexion.apis.abstract - ERROR - Failed to add operation for GET /v1.0/ldap +2022-02-07 12:49:13,793 - connexion.apis.abstract - ERROR - Failed to add operation for GET /v1.0/ldap diff --git a/tests/workflow/test_workflow_spec_service.py b/tests/workflow/test_workflow_spec_service.py index 92a61659..6c117e4d 100644 --- a/tests/workflow/test_workflow_spec_service.py +++ b/tests/workflow/test_workflow_spec_service.py @@ -7,13 +7,15 @@ from tests.base_test import BaseTest from crc.models.workflow import WorkflowSpecInfo, WorkflowSpecCategory from crc.services.file_system_service import FileSystemService from crc.services.spec_file_service import SpecFileService -from crc import db, app, workflow_spec_service +from crc.services.workflow_spec_service import WorkflowSpecService +from crc import db, app class TestWorkflowSync(BaseTest): spec_path = FileSystemService.root_path() import_spec_path = os.path.join(app.root_path, '..', 'tests', 'data', 'IMPORT_TEST') + service = WorkflowSpecService() def copy_files_to_file_system(self): """Some tests rely on a well populated file system """ @@ -34,24 +36,24 @@ class TestWorkflowSync(BaseTest): self.load_test_spec('email', category_id=c1.id, library=True) def test_from_file_system_blank_slate(self): - self.assertEquals(0, len(workflow_spec_service.get_categories())) - self.assertEquals(0, len(workflow_spec_service.get_specs())) + self.assertEquals(0, len(self.service.get_categories())) + self.assertEquals(0, len(self.service.get_specs())) self.copy_files_to_file_system() - workflow_spec_service.scan_file_system() - self.assertEquals(2, len(workflow_spec_service.get_categories())) - self.assertEquals(5, len(workflow_spec_service.get_specs())) - self.assertEquals(1, len(workflow_spec_service.get_category('Category Number One').workflows)) - self.assertEquals(2, len(workflow_spec_service.get_category('Category Number Two').workflows)) - self.assertIsNotNone(workflow_spec_service.master_spec) - self.assertEquals(1, len(workflow_spec_service.get_libraries())) - self.assertEquals(1, len(workflow_spec_service.master_spec.libraries)) + self.service.scan_file_system() + self.assertEquals(2, len(self.service.get_categories())) + self.assertEquals(5, len(self.service.get_specs())) + self.assertEquals(1, len(self.service.get_category('Category Number One').workflows)) + self.assertEquals(2, len(self.service.get_category('Category Number Two').workflows)) + self.assertIsNotNone(self.service.master_spec) + self.assertEquals(1, len(self.service.get_libraries())) + self.assertEquals(1, len(self.service.master_spec.libraries)) def test_delete_category_and_workflows(self): self.copy_files_to_file_system() cat_path = SpecFileService().category_path('Category Number One') shutil.rmtree(cat_path) # Remove the path, as if from a git pull and the path was removed. - self.assertEquals(3, len(workflow_spec_service.get_categories())) - self.assertEquals(4, len(workflow_spec_service.get_specs())) + self.assertEquals(3, len(self.service.get_categories())) + self.assertEquals(4, len(self.service.get_specs()))