sartography
/
cr-connect-workflow
mirror of https://github.com/sartography/cr-connect-workflow.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

WIP - its just caos right now.

This commit is contained in:
Dan 2022-02-07 12:59:48 -05:00
parent ad095aaaf5
commit 056f901f9a
6 changed files with 59 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
crc/api/spec_file.py
View File

@ -1,4 +1,4 @@
from crc import session, WorkflowSpecService
from crc import session
from crc.api.common import ApiError
from crc.models.file import FileSchema, FileType
from crc.services.spec_file_service import SpecFileService
@ -8,12 +8,15 @@ from flask import send_file
import io
import connexion
from crc.services.workflow_spec_service import WorkflowSpecService
workflow_spec_service = WorkflowSpecService()
def get_files(spec_id, include_libraries=False):
if spec_id is None:
raise ApiError(code='missing_spec_id',
message='Please specify the workflow_spec_id.')
workflow_spec = WorkflowSpecService.get_spec(spec_id)
workflow_spec = workflow_spec_service.get_spec(spec_id)
if workflow_spec is None:
raise ApiError(code='unknown_spec',
message=f'Unknown Spec: {spec_id}')
@ -23,7 +26,7 @@ def get_files(spec_id, include_libraries=False):
def get_file(spec_id, file_name):
workflow_spec = WorkflowSpecService.get_spec(spec_id)
workflow_spec = workflow_spec_service.get_spec(spec_id)
files = SpecFileService.get_files(workflow_spec, file_name)
if len(files) == 0:
raise ApiError(code='unknown file',
@ -33,7 +36,7 @@ def get_file(spec_id, file_name):
def add_file(spec_id):
workflow_spec = WorkflowSpecService.get_spec(spec_id)
workflow_spec = workflow_spec_service.get_spec(spec_id)
file = connexion.request.files['file']
file = SpecFileService.add_file(workflow_spec, file.filename, file.stream.read())
if not workflow_spec.primary_process_name and file.type == FileType.bpmn:
@ -42,7 +45,7 @@ def add_file(spec_id):
def update(spec_id, file_name, is_primary):
workflow_spec = WorkflowSpecService.get_spec(spec_id)
workflow_spec = workflow_spec_service.get_spec(spec_id)
files = SpecFileService.get_files(workflow_spec, file_name)
if len(files) < 1:
raise ApiError(code='unknown file',
@ -55,7 +58,7 @@ def update(spec_id, file_name, is_primary):
def update_data(spec_id, file_name):
workflow_spec_model = WorkflowSpecService.get_spec(spec_id)
workflow_spec_model = workflow_spec_service.get_spec(spec_id)
if workflow_spec_model is None:
raise ApiError(code='missing_spec',
message=f'The workflow spec for id {spec_id} does not exist.')
@ -65,7 +68,7 @@ def update_data(spec_id, file_name):
def get_data(spec_id, file_name):
workflow_spec = WorkflowSpecService.get_spec(spec_id)
workflow_spec = workflow_spec_service.get_spec(spec_id)
file_data = SpecFileService.get_data(workflow_spec, file_name)
if file_data is not None:
file_info = SpecFileService.get_files(workflow_spec, file_name)[0]
@ -82,5 +85,5 @@ def get_data(spec_id, file_name):
def delete(spec_id, file_name):
workflow_spec = WorkflowSpecService.get_spec(spec_id)
workflow_spec = workflow_spec_service.get_spec(spec_id)
SpecFileService.delete_file(workflow_spec, file_name)

56
crc/api/workflow.py
View File

@ -2,7 +2,7 @@ import uuid
from flask import g
from crc import session, WorkflowSpecService
from crc import session
from crc.api.common import ApiError, ApiErrorSchema
from crc.models.api_models import WorkflowApiSchema
from crc.models.study import StudyModel, WorkflowMetadata, StudyStatus
@ -18,20 +18,22 @@ from crc.services.task_logging_service import TaskLoggingService
from crc.services.user_service import UserService
from crc.services.workflow_processor import WorkflowProcessor
from crc.services.workflow_service import WorkflowService
from crc.services.workflow_spec_service import WorkflowSpecService
workflow_spec_service = WorkflowSpecService()
def all_specifications(libraries=False,standalone=False):
if libraries and standalone:
raise ApiError('inconceivable!', 'You should specify libraries or standalone, but not both')
if libraries:
return WorkflowSpecService.get_libraries()
return workflow_spec_service.get_libraries()
if standalone:
return WorkflowSpecService.get_standalones()
return workflow_spec_service.get_standalones()
# return standard workflows (not library, not standalone)
categories = WorkflowSpecService.get_categories()
categories = workflow_spec_service.get_categories()
workflows = []
for cat in categories:
workflows.extend(cat.workflows)
@ -47,14 +49,14 @@ def add_workflow_specification(body):
if body['library'] is True or body['standalone'] is True:
body['category_id'] = None
new_spec = WorkflowSpecService.add_spec(body)
new_spec = workflow_spec_service.add_spec(body)
return new_spec
def get_workflow_specification(spec_id):
if spec_id is None:
raise ApiError('unknown_spec', 'Please provide a valid Workflow Specification ID.')
spec = WorkflowSpecService.get_spec(spec_id)
spec = workflow_spec_service.get_spec(spec_id)
if spec is None:
raise ApiError('unknown_spec', 'The Workflow Specification "' + spec_id + '" is not recognized.')
@ -67,8 +69,8 @@ def validate_spec_and_library(spec_id,library_id):
if library_id is None:
raise ApiError('unknown_spec', 'Please provide a valid Library Specification ID.')
spec = WorkflowSpecService.get_spec(spec_id)
library = WorkflowSpecService.get_library(library_id);
spec = workflow_spec_service.get_spec(spec_id)
library = workflow_spec_service.get_library(library_id);
if spec is None:
raise ApiError('unknown_spec', 'The Workflow Specification "' + spec_id + '" is not recognized.')
if library is None:
@ -79,27 +81,27 @@ def validate_spec_and_library(spec_id,library_id):
def add_workflow_spec_library(spec_id, library_id):
validate_spec_and_library(spec_id, library_id)
libraries: WorkflowSpecService.get_libraries()
libraries: workflow_spec_service.get_libraries()
libraryids = [x.library_spec_id for x in libraries]
if library_id in libraryids:
raise ApiError('unknown_spec', 'The Library Specification "' + library_id + '" is already attached.')
spec = WorkflowSpecService.get_spec(spec_id)
library = WorkflowSpecService.get_spec(library_id)
spec = workflow_spec_service.get_spec(spec_id)
library = workflow_spec_service.get_spec(library_id)
spec.libraries.push(library)
WorkflowSpecService.update_spec(spec_id)
workflow_spec_service.update_spec(spec_id)
return spec
def drop_workflow_spec_library(spec_id, library_id):
validate_spec_and_library(spec_id, library_id)
spec = WorkflowSpecService.get_spec(spec_id)
spec = workflow_spec_service.get_spec(spec_id)
# heres a piece of code that certainly wont work
library = WorkflowSpecService.get_spec(library_id)
library = workflow_spec_service.get_spec(library_id)
if library in spec.libraries:
spec.libraries.pop(library)
WorkflowSpecService.update_spec(spec_id)
workflow_spec_service.update_spec(spec_id)
return spec
@ -118,7 +120,7 @@ def validate_workflow_specification(spec_id, study_id=None, test_until=None):
def update_workflow_specification(spec_id, body):
if spec_id is None:
raise ApiError('unknown_spec', 'Please provide a valid Workflow Spec ID.')
spec = WorkflowSpecService.get_spec(spec_id)
spec = workflow_spec_service.get_spec(spec_id)
if spec is None:
raise ApiError('unknown_study', 'The spec "' + spec_id + '" is not recognized.')
@ -131,7 +133,7 @@ def update_workflow_specification(spec_id, body):
if body['library'] is True or body['standalone'] is True:
body['category_id'] = None
spec = WorkflowSpecService.update_spec(spec_id, body)
spec = workflow_spec_service.update_spec(spec_id, body)
return spec
@ -139,7 +141,7 @@ def delete_workflow_specification(spec_id):
if spec_id is None:
raise ApiError('unknown_spec', 'Please provide a valid Workflow Specification ID.')
spec = WorkflowSpecService.get_spec(spec_id)
spec = workflow_spec_service.get_spec(spec_id)
if spec is None:
raise ApiError('unknown_spec', 'The Workflow Specification "' + spec_id + '" is not recognized.')
@ -155,7 +157,7 @@ def delete_workflow_specification(spec_id):
WorkflowService.delete_workflow_spec_task_events(spec_id)
# .delete() doesn't work when we need a cascade. Must grab the record, and explicitly delete
WorkflowSpecService.delete_spec(spec_id)
workflow_spec_service.delete_spec(spec_id)
# Reorder the remaining specs
WorkflowService.cleanup_workflow_spec_display_order(spec.category_id)
@ -165,9 +167,9 @@ def reorder_workflow_specification(spec_id, direction):
if direction not in ('up', 'down'):
raise ApiError(code='bad_direction',
message='The direction must be `up` or `down`.')
spec = WorkflowSpecService.get_spec(spec_id)
spec = workflow_spec_service.get_spec(spec_id)
if spec:
ordered_specs = WorkflowSpecService.reorder_spec(spec, direction)
ordered_specs = workflow_spec_service.reorder_spec(spec, direction)
else:
raise ApiError(code='bad_spec_id',
message=f'The spec_id {spec_id} did not return a specification. Please check that it is valid.')
@ -322,22 +324,22 @@ def __update_task(processor, task, data, user):
def list_workflow_spec_categories():
return WorkflowSpecService.get_workflow_categories()
return workflow_spec_service.get_workflow_categories()
def get_workflow_spec_category(cat_id):
return WorkflowSpecService.get_workflow_category(cat_id)
return workflow_spec_service.get_workflow_category(cat_id)
def add_workflow_spec_category(body):
return WorkflowSpecService.add_category(body)
return workflow_spec_service.add_category(body)
def update_workflow_spec_category(cat_id, body):
if cat_id is None:
raise ApiError('unknown_category', 'Please provide a valid Workflow Spec Category ID.')
category = WorkflowSpecService.update_category(cat_id, body)
category = workflow_spec_service.update_category(cat_id, body)
if category is None:
raise ApiError('unknown_category', 'The category "' + cat_id + '" is not recognized.')
@ -358,9 +360,9 @@ def reorder_workflow_spec_category(cat_id, direction):
raise ApiError(code='bad_direction',
message='The direction must be `up` or `down`.')
WorkflowService.cleanup_workflow_spec_category_display_order()
category = WorkflowSpecService.get_category(cat_id)
category = workflow_spec_service.get_category(cat_id)
if category:
ordered_categories = WorkflowSpecService.reorder_workflow_spec_category(category, direction)
ordered_categories = workflow_spec_service.reorder_workflow_spec_category(category, direction)
return ordered_categories
else:
raise ApiError(code='bad_category_id',

1
crc/services/workflow_spec_service.py
View File

@ -29,7 +29,6 @@ class WorkflowSpecService(FileSystemService):
self.master_spec = None
self.libraries = {}
self.standalone = {}
self.scan_file_system()
def add_spec(self, spec: WorkflowSpecInfo):
self.update_spec(spec)

4
tests/base_test.py
View File

@ -12,7 +12,7 @@ import datetime
import shutil
from flask import g
from crc import app, db, session, WorkflowSpecService
from crc import app, db, session
from crc.models.api_models import WorkflowApiSchema, MultiInstanceType
from crc.models.file import FileModel, CONTENT_TYPES
from crc.models.task_event import TaskEventModel
@ -26,7 +26,7 @@ from crc.services.user_service import UserService
from crc.services.workflow_service import WorkflowService
from crc.services.document_service import DocumentService
from example_data import ExampleDataLoader
from crc.services.user_file_service import UserFileService
from crc.services.workflow_spec_service import WorkflowSpecService
# UNCOMMENT THIS FOR DEBUGGING SQL ALCHEMY QUERIES
import logging

2
tests/workflow/cr_connect.log
View File

@ -1,3 +1,5 @@
2022-01-26 15:05:47,955 - connexion.apis.abstract - ERROR - Failed to add operation for GET /v1.0/workflow_sync/pullall
2022-01-26 15:06:36,303 - connexion.apis.abstract - ERROR - Failed to add operation for GET /v1.0/workflow_sync/pullall
2022-01-26 15:08:01,978 - connexion.apis.abstract - ERROR - Failed to add operation for GET /v1.0/file/{md5_hash}/hash_data
2022-02-07 12:46:21,875 - connexion.apis.abstract - ERROR - Failed to add operation for GET /v1.0/ldap
2022-02-07 12:49:13,793 - connexion.apis.abstract - ERROR - Failed to add operation for GET /v1.0/ldap

28
tests/workflow/test_workflow_spec_service.py
View File

@ -7,13 +7,15 @@ from tests.base_test import BaseTest
from crc.models.workflow import WorkflowSpecInfo, WorkflowSpecCategory
from crc.services.file_system_service import FileSystemService
from crc.services.spec_file_service import SpecFileService
from crc import db, app, workflow_spec_service
from crc.services.workflow_spec_service import WorkflowSpecService
from crc import db, app
class TestWorkflowSync(BaseTest):
spec_path = FileSystemService.root_path()
import_spec_path = os.path.join(app.root_path, '..', 'tests', 'data', 'IMPORT_TEST')
service = WorkflowSpecService()
def copy_files_to_file_system(self):
"""Some tests rely on a well populated file system """
@ -34,24 +36,24 @@ class TestWorkflowSync(BaseTest):
self.load_test_spec('email', category_id=c1.id, library=True)
def test_from_file_system_blank_slate(self):
self.assertEquals(0, len(workflow_spec_service.get_categories()))
self.assertEquals(0, len(workflow_spec_service.get_specs()))
self.assertEquals(0, len(self.service.get_categories()))
self.assertEquals(0, len(self.service.get_specs()))
self.copy_files_to_file_system()
workflow_spec_service.scan_file_system()
self.assertEquals(2, len(workflow_spec_service.get_categories()))
self.assertEquals(5, len(workflow_spec_service.get_specs()))
self.assertEquals(1, len(workflow_spec_service.get_category('Category Number One').workflows))
self.assertEquals(2, len(workflow_spec_service.get_category('Category Number Two').workflows))
self.assertIsNotNone(workflow_spec_service.master_spec)
self.assertEquals(1, len(workflow_spec_service.get_libraries()))
self.assertEquals(1, len(workflow_spec_service.master_spec.libraries))
self.service.scan_file_system()
self.assertEquals(2, len(self.service.get_categories()))
self.assertEquals(5, len(self.service.get_specs()))
self.assertEquals(1, len(self.service.get_category('Category Number One').workflows))
self.assertEquals(2, len(self.service.get_category('Category Number Two').workflows))
self.assertIsNotNone(self.service.master_spec)
self.assertEquals(1, len(self.service.get_libraries()))
self.assertEquals(1, len(self.service.master_spec.libraries))
def test_delete_category_and_workflows(self):
self.copy_files_to_file_system()
cat_path = SpecFileService().category_path('Category Number One')
shutil.rmtree(cat_path) # Remove the path, as if from a git pull and the path was removed.
self.assertEquals(3, len(workflow_spec_service.get_categories()))
self.assertEquals(4, len(workflow_spec_service.get_specs()))
self.assertEquals(3, len(self.service.get_categories()))
self.assertEquals(4, len(self.service.get_specs()))