mirror of
https://github.com/sartography/cr-connect-workflow.git
synced 2025-02-23 13:18:35 +00:00
Merge branch 'feature/spec_files_wthout_db' of https://github.com/sartography/cr-connect-workflow into feature/spec_files_wthout_db
This commit is contained in:
mike cullerton
commit
b6d6e21360
@ -55,10 +55,8 @@ from crc import models
|
||||
from crc import api
|
||||
from crc.api import admin
|
||||
from crc.services.workflow_service import WorkflowService
|
||||
from crc.services.workflow_spec_service import WorkflowSpecService
|
||||
connexion_app.add_api('api.yml', base_path='/v1.0')
|
||||
|
||||
|
||||
# needed function to avoid circular import
|
||||
def process_waiting_tasks():
|
||||
with app.app_context():
|
||||
|
||||
@ -8,12 +8,15 @@ from flask import send_file
|
||||
import io
|
||||
import connexion
|
||||
|
||||
from crc.services.workflow_spec_service import WorkflowSpecService
|
||||
|
||||
workflow_spec_service = WorkflowSpecService()
|
||||
|
||||
def get_files(spec_id, include_libraries=False):
|
||||
if spec_id is None:
|
||||
raise ApiError(code='missing_spec_id',
|
||||
message='Please specify the workflow_spec_id.')
|
||||
workflow_spec = session.query(WorkflowSpecModel).filter_by(id=spec_id).first()
|
||||
workflow_spec = workflow_spec_service.get_spec(spec_id)
|
||||
if workflow_spec is None:
|
||||
raise ApiError(code='unknown_spec',
|
||||
message=f'Unknown Spec: {spec_id}')
|
||||
@ -23,7 +26,7 @@ def get_files(spec_id, include_libraries=False):
|
||||
|
||||
|
||||
def get_file(spec_id, file_name):
|
||||
workflow_spec = session.query(WorkflowSpecModel).filter_by(id=spec_id).first()
|
||||
workflow_spec = workflow_spec_service.get_spec(spec_id)
|
||||
files = SpecFileService.get_files(workflow_spec, file_name)
|
||||
if len(files) == 0:
|
||||
raise ApiError(code='unknown file',
|
||||
@ -33,16 +36,16 @@ def get_file(spec_id, file_name):
|
||||
|
||||
|
||||
def add_file(spec_id):
|
||||
workflow_spec = session.query(WorkflowSpecModel).filter_by(id=spec_id).first()
|
||||
workflow_spec = workflow_spec_service.get_spec(spec_id)
|
||||
file = connexion.request.files['file']
|
||||
file = SpecFileService.add_file(workflow_spec, file.filename, file.stream.read())
|
||||
if not WorkflowSpecModel.primary_process_id and file.type == FileType.bpmn:
|
||||
if not workflow_spec.primary_process_name and file.type == FileType.bpmn:
|
||||
SpecFileService.set_primary_bpmn(workflow_spec, file.name)
|
||||
return FileSchema().dump(file)
|
||||
|
||||
|
||||
def update(spec_id, file_name, is_primary):
|
||||
workflow_spec = session.query(WorkflowSpecModel).filter_by(id=spec_id).first()
|
||||
workflow_spec = workflow_spec_service.get_spec(spec_id)
|
||||
files = SpecFileService.get_files(workflow_spec, file_name)
|
||||
if len(files) < 1:
|
||||
raise ApiError(code='unknown file',
|
||||
@ -55,7 +58,7 @@ def update(spec_id, file_name, is_primary):
|
||||
|
||||
|
||||
def update_data(spec_id, file_name):
|
||||
workflow_spec_model = session.query(WorkflowSpecModel).filter_by(id=spec_id).first()
|
||||
workflow_spec_model = workflow_spec_service.get_spec(spec_id)
|
||||
if workflow_spec_model is None:
|
||||
raise ApiError(code='missing_spec',
|
||||
message=f'The workflow spec for id {spec_id} does not exist.')
|
||||
@ -65,10 +68,10 @@ def update_data(spec_id, file_name):
|
||||
|
||||
|
||||
def get_data(spec_id, file_name):
|
||||
workflow_spec_model = session.query(WorkflowSpecModel).filter_by(id=spec_id).first()
|
||||
file_data = SpecFileService.get_data(workflow_spec_model, file_name)
|
||||
workflow_spec = workflow_spec_service.get_spec(spec_id)
|
||||
file_data = SpecFileService.get_data(workflow_spec, file_name)
|
||||
if file_data is not None:
|
||||
file_info = SpecFileService.get_files(workflow_spec_model, file_name)[0]
|
||||
file_info = SpecFileService.get_files(workflow_spec, file_name)[0]
|
||||
return send_file(
|
||||
io.BytesIO(file_data),
|
||||
attachment_filename=file_name,
|
||||
@ -82,5 +85,5 @@ def get_data(spec_id, file_name):
|
||||
|
||||
|
||||
def delete(spec_id, file_name):
|
||||
workflow_spec_model = session.query(WorkflowSpecModel).filter_by(id=spec_id).first()
|
||||
SpecFileService.delete_file(workflow_spec_model, file_name)
|
||||
workflow_spec = workflow_spec_service.get_spec(spec_id)
|
||||
SpecFileService.delete_file(workflow_spec, file_name)
|
||||
|
||||
@ -2,7 +2,7 @@ import uuid
|
||||
|
||||
from flask import g
|
||||
|
||||
from crc import session, WorkflowSpecService
|
||||
from crc import session
|
||||
from crc.api.common import ApiError, ApiErrorSchema
|
||||
from crc.models.api_models import WorkflowApiSchema
|
||||
from crc.models.study import StudyModel, WorkflowMetadata, StudyStatus
|
||||
@ -18,46 +18,45 @@ from crc.services.task_logging_service import TaskLoggingService
|
||||
from crc.services.user_service import UserService
|
||||
from crc.services.workflow_processor import WorkflowProcessor
|
||||
from crc.services.workflow_service import WorkflowService
|
||||
from crc.services.workflow_spec_service import WorkflowSpecService
|
||||
|
||||
workflow_spec_service = WorkflowSpecService()
|
||||
|
||||
def all_specifications(libraries=False,standalone=False):
|
||||
if libraries and standalone:
|
||||
raise ApiError('inconceivable!', 'You should specify libraries or standalone, but not both')
|
||||
schema = WorkflowSpecModelSchema(many=True)
|
||||
|
||||
if libraries:
|
||||
return schema.dump(session.query(WorkflowSpecModel)\
|
||||
.filter(WorkflowSpecModel.library==True).all())
|
||||
return workflow_spec_service.get_libraries()
|
||||
|
||||
if standalone:
|
||||
return schema.dump(session.query(WorkflowSpecModel)\
|
||||
.filter(WorkflowSpecModel.standalone==True).all())
|
||||
return workflow_spec_service.get_standalones()
|
||||
|
||||
# return standard workflows (not library, not standalone)
|
||||
return schema.dump(session.query(WorkflowSpecModel)
|
||||
.filter((WorkflowSpecModel.library==False) | (
|
||||
WorkflowSpecModel.library==None))
|
||||
.filter((WorkflowSpecModel.standalone==False) | (
|
||||
WorkflowSpecModel.standalone==None))
|
||||
.all())
|
||||
categories = workflow_spec_service.get_categories()
|
||||
workflows = []
|
||||
for cat in categories:
|
||||
workflows.extend(cat.workflows)
|
||||
return workflows
|
||||
|
||||
|
||||
def add_workflow_specification(body):
|
||||
category_id = body['display_name']
|
||||
category_name = body['display_name']
|
||||
# TODO: set this spec to the last one in the spec list
|
||||
# WorkflowService.cleanup_workflow_spec_display_order(category_id)
|
||||
WorkflowService.cleanup_workflow_spec_display_order(category_name)
|
||||
|
||||
# Libraries and standalone workflows don't get a category_id
|
||||
if body['library'] is True or body['standalone'] is True:
|
||||
body['category_id'] = None
|
||||
|
||||
new_spec = WorkflowSpecService.add_spec(body)
|
||||
new_spec = workflow_spec_service.add_spec(body)
|
||||
return new_spec
|
||||
|
||||
|
||||
def get_workflow_specification(spec_id):
|
||||
if spec_id is None:
|
||||
raise ApiError('unknown_spec', 'Please provide a valid Workflow Specification ID.')
|
||||
spec = WorkflowSpecService.get_spec(spec_id)
|
||||
spec = workflow_spec_service.get_spec(spec_id)
|
||||
|
||||
if spec is None:
|
||||
raise ApiError('unknown_spec', 'The Workflow Specification "' + spec_id + '" is not recognized.')
|
||||
@ -70,8 +69,8 @@ def validate_spec_and_library(spec_id,library_id):
|
||||
if library_id is None:
|
||||
raise ApiError('unknown_spec', 'Please provide a valid Library Specification ID.')
|
||||
|
||||
spec = WorkflowSpecService.get_spec(spec_id)
|
||||
library = WorkflowSpecService.get_library(library_id);
|
||||
spec = workflow_spec_service.get_spec(spec_id)
|
||||
library = workflow_spec_service.get_library(library_id);
|
||||
if spec is None:
|
||||
raise ApiError('unknown_spec', 'The Workflow Specification "' + spec_id + '" is not recognized.')
|
||||
if library is None:
|
||||
@ -85,17 +84,25 @@ def add_workflow_spec_library(spec_id, library_id):
|
||||
libraries: workflow_spec_service.get_libraries()
|
||||
libraryids = [x.library_spec_id for x in libraries]
|
||||
if library_id in libraryids:
|
||||
raise ApiError('unknown_spec', 'The Library Specification "' + spec_id + '" is already attached.')
|
||||
raise ApiError('unknown_spec', 'The Library Specification "' + library_id + '" is already attached.')
|
||||
|
||||
# TODO: this used to return all libraries. maybe we still want that here
|
||||
newlib = WorkflowSpecService.add_library(spec_id, library_id)
|
||||
return newlib
|
||||
spec = workflow_spec_service.get_spec(spec_id)
|
||||
library = workflow_spec_service.get_spec(library_id)
|
||||
spec.libraries.push(library)
|
||||
workflow_spec_service.update_spec(spec_id)
|
||||
return spec
|
||||
|
||||
def drop_workflow_spec_library(spec_id,library_id):
|
||||
|
||||
def drop_workflow_spec_library(spec_id, library_id):
|
||||
validate_spec_and_library(spec_id, library_id)
|
||||
WorkflowSpecService.delete_library(library_id)
|
||||
# TODO: this used to return all libraries. maybe we still want that here
|
||||
return WorkflowSpecService.get_libraries()
|
||||
spec = workflow_spec_service.get_spec(spec_id)
|
||||
|
||||
# heres a piece of code that certainly wont work
|
||||
library = workflow_spec_service.get_spec(library_id)
|
||||
if library in spec.libraries:
|
||||
spec.libraries.pop(library)
|
||||
workflow_spec_service.update_spec(spec_id)
|
||||
return spec
|
||||
|
||||
|
||||
def validate_workflow_specification(spec_id, study_id=None, test_until=None):
|
||||
@ -113,7 +120,7 @@ def validate_workflow_specification(spec_id, study_id=None, test_until=None):
|
||||
def update_workflow_specification(spec_id, body):
|
||||
if spec_id is None:
|
||||
raise ApiError('unknown_spec', 'Please provide a valid Workflow Spec ID.')
|
||||
spec = WorkflowSpecService.get_spec(spec_id)
|
||||
spec = workflow_spec_service.get_spec(spec_id)
|
||||
|
||||
if spec is None:
|
||||
raise ApiError('unknown_study', 'The spec "' + spec_id + '" is not recognized.')
|
||||
@ -126,7 +133,7 @@ def update_workflow_specification(spec_id, body):
|
||||
if body['library'] is True or body['standalone'] is True:
|
||||
body['category_id'] = None
|
||||
|
||||
spec = WorkflowSpecService.update_spec(spec_id, body)
|
||||
spec = workflow_spec_service.update_spec(spec_id, body)
|
||||
return spec
|
||||
|
||||
|
||||
@ -134,7 +141,7 @@ def delete_workflow_specification(spec_id):
|
||||
if spec_id is None:
|
||||
raise ApiError('unknown_spec', 'Please provide a valid Workflow Specification ID.')
|
||||
|
||||
spec = WorkflowSpecService.get_spec(spec_id)
|
||||
spec = workflow_spec_service.get_spec(spec_id)
|
||||
|
||||
if spec is None:
|
||||
raise ApiError('unknown_spec', 'The Workflow Specification "' + spec_id + '" is not recognized.')
|
||||
@ -150,9 +157,8 @@ def delete_workflow_specification(spec_id):
|
||||
WorkflowService.delete_workflow_spec_task_events(spec_id)
|
||||
|
||||
# .delete() doesn't work when we need a cascade. Must grab the record, and explicitly delete
|
||||
WorkflowSpecService.delete_spec(spec_id)
|
||||
workflow_spec_service.delete_spec(spec_id)
|
||||
|
||||
# TODO: fix reordering
|
||||
# Reorder the remaining specs
|
||||
WorkflowService.cleanup_workflow_spec_display_order(spec.category_id)
|
||||
|
||||
@ -161,9 +167,9 @@ def reorder_workflow_specification(spec_id, direction):
|
||||
if direction not in ('up', 'down'):
|
||||
raise ApiError(code='bad_direction',
|
||||
message='The direction must be `up` or `down`.')
|
||||
spec = WorkflowSpecService.get_spec(spec_id)
|
||||
spec = workflow_spec_service.get_spec(spec_id)
|
||||
if spec:
|
||||
ordered_specs = WorkflowSpecService.reorder_spec(spec, direction)
|
||||
ordered_specs = workflow_spec_service.reorder_spec(spec, direction)
|
||||
else:
|
||||
raise ApiError(code='bad_spec_id',
|
||||
message=f'The spec_id {spec_id} did not return a specification. Please check that it is valid.')
|
||||
@ -318,22 +324,22 @@ def __update_task(processor, task, data, user):
|
||||
|
||||
|
||||
def list_workflow_spec_categories():
|
||||
return WorkflowSpecService.get_workflow_categories()
|
||||
return workflow_spec_service.get_workflow_categories()
|
||||
|
||||
|
||||
def get_workflow_spec_category(cat_id):
|
||||
return WorkflowSpecService.get_workflow_category(cat_id)
|
||||
return workflow_spec_service.get_workflow_category(cat_id)
|
||||
|
||||
|
||||
def add_workflow_spec_category(body):
|
||||
return WorkflowSpecService.add_category(body)
|
||||
return workflow_spec_service.add_category(body)
|
||||
|
||||
|
||||
def update_workflow_spec_category(cat_id, body):
|
||||
if cat_id is None:
|
||||
raise ApiError('unknown_category', 'Please provide a valid Workflow Spec Category ID.')
|
||||
|
||||
category = WorkflowSpecService.update_category(cat_id, body)
|
||||
category = workflow_spec_service.update_category(cat_id, body)
|
||||
|
||||
if category is None:
|
||||
raise ApiError('unknown_category', 'The category "' + cat_id + '" is not recognized.')
|
||||
@ -348,19 +354,16 @@ def update_workflow_spec_category(cat_id, body):
|
||||
def delete_workflow_spec_category(cat_id):
|
||||
workflow_spec_service.delete_category(cat_id)
|
||||
|
||||
|
||||
def reorder_workflow_spec_category(cat_id, direction):
|
||||
# TODO: fix reordering here too
|
||||
if direction not in ('up', 'down'):
|
||||
raise ApiError(code='bad_direction',
|
||||
message='The direction must be `up` or `down`.')
|
||||
WorkflowService.cleanup_workflow_spec_category_display_order()
|
||||
# TODO: modify this to get_workflow_category(category)
|
||||
category = session.query(WorkflowSpecCategoryModel).\
|
||||
filter(WorkflowSpecCategoryModel.id == cat_id).first()
|
||||
category = workflow_spec_service.get_category(cat_id)
|
||||
if category:
|
||||
ordered_categories = WorkflowService.reorder_workflow_spec_category(category, direction)
|
||||
schema = WorkflowSpecCategoryModelSchema(many=True)
|
||||
return schema.dump(ordered_categories)
|
||||
ordered_categories = workflow_spec_service.reorder_workflow_spec_category(category, direction)
|
||||
return ordered_categories
|
||||
else:
|
||||
raise ApiError(code='bad_category_id',
|
||||
message=f'The category id {cat_id} did not return a Workflow Spec Category. Make sure it is a valid ID.')
|
||||
|
||||
@ -14,7 +14,7 @@ class TaskEventModel(db.Model):
|
||||
study_id = db.Column(db.Integer, db.ForeignKey('study.id'))
|
||||
user_uid = db.Column(db.String, nullable=False) # In some cases the unique user id may not exist in the db yet.
|
||||
workflow_id = db.Column(db.Integer, db.ForeignKey('workflow.id'), nullable=False)
|
||||
workflow_spec_id = db.Column(db.String, db.ForeignKey('workflow_spec.id'))
|
||||
workflow_spec_id = db.Column(db.String)
|
||||
spec_version = db.Column(db.String)
|
||||
action = db.Column(db.String)
|
||||
task_id = db.Column(db.String)
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
import enum
|
||||
|
||||
from marshmallow import EXCLUDE
|
||||
from marshmallow import EXCLUDE, post_load
|
||||
from sqlalchemy import func
|
||||
|
||||
from crc import db, ma
|
||||
@ -19,12 +19,15 @@ class WorkflowSpecCategorySchema(ma.Schema):
|
||||
model = WorkflowSpecCategory
|
||||
fields = ["id", "display_name", "display_order", "admin"]
|
||||
|
||||
@post_load
|
||||
def make_cat(self, data, **kwargs):
|
||||
return WorkflowSpecCategory(**data)
|
||||
|
||||
|
||||
class WorkflowSpecInfo(object):
|
||||
def __init__(self, id, display_name, description, category_name,
|
||||
display_order, is_master_spec,
|
||||
standalone, library, primary_file_name, primary_process_id, is_review,
|
||||
libraries):
|
||||
def __init__(self, id, display_name, description, is_master_spec,
|
||||
standalone, library, primary_file_name, primary_process_id,
|
||||
libraries, category_name=None, display_order=0, is_review=False):
|
||||
self.id = id # Sting unique id
|
||||
self.display_name = display_name
|
||||
self.description = description
|
||||
@ -45,9 +48,12 @@ class WorkflowSpecInfoSchema(ma.Schema):
|
||||
model = WorkflowSpecInfo
|
||||
fields = ["id", "display_name", "description", "category_id", "is_master_spec,",
|
||||
"standalone", "library", "primary_file_name", "primary_process_id", "is_review",
|
||||
"libraries"]
|
||||
"libraries", "category_name", "display_order", "is_master_spec", "is_review"]
|
||||
unknown = EXCLUDE
|
||||
|
||||
@post_load
|
||||
def make_spec(self, data, **kwargs):
|
||||
return WorkflowSpecInfo(**data)
|
||||
|
||||
class WorkflowState(enum.Enum):
|
||||
hidden = "hidden"
|
||||
@ -79,7 +85,7 @@ class WorkflowModel(db.Model):
|
||||
status = db.Column(db.Enum(WorkflowStatus))
|
||||
study_id = db.Column(db.Integer, db.ForeignKey('study.id'))
|
||||
study = db.relationship("StudyModel", backref='workflow')
|
||||
workflow_spec_id = db.Column(db.String, db.ForeignKey('workflow_spec.id'))
|
||||
workflow_spec_id = db.Column(db.String)
|
||||
total_tasks = db.Column(db.Integer, default=0)
|
||||
completed_tasks = db.Column(db.Integer, default=0)
|
||||
last_updated = db.Column(db.DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
@ -29,7 +29,6 @@ class WorkflowSpecService(FileSystemService):
|
||||
self.master_spec = None
|
||||
self.libraries = {}
|
||||
self.standalone = {}
|
||||
self.scan_file_system()
|
||||
|
||||
def add_spec(self, spec: WorkflowSpecInfo):
|
||||
self.update_spec(spec)
|
||||
@ -54,6 +53,9 @@ class WorkflowSpecService(FileSystemService):
|
||||
raise ApiError('unknown spec', 'unable to find a spec with id:' + spec_id)
|
||||
return self.specs[spec_id]
|
||||
|
||||
def get_specs(self):
|
||||
return list(self.specs.values())
|
||||
|
||||
def reorder_spec(self, spec:WorkflowSpecInfo, direction):
|
||||
workflows = spec.category.workflows
|
||||
workflows.sort(key=lambda w: w.display_order)
|
||||
@ -70,23 +72,45 @@ class WorkflowSpecService(FileSystemService):
|
||||
return workflows
|
||||
|
||||
def get_libraries(self) -> List[WorkflowSpecInfo]:
|
||||
# fixme
|
||||
spec_list = list(self.libraries.values())
|
||||
spec_list.sort(key=lambda w: w.display_order)
|
||||
return spec_list
|
||||
|
||||
def get_standalones(self) -> List[WorkflowSpecInfo]:
|
||||
spec_list = list(self.standalone.values())
|
||||
spec_list.sort(key=lambda w: w.display_order)
|
||||
return spec_list
|
||||
|
||||
def get_standalones(self) -> List[WorkflowSpecInfo]:
|
||||
pass
|
||||
|
||||
def get_categories(self) -> List[WorkflowSpecCategory]:
|
||||
pass
|
||||
"""Returns the categories as a list in display order"""
|
||||
cat_list = list(self.categories.values())
|
||||
cat_list.sort(key=lambda w: w.display_order)
|
||||
return cat_list
|
||||
|
||||
def get_category(self, category_id) -> WorkflowSpecCategory:
|
||||
pass
|
||||
if category_id not in self.categories:
|
||||
raise ApiError('unknown category', 'unable to find a category with id:' + category_id)
|
||||
return self.categories[category_id]
|
||||
|
||||
def add_category(self, category: WorkflowSpecCategory):
|
||||
pass
|
||||
self.update_category(category)
|
||||
|
||||
def update_category(self, category: WorkflowSpecCategory):
|
||||
pass
|
||||
cat_path = self.category_path(category.display_name)
|
||||
os.makedirs(os.path.dirname(cat_path), exist_ok=True)
|
||||
json_path = os.path.join(cat_path, self.CAT_JSON_FILE)
|
||||
with open(json_path, "w") as cat_json:
|
||||
json.dump(self.CAT_SCHEMA.dump(category), cat_json, indent=4)
|
||||
self.scan_file_system()
|
||||
|
||||
def delete_category(self, category: WorkflowSpecCategory):
|
||||
pass
|
||||
def delete_category(self, category_id: str):
|
||||
if category_id in self.specs:
|
||||
path = self.category_path(category_id)
|
||||
shutil.rmtree(path)
|
||||
self.scan_file_system()
|
||||
|
||||
def reorder_workflow_spec_category(self, spec:WorkflowSpecInfo, direction):
|
||||
# Fixme: Resort Workflow categories
|
||||
@ -101,18 +125,20 @@ class WorkflowSpecService(FileSystemService):
|
||||
self.master_spec = None
|
||||
self.libraries = {}
|
||||
if not os.path.exists(FileSystemService.root_path()):
|
||||
raise ApiError('missing_specs', 'The path for workflow specifications does not exist.')
|
||||
return # Nothing to scan yet. There are no files.
|
||||
directory_items = os.scandir(FileSystemService.root_path())
|
||||
for item in directory_items:
|
||||
if item.is_dir():
|
||||
if item.name == self.LIBRARY_SPECS:
|
||||
self.scan_category(item, is_library=True)
|
||||
if item.name == self.STAND_ALONE_SPECS:
|
||||
self.scan_category(item, is_standalone=True)
|
||||
elif item.name == self.MASTER_SPECIFICATION:
|
||||
self.scan_spec(item, is_master=True)
|
||||
else:
|
||||
self.scan_category(item)
|
||||
|
||||
def scan_category(self, dir_item: os.DirEntry, is_library=False):
|
||||
def scan_category(self, dir_item: os.DirEntry, is_library=False, is_standalone=False):
|
||||
"""Reads the category.json file, and any workflow directories """
|
||||
cat_path = os.path.join(dir_item.path, self.CAT_JSON_FILE)
|
||||
if os.path.exists(cat_path):
|
||||
@ -125,6 +151,8 @@ class WorkflowSpecService(FileSystemService):
|
||||
json.dump(self.CAT_SCHEMA.dump(cat), wf_json, indent=4)
|
||||
if is_library:
|
||||
self.libraries = cat
|
||||
elif is_standalone:
|
||||
self.standalone = cat
|
||||
else:
|
||||
self.categories[cat.id] = cat
|
||||
workflow_dirs = os.scandir(FileSystemService.root_path())
|
||||
@ -141,7 +169,6 @@ class WorkflowSpecService(FileSystemService):
|
||||
# workflow_metas.append(WorkflowMetadata.from_workflow(workflow))
|
||||
return workflow_metas
|
||||
|
||||
|
||||
def scan_spec(self, dir_item: os.DirEntry, is_master=False, category=None):
|
||||
if not is_master and not category:
|
||||
raise ApiError("invalid_spec_dir", "Please specify what category this workflow belongs to.")
|
||||
@ -153,15 +180,16 @@ class WorkflowSpecService(FileSystemService):
|
||||
else:
|
||||
spec = WorkflowSpecInfo(id=dir_item.name, library=False, standalone=False, is_master_spec=is_master,
|
||||
display_name=dir_item.name, description="", primary_process_id="",
|
||||
primary_file_name="")
|
||||
primary_file_name="", category_name="", display_order=0, is_review=False,
|
||||
libraries=[])
|
||||
with open(spec_path, "w") as wf_json:
|
||||
json.dump(WorkflowSpecInfoSchema.dump(spec), wf_json, indent=4)
|
||||
json.dump(self.WF_SCHEMA.dump(spec), wf_json, indent=4)
|
||||
if is_master:
|
||||
self.master_spec = spec
|
||||
elif category:
|
||||
self.specs[spec.id] = spec
|
||||
spec.category = category
|
||||
category.workflow_specs.append(spec)
|
||||
category.workflows.append(spec)
|
||||
|
||||
def set_primary_bpmn(self, workflow_spec: WorkflowSpecInfo, file_name: str, binary_data=None):
|
||||
# If this is a BPMN, extract the process id, and determine if it is contains swim lanes.
|
||||
|
||||
@ -12,7 +12,7 @@ import datetime
|
||||
import shutil
|
||||
from flask import g
|
||||
|
||||
from crc import app, db, session, WorkflowSpecService
|
||||
from crc import app, db, session
|
||||
from crc.models.api_models import WorkflowApiSchema, MultiInstanceType
|
||||
from crc.models.file import FileModel, CONTENT_TYPES
|
||||
from crc.models.task_event import TaskEventModel
|
||||
@ -26,7 +26,7 @@ from crc.services.user_service import UserService
|
||||
from crc.services.workflow_service import WorkflowService
|
||||
from crc.services.document_service import DocumentService
|
||||
from example_data import ExampleDataLoader
|
||||
from crc.services.user_file_service import UserFileService
|
||||
from crc.services.workflow_spec_service import WorkflowSpecService
|
||||
|
||||
# UNCOMMENT THIS FOR DEBUGGING SQL ALCHEMY QUERIES
|
||||
import logging
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
{
|
||||
"id": 2,
|
||||
"admin": false,
|
||||
"display_order": 1
|
||||
"display_order": 1,
|
||||
"display_name": "Category Number Two"
|
||||
}
|
||||
|
||||
@ -1,3 +1,5 @@
|
||||
2022-01-26 15:05:47,955 - connexion.apis.abstract - ERROR - Failed to add operation for GET /v1.0/workflow_sync/pullall
|
||||
2022-01-26 15:06:36,303 - connexion.apis.abstract - ERROR - Failed to add operation for GET /v1.0/workflow_sync/pullall
|
||||
2022-01-26 15:08:01,978 - connexion.apis.abstract - ERROR - Failed to add operation for GET /v1.0/file/{md5_hash}/hash_data
|
||||
2022-02-07 12:46:21,875 - connexion.apis.abstract - ERROR - Failed to add operation for GET /v1.0/ldap
|
||||
2022-02-07 12:49:13,793 - connexion.apis.abstract - ERROR - Failed to add operation for GET /v1.0/ldap
|
||||
|
||||
@ -4,10 +4,10 @@ import shutil
|
||||
|
||||
from tests.base_test import BaseTest
|
||||
|
||||
from crc.services.workflow_sync import WorkflowSyncService
|
||||
from crc.models.workflow import WorkflowSpecInfo, WorkflowSpecCategory
|
||||
from crc.services.file_system_service import FileSystemService
|
||||
from crc.services.spec_file_service import SpecFileService
|
||||
from crc.services.workflow_spec_service import WorkflowSpecService
|
||||
from crc import db, app
|
||||
|
||||
|
||||
@ -15,13 +15,14 @@ class TestWorkflowSync(BaseTest):
|
||||
|
||||
spec_path = FileSystemService.root_path()
|
||||
import_spec_path = os.path.join(app.root_path, '..', 'tests', 'data', 'IMPORT_TEST')
|
||||
service = WorkflowSpecService()
|
||||
|
||||
def set_up_file_system(self):
|
||||
"""Some tests rely on a well populated file system and an empty database to start"""
|
||||
def copy_files_to_file_system(self):
|
||||
"""Some tests rely on a well populated file system """
|
||||
shutil.copytree(self.import_spec_path, self.spec_path)
|
||||
|
||||
def set_up_database(self):
|
||||
"""Some tests rely on a well populated database and an empty file system to start"""
|
||||
def build_file_system_from_models(self):
|
||||
"""Some tests check to see what happens when we write data to an empty file system."""
|
||||
|
||||
# Construct Two Categories, with one workflow in first category, two in the second.
|
||||
# assure that the data in categories.json is correct
|
||||
@ -35,77 +36,24 @@ class TestWorkflowSync(BaseTest):
|
||||
self.load_test_spec('email', category_id=c1.id, library=True)
|
||||
|
||||
def test_from_file_system_blank_slate(self):
|
||||
self.assertEquals(0, len(db.session.query(WorkflowSpecModel).all()))
|
||||
self.assertEquals(0, len(db.session.query(WorkflowSpecCategoryModel).all()))
|
||||
self.set_up_file_system()
|
||||
WorkflowSyncService().from_file_system()
|
||||
self.assertEquals(2, len(db.session.query(WorkflowSpecCategoryModel).all()))
|
||||
self.assertEquals(5, len(db.session.query(WorkflowSpecModel).all()))
|
||||
self.assertEquals(1, len(db.session.query(WorkflowSpecModel).filter(WorkflowSpecModel.category_id == 1).all()))
|
||||
self.assertEquals(2, len(db.session.query(WorkflowSpecModel).filter(WorkflowSpecModel.category_id == 2).all()))
|
||||
self.assertEquals(1, len(db.session.query(WorkflowSpecModel).filter(WorkflowSpecModel.is_master_spec).all()))
|
||||
self.assertEquals(1, len(db.session.query(WorkflowSpecModel).filter(WorkflowSpecModel.library).all()))
|
||||
# The top level workflow, has a library
|
||||
tlw = db.session.query(WorkflowSpecModel).filter(WorkflowSpecModel.is_master_spec).first()
|
||||
self.assertEquals(1, len(tlw.libraries))
|
||||
|
||||
def test_repeated_imports(self):
|
||||
self.set_up_file_system()
|
||||
WorkflowSyncService().from_file_system()
|
||||
WorkflowSyncService().from_file_system()
|
||||
WorkflowSyncService().from_file_system()
|
||||
self.assertEquals(2, len(db.session.query(WorkflowSpecCategoryModel).all()))
|
||||
self.assertEquals(5, len(db.session.query(WorkflowSpecModel).all()))
|
||||
self.assertEquals(0, len(self.service.get_categories()))
|
||||
self.assertEquals(0, len(self.service.get_specs()))
|
||||
self.copy_files_to_file_system()
|
||||
self.service.scan_file_system()
|
||||
self.assertEquals(2, len(self.service.get_categories()))
|
||||
self.assertEquals(5, len(self.service.get_specs()))
|
||||
self.assertEquals(1, len(self.service.get_category('Category Number One').workflows))
|
||||
self.assertEquals(2, len(self.service.get_category('Category Number Two').workflows))
|
||||
self.assertIsNotNone(self.service.master_spec)
|
||||
self.assertEquals(1, len(self.service.get_libraries()))
|
||||
self.assertEquals(1, len(self.service.master_spec.libraries))
|
||||
|
||||
def test_delete_category_and_workflows(self):
|
||||
self.set_up_file_system()
|
||||
WorkflowSyncService().from_file_system()
|
||||
self.copy_files_to_file_system()
|
||||
cat_path = SpecFileService().category_path('Category Number One')
|
||||
shutil.rmtree(cat_path)
|
||||
WorkflowSyncService().from_file_system()
|
||||
self.assertEquals(1, len(db.session.query(WorkflowSpecCategoryModel).all()))
|
||||
self.assertEquals(4, len(db.session.query(WorkflowSpecModel).all()))
|
||||
self.assertEquals(0, len(db.session.query(WorkflowSpecModel).filter(WorkflowSpecModel.category_id == 1).all()))
|
||||
|
||||
WorkflowSyncService().to_file_system()
|
||||
json_path = os.path.join(FileSystemService.root_path(), "categories.json")
|
||||
with open(json_path) as json_file:
|
||||
data = json.load(json_file)
|
||||
self.assertEquals(1, len(data['categories']), "When the json file is written back to disk, there is only one category now.")
|
||||
|
||||
def test_to_file_system(self):
|
||||
"""Assure we get the basic paths on the file system that we would expect."""
|
||||
self.assertFalse(os.path.exists(self.spec_path))
|
||||
self.set_up_database()
|
||||
self.assertEqual(4, len(os.listdir(self.spec_path)), "Adding workflows should create dir structure")
|
||||
WorkflowSyncService().to_file_system()
|
||||
self.assertEqual(5, len(os.listdir(self.spec_path)), "Sync service should create categories.json file")
|
||||
|
||||
def test_to_file_system_correct_categories(self):
|
||||
"""Assure we have two categories in the json file, and that these directories exist, and contain
|
||||
workflow.json files for each workflow."""
|
||||
self.set_up_database()
|
||||
WorkflowSyncService().to_file_system()
|
||||
json_path = os.path.join(self.spec_path, 'categories.json')
|
||||
|
||||
with open(json_path) as json_file:
|
||||
data = json.load(json_file)
|
||||
self.assertTrue('categories' in data)
|
||||
self.assertEqual(2, len(data['categories']))
|
||||
counter = 0
|
||||
for c in data['categories']:
|
||||
cat_path = os.path.join(self.spec_path, c['display_name'])
|
||||
self.assertTrue(os.path.exists(cat_path), "The category directories exist.")
|
||||
self.assertEqual(data['categories'][counter]['display_order'], counter, "Order is correct")
|
||||
counter += 1
|
||||
workflow_dirs = os.listdir(cat_path)
|
||||
for wd in workflow_dirs:
|
||||
wf_json_path = os.path.join(cat_path, wd, 'workflow.json')
|
||||
self.assertTrue(os.path.exists(wf_json_path), "A workflow.json file should exist.")
|
||||
# Fixme: Assure the master workflow spec, and Libraries are also exported to file system.
|
||||
shutil.rmtree(cat_path) # Remove the path, as if from a git pull and the path was removed.
|
||||
self.assertEquals(3, len(self.service.get_categories()))
|
||||
self.assertEquals(4, len(self.service.get_specs()))
|
||||
|
||||
|
||||
|
||||
# Todo:
|
||||
# * What if category json files, and directories don't match?
|
||||
# * Test renaming a category
|
||||
# * Test moving a workflow to a different category
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user