From ff9b2694aedeb0d4bf9d28f26fa7778c27c84581 Mon Sep 17 00:00:00 2001 From: Dan Date: Mon, 7 Feb 2022 11:56:22 -0500 Subject: [PATCH] Fix migrations that depend on removed models. --- .../7225d990740e_move_files_to_filesystem.py | 4 +- migrations/versions/ac1141d29d37_.py | 2 +- tests/workflow/test_workflow_spec_service.py | 111 ++++++++++++++++++ 3 files changed, 115 insertions(+), 2 deletions(-) create mode 100644 tests/workflow/test_workflow_spec_service.py diff --git a/migrations/versions/7225d990740e_move_files_to_filesystem.py b/migrations/versions/7225d990740e_move_files_to_filesystem.py index e2334f45..365fd900 100644 --- a/migrations/versions/7225d990740e_move_files_to_filesystem.py +++ b/migrations/versions/7225d990740e_move_files_to_filesystem.py @@ -12,7 +12,6 @@ import sqlalchemy as sa # import crc from crc import app, session from crc.models.file import FileModel, FileModelSchema, FileDataModel, LookupFileModel, CONTENT_TYPES -from crc.models.workflow import WorkflowSpecModel, WorkflowSpecModelSchema, WorkflowSpecCategoryModel, WorkflowSpecCategoryModelSchema from crc.services.spec_file_service import SpecFileService from crc.services.reference_file_service import ReferenceFileService from crc.services.workflow_service import WorkflowService @@ -260,6 +259,9 @@ class ToFilesystemService(object): def upgrade(): + from crc.models.workflow import WorkflowSpecModel, WorkflowSpecModelSchema, WorkflowSpecCategoryModel, \ + WorkflowSpecCategoryModelSchema + """""" bind = op.get_bind() session = sa.orm.Session(bind=bind) diff --git a/migrations/versions/ac1141d29d37_.py b/migrations/versions/ac1141d29d37_.py index 465ed4f0..f66ccc0f 100644 --- a/migrations/versions/ac1141d29d37_.py +++ b/migrations/versions/ac1141d29d37_.py @@ -11,7 +11,6 @@ from alembic import op import sqlalchemy as sa # revision identifiers, used by Alembic. -from crc.models.workflow import WorkflowSpecModel, WorkflowModel revision = 'ac1141d29d37' down_revision = '8580676e5302' @@ -20,6 +19,7 @@ depends_on = None def upgrade(): + from crc.models.workflow import WorkflowSpecModel, WorkflowModel print("Doing the upgrade") op.execute('ALTER TABLE workflow DROP CONSTRAINT workflow_workflow_spec_id_fkey') op.execute('ALTER TABLE file DROP CONSTRAINT file_workflow_spec_id_fkey') diff --git a/tests/workflow/test_workflow_spec_service.py b/tests/workflow/test_workflow_spec_service.py new file mode 100644 index 00000000..861fba7a --- /dev/null +++ b/tests/workflow/test_workflow_spec_service.py @@ -0,0 +1,111 @@ +import json +import os +import shutil + +from tests.base_test import BaseTest + +from crc.services.workflow_sync import WorkflowSyncService +from crc.models.workflow import WorkflowSpecInfo, WorkflowSpecCategory +from crc.services.file_system_service import FileSystemService +from crc.services.spec_file_service import SpecFileService +from crc import db, app + + +class TestWorkflowSync(BaseTest): + + spec_path = FileSystemService.root_path() + import_spec_path = os.path.join(app.root_path, '..', 'tests', 'data', 'IMPORT_TEST') + + def set_up_file_system(self): + """Some tests rely on a well populated file system and an empty database to start""" + shutil.copytree(self.import_spec_path, self.spec_path) + + def set_up_database(self): + """Some tests rely on a well populated database and an empty file system to start""" + + # Construct Two Categories, with one workflow in first category, two in the second. + # assure that the data in categories.json is correct + # and that there is the correct data structure. + c1 = self.assure_category_name_exists("Category Number One") + c2 = self.assure_category_name_exists("Category Number Two") + self.load_test_spec('random_fact', category_id=c1.id) + self.load_test_spec('hello_world', category_id=c2.id) + self.load_test_spec('decision_table', category_id=c2.id) + self.load_test_spec('empty_workflow', category_id=c1.id, master_spec=True) + self.load_test_spec('email', category_id=c1.id, library=True) + + def test_from_file_system_blank_slate(self): + self.assertEquals(0, len(db.session.query(WorkflowSpecModel).all())) + self.assertEquals(0, len(db.session.query(WorkflowSpecCategoryModel).all())) + self.set_up_file_system() + WorkflowSyncService().from_file_system() + self.assertEquals(2, len(db.session.query(WorkflowSpecCategoryModel).all())) + self.assertEquals(5, len(db.session.query(WorkflowSpecModel).all())) + self.assertEquals(1, len(db.session.query(WorkflowSpecModel).filter(WorkflowSpecModel.category_id == 1).all())) + self.assertEquals(2, len(db.session.query(WorkflowSpecModel).filter(WorkflowSpecModel.category_id == 2).all())) + self.assertEquals(1, len(db.session.query(WorkflowSpecModel).filter(WorkflowSpecModel.is_master_spec).all())) + self.assertEquals(1, len(db.session.query(WorkflowSpecModel).filter(WorkflowSpecModel.library).all())) + # The top level workflow, has a library + tlw = db.session.query(WorkflowSpecModel).filter(WorkflowSpecModel.is_master_spec).first() + self.assertEquals(1, len(tlw.libraries)) + + def test_repeated_imports(self): + self.set_up_file_system() + WorkflowSyncService().from_file_system() + WorkflowSyncService().from_file_system() + WorkflowSyncService().from_file_system() + self.assertEquals(2, len(db.session.query(WorkflowSpecCategoryModel).all())) + self.assertEquals(5, len(db.session.query(WorkflowSpecModel).all())) + + def test_delete_category_and_workflows(self): + self.set_up_file_system() + WorkflowSyncService().from_file_system() + cat_path = SpecFileService().category_path('Category Number One') + shutil.rmtree(cat_path) + WorkflowSyncService().from_file_system() + self.assertEquals(1, len(db.session.query(WorkflowSpecCategoryModel).all())) + self.assertEquals(4, len(db.session.query(WorkflowSpecModel).all())) + self.assertEquals(0, len(db.session.query(WorkflowSpecModel).filter(WorkflowSpecModel.category_id == 1).all())) + + WorkflowSyncService().to_file_system() + json_path = os.path.join(FileSystemService.root_path(), "categories.json") + with open(json_path) as json_file: + data = json.load(json_file) + self.assertEquals(1, len(data['categories']), "When the json file is written back to disk, there is only one category now.") + + def test_to_file_system(self): + """Assure we get the basic paths on the file system that we would expect.""" + self.assertFalse(os.path.exists(self.spec_path)) + self.set_up_database() + self.assertEqual(4, len(os.listdir(self.spec_path)), "Adding workflows should create dir structure") + WorkflowSyncService().to_file_system() + self.assertEqual(5, len(os.listdir(self.spec_path)), "Sync service should create categories.json file") + + def test_to_file_system_correct_categories(self): + """Assure we have two categories in the json file, and that these directories exist, and contain + workflow.json files for each workflow.""" + self.set_up_database() + WorkflowSyncService().to_file_system() + json_path = os.path.join(self.spec_path, 'categories.json') + + with open(json_path) as json_file: + data = json.load(json_file) + self.assertTrue('categories' in data) + self.assertEqual(2, len(data['categories'])) + counter = 0 + for c in data['categories']: + cat_path = os.path.join(self.spec_path, c['display_name']) + self.assertTrue(os.path.exists(cat_path), "The category directories exist.") + self.assertEqual(data['categories'][counter]['display_order'], counter, "Order is correct") + counter += 1 + workflow_dirs = os.listdir(cat_path) + for wd in workflow_dirs: + wf_json_path = os.path.join(cat_path, wd, 'workflow.json') + self.assertTrue(os.path.exists(wf_json_path), "A workflow.json file should exist.") + # Fixme: Assure the master workflow spec, and Libraries are also exported to file system. + + + # Todo: + # * What if category json files, and directories don't match? + # * Test renaming a category + # * Test moving a workflow to a different category