From 88b662ea384260192ef2b4e7fe4c958f0049428b Mon Sep 17 00:00:00 2001 From: mike cullerton Date: Fri, 11 Feb 2022 12:17:32 -0500 Subject: [PATCH 01/10] modified get_remote_url to use configuration, instead of hard-coding the remote host --- config/default.py | 10 +++++----- crc/services/git_service.py | 5 ++--- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/config/default.py b/config/default.py index a916a6c6..710b6498 100644 --- a/config/default.py +++ b/config/default.py @@ -79,11 +79,11 @@ GITHUB_REPO = environ.get('GITHUB_REPO', None) TARGET_BRANCH = environ.get('TARGET_BRANCH', None) # Git settings, used by git_service -# The above Github settings--used in file_service, will likely be deprecated -# You can override these settings in instance/config -GIT_REMOTE_PATH = environ.get('GIT_REMOTE_PATH', None) -GIT_BRANCH = environ.get('GIT_BRANCH', None) -GIT_MERGE_BRANCH = environ.get('GIT_MERGE_BRANCH', None) # Developers can set this to 'all' in instance.config +# Among other things, we use these to build a remote URL like https://username:password@host/path.git +GIT_REMOTE_SERVER = environ.get('GIT_REMOTE_SERVER', None) # example: 'github.com' +GIT_REMOTE_PATH = environ.get('GIT_REMOTE_PATH', None) # example: 'sartography/crconnect-workflow-specs +GIT_BRANCH = environ.get('GIT_BRANCH', None) # example: 'main' +GIT_MERGE_BRANCH = environ.get('GIT_MERGE_BRANCH', None) # Example: 'staging' # Email configuration DEFAULT_SENDER = 'uvacrconnect@virginia.edu' diff --git a/crc/services/git_service.py b/crc/services/git_service.py index 38786bdd..d0672a21 100644 --- a/crc/services/git_service.py +++ b/crc/services/git_service.py @@ -29,11 +29,10 @@ class GitService(object): @staticmethod def get_remote_url(remote_path): - # we use github - # Note that the 'password' is a token generated by github, not the site password + host = app.config['GIT_REMOTE_SERVER'] username = app.config["GIT_USER_NAME"] password = app.config["GIT_USER_PASS"] - remote_url = f"https://{username}:{password}@github.com/{remote_path}.git" + remote_url = f"https://{username}:{password}@{host}/{remote_path}.git" return remote_url @staticmethod From bf3a75cfbb5c66a2840685d73b2c17b9fbf4cd84 Mon Sep 17 00:00:00 2001 From: mike cullerton Date: Fri, 11 Feb 2022 12:17:48 -0500 Subject: [PATCH 02/10] Added test --- tests/test_git_service.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/test_git_service.py b/tests/test_git_service.py index 14549a63..5fd11a76 100644 --- a/tests/test_git_service.py +++ b/tests/test_git_service.py @@ -1,5 +1,6 @@ from tests.base_test import BaseTest +from crc import app from crc.services.git_service import GitService from unittest.mock import patch, Mock, call @@ -57,6 +58,10 @@ class TestGitService(BaseTest): self.assertIn(call.index.commit('This is my comment'), method_calls) self.assertIn(call.remotes.origin.push(), method_calls) - # def test_pull_from_remote(self): - # result = GitService.pull_from_remote() - # print(result) + def test_get_remote_url(self): + app.config['GIT_REMOTE_SERVER'] = 'test_server.com' + app.config['GIT_USER_NAME'] = 'test_username' + app.config['GIT_USER_PASS'] = 'test_pass' + + result = GitService.get_remote_url('my_test_path') + self.assertEqual('https://test_username:test_pass@test_server.com/my_test_path.git', result) From 0e51def09fc2e71fa5f7356766cd02a524a9c950 Mon Sep 17 00:00:00 2001 From: mike cullerton Date: Fri, 11 Feb 2022 12:18:20 -0500 Subject: [PATCH 03/10] Cleanup - removed old unused methods --- crc/services/user_file_service.py | 60 ------------------------------- 1 file changed, 60 deletions(-) diff --git a/crc/services/user_file_service.py b/crc/services/user_file_service.py index da8e32fa..10e923d2 100644 --- a/crc/services/user_file_service.py +++ b/crc/services/user_file_service.py @@ -195,66 +195,6 @@ class UserFileService(object): app.logger.info("Failed to delete file, so archiving it instead. %i, due to %s" % (file_id, str(ie))) raise ApiError('Delete Failed', "Unable to delete file. ") - @staticmethod - def get_repo_branches(): - gh_token = app.config['GITHUB_TOKEN'] - github_repo = app.config['GITHUB_REPO'] - _github = Github(gh_token) - repo = _github.get_user().get_repo(github_repo) - branches = [branch.name for branch in repo.get_branches()] - return branches - - @staticmethod - def update_from_github(file_ids, source_target=GithubObject.NotSet): - gh_token = app.config['GITHUB_TOKEN'] - github_repo = app.config['GITHUB_REPO'] - _github = Github(gh_token) - repo = _github.get_user().get_repo(github_repo) - - for file_id in file_ids: - file_data_model = FileDataModel.query.filter_by( - file_model_id=file_id - ).order_by( - desc(FileDataModel.version) - ).first() - try: - repo_file = repo.get_contents(file_data_model.file_model.name, ref=source_target) - except UnknownObjectException: - return {'error': 'Attempted to update from repository but file was not present'} - else: - file_data_model.data = repo_file.decoded_content - session.add(file_data_model) - session.commit() - - @staticmethod - def publish_to_github(file_ids): - target_branch = app.config['TARGET_BRANCH'] if app.config['TARGET_BRANCH'] else GithubObject.NotSet - gh_token = app.config['GITHUB_TOKEN'] - github_repo = app.config['GITHUB_REPO'] - _github = Github(gh_token) - repo = _github.get_user().get_repo(github_repo) - for file_id in file_ids: - file_data_model = FileDataModel.query.filter_by(file_model_id=file_id).first() - try: - repo_file = repo.get_contents(file_data_model.file_model.name, ref=target_branch) - except UnknownObjectException: - repo.create_file( - path=file_data_model.file_model.name, - message=f'Creating {file_data_model.file_model.name}', - content=file_data_model.data, - branch=target_branch - ) - return {'created': True} - else: - updated = repo.update_file( - path=repo_file.path, - message=f'Updating {file_data_model.file_model.name}', - content=file_data_model.data + b'brah-model', - sha=repo_file.sha, - branch=target_branch - ) - return {'updated': True} - @staticmethod def dmn_from_spreadsheet(ss_data): From de797c4f3264f7a00855ac218fcf7743f4a04111 Mon Sep 17 00:00:00 2001 From: mike cullerton Date: Fri, 11 Feb 2022 12:45:29 -0500 Subject: [PATCH 04/10] Added examples for changes and untracked --- crc/api.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/crc/api.yml b/crc/api.yml index 0102b0e0..710a7c65 100755 --- a/crc/api.yml +++ b/crc/api.yml @@ -2381,10 +2381,9 @@ components: merge_branch: type: string example: staging -# status: -# type: string -# example: staging changes: type: array + example: ['file_1.txt', 'file_2.txt'] untracked: type: array + example: ['a_file.txt', 'b_file.txt'] From fca6eb158271498eea6a8055f4e197c2897c5a11 Mon Sep 17 00:00:00 2001 From: mike cullerton Date: Wed, 16 Feb 2022 15:54:30 -0500 Subject: [PATCH 05/10] Script to get workflow spec information from a workflow spec id --- crc/scripts/get_spec_from_id.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 crc/scripts/get_spec_from_id.py diff --git a/crc/scripts/get_spec_from_id.py b/crc/scripts/get_spec_from_id.py new file mode 100644 index 00000000..915d709d --- /dev/null +++ b/crc/scripts/get_spec_from_id.py @@ -0,0 +1,22 @@ +from crc.api.common import ApiError +from crc.models.workflow import WorkflowSpecInfoSchema +from crc.scripts.script import Script +from crc.services.workflow_spec_service import WorkflowSpecService + + +class ScriptTemplate(Script): + + def get_description(self): + return """Get workflow spec information from a workflow spec id""" + + def do_task_validate_only(self, task, study_id, workflow_id, *args, **kwargs): + return self.do_task(task, study_id, workflow_id, *args, **kwargs) + + def do_task(self, task, study_id, workflow_id, *args, **kwargs): + if len(args) < 1: + raise ApiError(code='missing_spec_id', + message='The get_spec_from_id script requires a spec_id.') + spec_id = args[0] + workflow_spec = WorkflowSpecService().get_spec(spec_id) + + return WorkflowSpecInfoSchema().dump(workflow_spec) From e74e44e8e5feae90b79a2ffe3dba3e9fc0b97ffe Mon Sep 17 00:00:00 2001 From: mike cullerton Date: Wed, 16 Feb 2022 15:54:56 -0500 Subject: [PATCH 06/10] Test for new script --- tests/data/spec_from_id/spec_from_id.bpmn | 75 +++++++++++++++++++++++ tests/scripts/test_get_spec_from_id.py | 15 +++++ 2 files changed, 90 insertions(+) create mode 100644 tests/data/spec_from_id/spec_from_id.bpmn create mode 100644 tests/scripts/test_get_spec_from_id.py diff --git a/tests/data/spec_from_id/spec_from_id.bpmn b/tests/data/spec_from_id/spec_from_id.bpmn new file mode 100644 index 00000000..2c104af1 --- /dev/null +++ b/tests/data/spec_from_id/spec_from_id.bpmn @@ -0,0 +1,75 @@ + + + + + Flow_08i8lxh + + + + + + Flow_0saprky + + + + + + + + + + + + + Flow_08i8lxh + Flow_1fhu6em + + + Flow_1fhu6em + Flow_06e4nx2 + spec = get_spec_from_id(spec_id) + + + ## Spec +{{ spec }} + + Flow_06e4nx2 + Flow_0saprky + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/scripts/test_get_spec_from_id.py b/tests/scripts/test_get_spec_from_id.py new file mode 100644 index 00000000..3cb7c3d9 --- /dev/null +++ b/tests/scripts/test_get_spec_from_id.py @@ -0,0 +1,15 @@ +from tests.base_test import BaseTest + + +class TestSpecFromWorkflowID(BaseTest): + + def test_get_spec_from_workflow_id(self): + workflow = self.create_workflow('spec_from_id') + workflow_spec_id = workflow.workflow_spec_id + workflow_api = self.get_workflow_api(workflow) + task = workflow_api.next_task + workflow_api = self.complete_form(workflow, task, {'spec_id': workflow_spec_id}) + task = workflow_api.next_task + + self.assertEqual('spec_from_id', task.data['spec']['id']) + self.assertEqual('spec_from_id', task.data['spec']['display_name']) From 7885481b840d09fc1ec69f30baa8c708e920dff9 Mon Sep 17 00:00:00 2001 From: mike cullerton Date: Wed, 16 Feb 2022 15:55:42 -0500 Subject: [PATCH 07/10] Ignore hidden files --- crc/services/file_system_service.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crc/services/file_system_service.py b/crc/services/file_system_service.py index e1dee313..c9453d23 100644 --- a/crc/services/file_system_service.py +++ b/crc/services/file_system_service.py @@ -100,8 +100,10 @@ class FileSystemService(object): items = os.scandir(file_path) for item in items: if item.is_file(): + if item.name.startswith('.'): + continue # Ignore hidden files if item.name == FileSystemService.WF_JSON_FILE: - continue # Ignore the json files. + continue # Ignore the json files. if file_name is not None and item.name != file_name: continue file = FileSystemService.to_file_object_from_dir_entry(item) From 31f724b0c35444399203ac49090c3f49d72148b6 Mon Sep 17 00:00:00 2001 From: mike cullerton Date: Thu, 17 Feb 2022 11:48:08 -0500 Subject: [PATCH 08/10] Seed date fields correctly during validation. We were seeding them with random strings. --- crc/services/workflow_service.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crc/services/workflow_service.py b/crc/services/workflow_service.py index 3872c0e8..8fa924fc 100755 --- a/crc/services/workflow_service.py +++ b/crc/services/workflow_service.py @@ -541,6 +541,8 @@ class WorkflowService(object): return FileSchema().dump(file) elif field.type == 'files': return random.randrange(1, 100) + elif field.type == 'date': + return datetime.utcnow() else: return WorkflowService._random_string() From 92eb8e9cb9d57f30eecb5249c6ef6bd8bc979643 Mon Sep 17 00:00:00 2001 From: mike cullerton Date: Thu, 17 Feb 2022 11:48:31 -0500 Subject: [PATCH 09/10] test for date field validation --- .../data/date_validation/date_validation.bpmn | 89 +++++++++++++++++++ .../test_workflow_date_field_validation.py | 12 +++ 2 files changed, 101 insertions(+) create mode 100644 tests/data/date_validation/date_validation.bpmn create mode 100644 tests/workflow/test_workflow_date_field_validation.py diff --git a/tests/data/date_validation/date_validation.bpmn b/tests/data/date_validation/date_validation.bpmn new file mode 100644 index 00000000..42e6d532 --- /dev/null +++ b/tests/data/date_validation/date_validation.bpmn @@ -0,0 +1,89 @@ + + + + + Flow_09e6w2a + + + + + + Flow_19hbirj + + + + + + + + + + + + + Flow_09e6w2a + Flow_0cbbsi7 + + + Flow_0cbbsi7 + Flow_0dvxkh6 + delta1 = timedelta(hours=2) +format = '%Y-%m-%dT%H:%M:%S.%fZ' +the_date = datetime.datetime.strptime(a_date, format) +modified_date = the_date + delta1 +del(delta1) + + + # Dates + + +## A Date +{{ a_date }} + + +## A Delta +{{ a_delta }} + + +## Modified Date +{{ modified_date }} + Flow_0dvxkh6 + Flow_19hbirj + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/workflow/test_workflow_date_field_validation.py b/tests/workflow/test_workflow_date_field_validation.py new file mode 100644 index 00000000..8f468cd6 --- /dev/null +++ b/tests/workflow/test_workflow_date_field_validation.py @@ -0,0 +1,12 @@ +from tests.base_test import BaseTest + + +class TestDateValidation(BaseTest): + + def test_date_validation(self): + + """We were not instantiating date fields correctly during validation. + This is a simple test to make sure we seed an actual date in date fields instead of a random string.""" + spec_model = self.load_test_spec('date_validation') + rv = self.app.get('/v1.0/workflow-specification/%s/validate' % spec_model.id, headers=self.logged_in_headers()) + self.assertEqual([], rv.json) From d3883323f97444052d6e05aefa68ceeade000dc4 Mon Sep 17 00:00:00 2001 From: mike cullerton Date: Thu, 17 Feb 2022 11:51:33 -0500 Subject: [PATCH 10/10] Simplified test workflow --- .../data/date_validation/date_validation.bpmn | 29 +++++++------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/tests/data/date_validation/date_validation.bpmn b/tests/data/date_validation/date_validation.bpmn index 42e6d532..8cf29260 100644 --- a/tests/data/date_validation/date_validation.bpmn +++ b/tests/data/date_validation/date_validation.bpmn @@ -36,38 +36,31 @@ del(delta1) # Dates - ## A Date {{ a_date }} - -## A Delta -{{ a_delta }} - - -## Modified Date -{{ modified_date }} + Flow_0dvxkh6 Flow_19hbirj - - - - - - - + + + - - - + + + + + + +