Merge remote-tracking branch 'origin/dev' into get-erroring-workflows-574
This commit is contained in:
Dan
commit
e0a8e7300c
|
|
@ -79,11 +79,11 @@ GITHUB_REPO = environ.get('GITHUB_REPO', None)
|
|||
TARGET_BRANCH = environ.get('TARGET_BRANCH', None)
|
||||
|
||||
# Git settings, used by git_service
|
||||
# The above Github settings--used in file_service, will likely be deprecated
|
||||
# You can override these settings in instance/config
|
||||
GIT_REMOTE_PATH = environ.get('GIT_REMOTE_PATH', None)
|
||||
GIT_BRANCH = environ.get('GIT_BRANCH', None)
|
||||
GIT_MERGE_BRANCH = environ.get('GIT_MERGE_BRANCH', None) # Developers can set this to 'all' in instance.config
|
||||
# Among other things, we use these to build a remote URL like https://username:password@host/path.git
|
||||
GIT_REMOTE_SERVER = environ.get('GIT_REMOTE_SERVER', None) # example: 'github.com'
|
||||
GIT_REMOTE_PATH = environ.get('GIT_REMOTE_PATH', None) # example: 'sartography/crconnect-workflow-specs
|
||||
GIT_BRANCH = environ.get('GIT_BRANCH', None) # example: 'main'
|
||||
GIT_MERGE_BRANCH = environ.get('GIT_MERGE_BRANCH', None) # Example: 'staging'
|
||||
|
||||
# Email configuration
|
||||
DEFAULT_SENDER = 'uvacrconnect@virginia.edu'
|
||||
|
|
|
|||
|
|
@ -2381,10 +2381,9 @@ components:
|
|||
merge_branch:
|
||||
type: string
|
||||
example: staging
|
||||
# status:
|
||||
# type: string
|
||||
# example: staging
|
||||
changes:
|
||||
type: array
|
||||
example: ['file_1.txt', 'file_2.txt']
|
||||
untracked:
|
||||
type: array
|
||||
example: ['a_file.txt', 'b_file.txt']
|
||||
|
|
|
|||
|
|
@ -0,0 +1,22 @@
|
|||
from crc.api.common import ApiError
|
||||
from crc.models.workflow import WorkflowSpecInfoSchema
|
||||
from crc.scripts.script import Script
|
||||
from crc.services.workflow_spec_service import WorkflowSpecService
|
||||
|
||||
|
||||
class ScriptTemplate(Script):
|
||||
|
||||
def get_description(self):
|
||||
return """Get workflow spec information from a workflow spec id"""
|
||||
|
||||
def do_task_validate_only(self, task, study_id, workflow_id, *args, **kwargs):
|
||||
return self.do_task(task, study_id, workflow_id, *args, **kwargs)
|
||||
|
||||
def do_task(self, task, study_id, workflow_id, *args, **kwargs):
|
||||
if len(args) < 1:
|
||||
raise ApiError(code='missing_spec_id',
|
||||
message='The get_spec_from_id script requires a spec_id.')
|
||||
spec_id = args[0]
|
||||
workflow_spec = WorkflowSpecService().get_spec(spec_id)
|
||||
|
||||
return WorkflowSpecInfoSchema().dump(workflow_spec)
|
||||
|
|
@ -104,8 +104,10 @@ class FileSystemService(object):
|
|||
items = os.scandir(file_path)
|
||||
for item in items:
|
||||
if item.is_file():
|
||||
if item.name.startswith('.'):
|
||||
continue # Ignore hidden files
|
||||
if item.name == FileSystemService.WF_JSON_FILE:
|
||||
continue # Ignore the json files.
|
||||
continue # Ignore the json files.
|
||||
if file_name is not None and item.name != file_name:
|
||||
continue
|
||||
file = FileSystemService.to_file_object_from_dir_entry(item)
|
||||
|
|
|
|||
|
|
@ -29,11 +29,10 @@ class GitService(object):
|
|||
|
||||
@staticmethod
|
||||
def get_remote_url(remote_path):
|
||||
# we use github
|
||||
# Note that the 'password' is a token generated by github, not the site password
|
||||
host = app.config['GIT_REMOTE_SERVER']
|
||||
username = app.config["GIT_USER_NAME"]
|
||||
password = app.config["GIT_USER_PASS"]
|
||||
remote_url = f"https://{username}:{password}@github.com/{remote_path}.git"
|
||||
remote_url = f"https://{username}:{password}@{host}/{remote_path}.git"
|
||||
return remote_url
|
||||
|
||||
@staticmethod
|
||||
|
|
|
|||
|
|
@ -195,66 +195,6 @@ class UserFileService(object):
|
|||
app.logger.info("Failed to delete file, so archiving it instead. %i, due to %s" % (file_id, str(ie)))
|
||||
raise ApiError('Delete Failed', "Unable to delete file. ")
|
||||
|
||||
@staticmethod
|
||||
def get_repo_branches():
|
||||
gh_token = app.config['GITHUB_TOKEN']
|
||||
github_repo = app.config['GITHUB_REPO']
|
||||
_github = Github(gh_token)
|
||||
repo = _github.get_user().get_repo(github_repo)
|
||||
branches = [branch.name for branch in repo.get_branches()]
|
||||
return branches
|
||||
|
||||
@staticmethod
|
||||
def update_from_github(file_ids, source_target=GithubObject.NotSet):
|
||||
gh_token = app.config['GITHUB_TOKEN']
|
||||
github_repo = app.config['GITHUB_REPO']
|
||||
_github = Github(gh_token)
|
||||
repo = _github.get_user().get_repo(github_repo)
|
||||
|
||||
for file_id in file_ids:
|
||||
file_data_model = FileDataModel.query.filter_by(
|
||||
file_model_id=file_id
|
||||
).order_by(
|
||||
desc(FileDataModel.version)
|
||||
).first()
|
||||
try:
|
||||
repo_file = repo.get_contents(file_data_model.file_model.name, ref=source_target)
|
||||
except UnknownObjectException:
|
||||
return {'error': 'Attempted to update from repository but file was not present'}
|
||||
else:
|
||||
file_data_model.data = repo_file.decoded_content
|
||||
session.add(file_data_model)
|
||||
session.commit()
|
||||
|
||||
@staticmethod
|
||||
def publish_to_github(file_ids):
|
||||
target_branch = app.config['TARGET_BRANCH'] if app.config['TARGET_BRANCH'] else GithubObject.NotSet
|
||||
gh_token = app.config['GITHUB_TOKEN']
|
||||
github_repo = app.config['GITHUB_REPO']
|
||||
_github = Github(gh_token)
|
||||
repo = _github.get_user().get_repo(github_repo)
|
||||
for file_id in file_ids:
|
||||
file_data_model = FileDataModel.query.filter_by(file_model_id=file_id).first()
|
||||
try:
|
||||
repo_file = repo.get_contents(file_data_model.file_model.name, ref=target_branch)
|
||||
except UnknownObjectException:
|
||||
repo.create_file(
|
||||
path=file_data_model.file_model.name,
|
||||
message=f'Creating {file_data_model.file_model.name}',
|
||||
content=file_data_model.data,
|
||||
branch=target_branch
|
||||
)
|
||||
return {'created': True}
|
||||
else:
|
||||
updated = repo.update_file(
|
||||
path=repo_file.path,
|
||||
message=f'Updating {file_data_model.file_model.name}',
|
||||
content=file_data_model.data + b'brah-model',
|
||||
sha=repo_file.sha,
|
||||
branch=target_branch
|
||||
)
|
||||
return {'updated': True}
|
||||
|
||||
@staticmethod
|
||||
def dmn_from_spreadsheet(ss_data):
|
||||
|
||||
|
|
|
|||
|
|
@ -591,6 +591,8 @@ class WorkflowService(object):
|
|||
return FileSchema().dump(file)
|
||||
elif field.type == 'files':
|
||||
return random.randrange(1, 100)
|
||||
elif field.type == 'date':
|
||||
return datetime.utcnow()
|
||||
else:
|
||||
return WorkflowService._random_string()
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,82 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<bpmn:definitions xmlns:bpmn="http://www.omg.org/spec/BPMN/20100524/MODEL" xmlns:bpmndi="http://www.omg.org/spec/BPMN/20100524/DI" xmlns:dc="http://www.omg.org/spec/DD/20100524/DC" xmlns:camunda="http://camunda.org/schema/1.0/bpmn" xmlns:di="http://www.omg.org/spec/DD/20100524/DI" id="Definitions_19xdwix" targetNamespace="http://bpmn.io/schema/bpmn" exporter="Camunda Modeler" exporterVersion="4.2.0">
|
||||
<bpmn:process id="Process_1wfi0e5" isExecutable="true">
|
||||
<bpmn:startEvent id="StartEvent_1">
|
||||
<bpmn:outgoing>Flow_09e6w2a</bpmn:outgoing>
|
||||
</bpmn:startEvent>
|
||||
<bpmn:sequenceFlow id="Flow_09e6w2a" sourceRef="StartEvent_1" targetRef="Activity_1lg09if" />
|
||||
<bpmn:sequenceFlow id="Flow_0cbbsi7" sourceRef="Activity_1lg09if" targetRef="Activity_05yevzg" />
|
||||
<bpmn:sequenceFlow id="Flow_0dvxkh6" sourceRef="Activity_05yevzg" targetRef="Activity_0phz7ks" />
|
||||
<bpmn:endEvent id="Event_1ieukoa">
|
||||
<bpmn:incoming>Flow_19hbirj</bpmn:incoming>
|
||||
</bpmn:endEvent>
|
||||
<bpmn:sequenceFlow id="Flow_19hbirj" sourceRef="Activity_0phz7ks" targetRef="Event_1ieukoa" />
|
||||
<bpmn:userTask id="Activity_1lg09if" name="Get Date" camunda:formKey="DateForm">
|
||||
<bpmn:extensionElements>
|
||||
<camunda:formData>
|
||||
<camunda:formField id="a_date" label="A Date" type="date">
|
||||
<camunda:validation>
|
||||
<camunda:constraint name="required" config="True" />
|
||||
</camunda:validation>
|
||||
</camunda:formField>
|
||||
</camunda:formData>
|
||||
</bpmn:extensionElements>
|
||||
<bpmn:incoming>Flow_09e6w2a</bpmn:incoming>
|
||||
<bpmn:outgoing>Flow_0cbbsi7</bpmn:outgoing>
|
||||
</bpmn:userTask>
|
||||
<bpmn:scriptTask id="Activity_05yevzg" name="Modify Date">
|
||||
<bpmn:incoming>Flow_0cbbsi7</bpmn:incoming>
|
||||
<bpmn:outgoing>Flow_0dvxkh6</bpmn:outgoing>
|
||||
<bpmn:script>delta1 = timedelta(hours=2)
|
||||
format = '%Y-%m-%dT%H:%M:%S.%fZ'
|
||||
the_date = datetime.datetime.strptime(a_date, format)
|
||||
modified_date = the_date + delta1
|
||||
del(delta1)</bpmn:script>
|
||||
</bpmn:scriptTask>
|
||||
<bpmn:manualTask id="Activity_0phz7ks" name="Display Dates">
|
||||
<bpmn:documentation># Dates
|
||||
|
||||
## A Date
|
||||
{{ a_date }}
|
||||
|
||||
</bpmn:documentation>
|
||||
<bpmn:incoming>Flow_0dvxkh6</bpmn:incoming>
|
||||
<bpmn:outgoing>Flow_19hbirj</bpmn:outgoing>
|
||||
</bpmn:manualTask>
|
||||
</bpmn:process>
|
||||
<bpmndi:BPMNDiagram id="BPMNDiagram_1">
|
||||
<bpmndi:BPMNPlane id="BPMNPlane_1" bpmnElement="Process_1wfi0e5">
|
||||
<bpmndi:BPMNEdge id="Flow_19hbirj_di" bpmnElement="Flow_19hbirj">
|
||||
<di:waypoint x="690" y="117" />
|
||||
<di:waypoint x="752" y="117" />
|
||||
</bpmndi:BPMNEdge>
|
||||
<bpmndi:BPMNEdge id="Flow_0dvxkh6_di" bpmnElement="Flow_0dvxkh6">
|
||||
<di:waypoint x="530" y="117" />
|
||||
<di:waypoint x="590" y="117" />
|
||||
</bpmndi:BPMNEdge>
|
||||
<bpmndi:BPMNEdge id="Flow_0cbbsi7_di" bpmnElement="Flow_0cbbsi7">
|
||||
<di:waypoint x="370" y="117" />
|
||||
<di:waypoint x="430" y="117" />
|
||||
</bpmndi:BPMNEdge>
|
||||
<bpmndi:BPMNEdge id="Flow_09e6w2a_di" bpmnElement="Flow_09e6w2a">
|
||||
<di:waypoint x="215" y="117" />
|
||||
<di:waypoint x="270" y="117" />
|
||||
</bpmndi:BPMNEdge>
|
||||
<bpmndi:BPMNShape id="_BPMNShape_StartEvent_2" bpmnElement="StartEvent_1">
|
||||
<dc:Bounds x="179" y="99" width="36" height="36" />
|
||||
</bpmndi:BPMNShape>
|
||||
<bpmndi:BPMNShape id="Event_1ieukoa_di" bpmnElement="Event_1ieukoa">
|
||||
<dc:Bounds x="752" y="99" width="36" height="36" />
|
||||
</bpmndi:BPMNShape>
|
||||
<bpmndi:BPMNShape id="Activity_05be3cs_di" bpmnElement="Activity_1lg09if">
|
||||
<dc:Bounds x="270" y="77" width="100" height="80" />
|
||||
</bpmndi:BPMNShape>
|
||||
<bpmndi:BPMNShape id="Activity_1s4snzz_di" bpmnElement="Activity_05yevzg">
|
||||
<dc:Bounds x="430" y="77" width="100" height="80" />
|
||||
</bpmndi:BPMNShape>
|
||||
<bpmndi:BPMNShape id="Activity_15nchcr_di" bpmnElement="Activity_0phz7ks">
|
||||
<dc:Bounds x="590" y="77" width="100" height="80" />
|
||||
</bpmndi:BPMNShape>
|
||||
</bpmndi:BPMNPlane>
|
||||
</bpmndi:BPMNDiagram>
|
||||
</bpmn:definitions>
|
||||
|
|
@ -0,0 +1,75 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<bpmn:definitions xmlns:bpmn="http://www.omg.org/spec/BPMN/20100524/MODEL" xmlns:bpmndi="http://www.omg.org/spec/BPMN/20100524/DI" xmlns:dc="http://www.omg.org/spec/DD/20100524/DC" xmlns:camunda="http://camunda.org/schema/1.0/bpmn" xmlns:di="http://www.omg.org/spec/DD/20100524/DI" id="Definitions_daa6ad0" targetNamespace="http://bpmn.io/schema/bpmn" exporter="Camunda Modeler" exporterVersion="3.0.0-dev">
|
||||
<bpmn:process id="Process_GetSpecFromID" name="Get Spec From ID" isExecutable="true">
|
||||
<bpmn:startEvent id="StartEvent_1">
|
||||
<bpmn:outgoing>Flow_08i8lxh</bpmn:outgoing>
|
||||
</bpmn:startEvent>
|
||||
<bpmn:sequenceFlow id="Flow_08i8lxh" sourceRef="StartEvent_1" targetRef="Activity_GetSpecID" />
|
||||
<bpmn:sequenceFlow id="Flow_1fhu6em" sourceRef="Activity_GetSpecID" targetRef="Activity_GetSpec" />
|
||||
<bpmn:sequenceFlow id="Flow_06e4nx2" sourceRef="Activity_GetSpec" targetRef="Activity_DisplaySpec" />
|
||||
<bpmn:endEvent id="Event_1o8wpdp">
|
||||
<bpmn:incoming>Flow_0saprky</bpmn:incoming>
|
||||
</bpmn:endEvent>
|
||||
<bpmn:sequenceFlow id="Flow_0saprky" sourceRef="Activity_DisplaySpec" targetRef="Event_1o8wpdp" />
|
||||
<bpmn:userTask id="Activity_GetSpecID" name="Get Spec ID" camunda:formKey="SpecIDForm">
|
||||
<bpmn:extensionElements>
|
||||
<camunda:formData>
|
||||
<camunda:formField id="spec_id" label="Spec ID" type="string">
|
||||
<camunda:validation>
|
||||
<camunda:constraint name="required" config="True" />
|
||||
</camunda:validation>
|
||||
</camunda:formField>
|
||||
</camunda:formData>
|
||||
</bpmn:extensionElements>
|
||||
<bpmn:incoming>Flow_08i8lxh</bpmn:incoming>
|
||||
<bpmn:outgoing>Flow_1fhu6em</bpmn:outgoing>
|
||||
</bpmn:userTask>
|
||||
<bpmn:scriptTask id="Activity_GetSpec" name="Get Spec">
|
||||
<bpmn:incoming>Flow_1fhu6em</bpmn:incoming>
|
||||
<bpmn:outgoing>Flow_06e4nx2</bpmn:outgoing>
|
||||
<bpmn:script>spec = get_spec_from_id(spec_id)</bpmn:script>
|
||||
</bpmn:scriptTask>
|
||||
<bpmn:manualTask id="Activity_DisplaySpec" name="Display Spec">
|
||||
<bpmn:documentation>## Spec
|
||||
{{ spec }}
|
||||
</bpmn:documentation>
|
||||
<bpmn:incoming>Flow_06e4nx2</bpmn:incoming>
|
||||
<bpmn:outgoing>Flow_0saprky</bpmn:outgoing>
|
||||
</bpmn:manualTask>
|
||||
</bpmn:process>
|
||||
<bpmndi:BPMNDiagram id="BPMNDiagram_1">
|
||||
<bpmndi:BPMNPlane id="BPMNPlane_1" bpmnElement="Process_GetSpecFromID">
|
||||
<bpmndi:BPMNEdge id="Flow_0saprky_di" bpmnElement="Flow_0saprky">
|
||||
<di:waypoint x="690" y="177" />
|
||||
<di:waypoint x="752" y="177" />
|
||||
</bpmndi:BPMNEdge>
|
||||
<bpmndi:BPMNEdge id="Flow_06e4nx2_di" bpmnElement="Flow_06e4nx2">
|
||||
<di:waypoint x="530" y="177" />
|
||||
<di:waypoint x="590" y="177" />
|
||||
</bpmndi:BPMNEdge>
|
||||
<bpmndi:BPMNEdge id="Flow_1fhu6em_di" bpmnElement="Flow_1fhu6em">
|
||||
<di:waypoint x="370" y="177" />
|
||||
<di:waypoint x="430" y="177" />
|
||||
</bpmndi:BPMNEdge>
|
||||
<bpmndi:BPMNEdge id="Flow_08i8lxh_di" bpmnElement="Flow_08i8lxh">
|
||||
<di:waypoint x="215" y="177" />
|
||||
<di:waypoint x="270" y="177" />
|
||||
</bpmndi:BPMNEdge>
|
||||
<bpmndi:BPMNShape id="_BPMNShape_StartEvent_2" bpmnElement="StartEvent_1">
|
||||
<dc:Bounds x="179" y="159" width="36" height="36" />
|
||||
</bpmndi:BPMNShape>
|
||||
<bpmndi:BPMNShape id="Event_1o8wpdp_di" bpmnElement="Event_1o8wpdp">
|
||||
<dc:Bounds x="752" y="159" width="36" height="36" />
|
||||
</bpmndi:BPMNShape>
|
||||
<bpmndi:BPMNShape id="Activity_0dq9ius_di" bpmnElement="Activity_GetSpecID">
|
||||
<dc:Bounds x="270" y="137" width="100" height="80" />
|
||||
</bpmndi:BPMNShape>
|
||||
<bpmndi:BPMNShape id="Activity_0ux2vk8_di" bpmnElement="Activity_GetSpec">
|
||||
<dc:Bounds x="430" y="137" width="100" height="80" />
|
||||
</bpmndi:BPMNShape>
|
||||
<bpmndi:BPMNShape id="Activity_0fzla1m_di" bpmnElement="Activity_DisplaySpec">
|
||||
<dc:Bounds x="590" y="137" width="100" height="80" />
|
||||
</bpmndi:BPMNShape>
|
||||
</bpmndi:BPMNPlane>
|
||||
</bpmndi:BPMNDiagram>
|
||||
</bpmn:definitions>
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
from tests.base_test import BaseTest
|
||||
|
||||
|
||||
class TestSpecFromWorkflowID(BaseTest):
|
||||
|
||||
def test_get_spec_from_workflow_id(self):
|
||||
workflow = self.create_workflow('spec_from_id')
|
||||
workflow_spec_id = workflow.workflow_spec_id
|
||||
workflow_api = self.get_workflow_api(workflow)
|
||||
task = workflow_api.next_task
|
||||
workflow_api = self.complete_form(workflow, task, {'spec_id': workflow_spec_id})
|
||||
task = workflow_api.next_task
|
||||
|
||||
self.assertEqual('spec_from_id', task.data['spec']['id'])
|
||||
self.assertEqual('spec_from_id', task.data['spec']['display_name'])
|
||||
|
|
@ -1,5 +1,6 @@
|
|||
from tests.base_test import BaseTest
|
||||
|
||||
from crc import app
|
||||
from crc.services.git_service import GitService
|
||||
|
||||
from unittest.mock import patch, Mock, call
|
||||
|
|
@ -57,6 +58,10 @@ class TestGitService(BaseTest):
|
|||
self.assertIn(call.index.commit('This is my comment'), method_calls)
|
||||
self.assertIn(call.remotes.origin.push(), method_calls)
|
||||
|
||||
# def test_pull_from_remote(self):
|
||||
# result = GitService.pull_from_remote()
|
||||
# print(result)
|
||||
def test_get_remote_url(self):
|
||||
app.config['GIT_REMOTE_SERVER'] = 'test_server.com'
|
||||
app.config['GIT_USER_NAME'] = 'test_username'
|
||||
app.config['GIT_USER_PASS'] = 'test_pass'
|
||||
|
||||
result = GitService.get_remote_url('my_test_path')
|
||||
self.assertEqual('https://test_username:test_pass@test_server.com/my_test_path.git', result)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,12 @@
|
|||
from tests.base_test import BaseTest
|
||||
|
||||
|
||||
class TestDateValidation(BaseTest):
|
||||
|
||||
def test_date_validation(self):
|
||||
|
||||
"""We were not instantiating date fields correctly during validation.
|
||||
This is a simple test to make sure we seed an actual date in date fields instead of a random string."""
|
||||
spec_model = self.load_test_spec('date_validation')
|
||||
rv = self.app.get('/v1.0/workflow-specification/%s/validate' % spec_model.id, headers=self.logged_in_headers())
|
||||
self.assertEqual([], rv.json)
|
||||
Loading…
Reference in New Issue