sartography
/
cr-connect-workflow
mirror of https://github.com/sartography/cr-connect-workflow.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

refactored calls into a service

This commit is contained in:
Kelly McDonald 2020-12-11 11:41:32 -05:00
parent 9eea26e019
commit 3a1160efac
4 changed files with 76 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
crc/api.yml
View File

@ -657,7 +657,8 @@ paths:
get:
operationId: crc.api.file.get_file_data_by_hash
summary: Returns only the file contents
security: [] # Disable security for this endpoint only.
security:
- ApiKeyAuth: []
tags:
- Files
responses:

51
crc/api/workflow_sync.py
View File

@ -7,6 +7,7 @@ from crc.api.common import ApiError
from crc.models.file import FileModel, FileDataModel
from crc.models.workflow import WorkflowSpecModel, WorkflowSpecCategoryModel
from crc.services.file_service import FileService
from crc.services.workflow_sync import WorkflowSyncService
def join_uuids(uuids):
@ -27,21 +28,16 @@ def get_changed_workflows(remote,as_df=False):
gets a remote endpoint - gets the workflows and then
determines what workflows are different from the remote endpoint
"""
try:
response = requests.get('http://'+remote+'/v1.0/workflow_sync/all',headers={'X-CR-API-KEY':app.config['API_TOKEN']})
except:
raise ApiError("endpoint error", 'had a problem connecting to '+remote)
if not response.ok:
raise ApiError("endpoint error", response.text)
remote = pd.DataFrame(json.loads(response.text))
remote_workflows_list = WorkflowSyncService.get_all_remote_workflows(remote)
remote_workflows = pd.DataFrame(remote_workflows_list)
# get the local thumbprints & make sure that 'workflow_spec_id' is a column, not an index
local = get_all_spec_state_dataframe().reset_index()
# merge these on workflow spec id and hash - this will
# make two different date columns date_x and date_y
different = remote.merge(local,
different = remote_workflows.merge(local,
right_on=['workflow_spec_id','md5_hash'],
left_on=['workflow_spec_id','md5_hash'],
how = 'outer' ,
@ -66,7 +62,7 @@ def get_changed_workflows(remote,as_df=False):
# get an exclusive or list of workflow ids - that is we want lists of files that are
# on one machine or the other, but not both
remote_spec_ids = remote[['workflow_spec_id']]
remote_spec_ids = remote_workflows[['workflow_spec_id']]
local_spec_ids = local[['workflow_spec_id']]
left = remote_spec_ids[~remote_spec_ids['workflow_spec_id'].isin(local_spec_ids['workflow_spec_id'])]
right = local_spec_ids[~local_spec_ids['workflow_spec_id'].isin(remote_spec_ids['workflow_spec_id'])]
@ -96,16 +92,8 @@ def sync_all_changed_workflows(remote):
def sync_changed_files(remote,workflow_spec_id):
# make sure that spec is local before syncing files
try:
remotespectext = requests.get('http://'+remote+'/v1.0/workflow-specification/'+workflow_spec_id,
headers={'X-CR-API-KEY': app.config['API_TOKEN']})
except:
raise ApiError("endpoint error", 'had a problem connecting to '+remote)
if not remotespectext.ok:
raise ApiError("endpoint error", response.text)
specdict = json.loads(remotespectext.text)
specdict = WorkflowSyncService.get_remote_workfow_spec(remote,workflow_spec_id)
localspec = session.query(WorkflowSpecModel).filter(WorkflowSpecModel.id == workflow_spec_id).first()
if localspec is None:
@ -164,16 +152,8 @@ def sync_changed_files(remote,workflow_spec_id):
currentfile.content_type = updatefile['content_type']
currentfile.primary_process_id = updatefile['primary_process_id']
session.add(currentfile)
try:
response = requests.get('http://'+remote+'/v1.0/file/'+updatefile['md5_hash']+'/hash_data',
headers={'X-CR-API-KEY': app.config['API_TOKEN']})
except:
raise ApiError("endpoint error", 'had a problem connecting to ' + remote)
if not response.ok:
raise ApiError("endpoint error", response.text)
FileService.update_file(currentfile,response.content,updatefile['type'])
content = WorkflowSyncService.get_remote_file_by_hash(remote,updatefile['md5_hash'])
FileService.update_file(currentfile,content,updatefile['type'])
session.commit()
return [x['filename'] for x in updatefiles]
@ -184,21 +164,12 @@ def get_changed_files(remote,workflow_spec_id,as_df=False):
local and remote and determines what files have been change and returns a list of those
files
"""
try:
response = requests.get('http://'+remote+'/v1.0/workflow_sync/'+workflow_spec_id+'/files',
headers={'X-CR-API-KEY':app.config['API_TOKEN']})
except:
raise ApiError("endpoint error", 'had a problem connecting to '+remote)
if not response.ok:
raise ApiError("endpoint error", response.text)
# This is probably very and may allow cross site attacks - fix later
remote = pd.DataFrame(json.loads(response.text))
remote_file_list = WorkflowSyncService.get_remote_workflow_spec_files(remote,workflow_spec_id)
remote_files = pd.DataFrame(remote_file_list)
# get the local thumbprints & make sure that 'workflow_spec_id' is a column, not an index
local = get_workflow_spec_files_dataframe(workflow_spec_id).reset_index()
local['md5_hash'] = local['md5_hash'].astype('str')
different = remote.merge(local,
different = remote_files.merge(local,
right_on=['filename','md5_hash'],
left_on=['filename','md5_hash'],
how = 'outer' ,

2
tests/study/test_study_api.py
View File

@ -165,7 +165,7 @@ class TestStudyApi(BaseTest):
self.assertEqual(study_event.comment, update_comment)
self.assertEqual(study_event.user_uid, self.test_uid)
@patch('crc.services.protocol_builder.ProtocolBuilderService.get_investigators') # mock_studies
@patch('crc.services.protocol_builder.ProtocolBuilderService.get_investigators') # mock_investigators
@patch('crc.services.protocol_builder.ProtocolBuilderService.get_required_docs') # mock_docs
@patch('crc.services.protocol_builder.ProtocolBuilderService.get_study_details') # mock_details
@patch('crc.services.protocol_builder.ProtocolBuilderService.get_studies') # mock_studies

68
tests/test_workflow_sync.py
View File

@ -1,19 +1,75 @@
from unittest.mock import patch
from crc import app
from crc import db
from tests.base_test import BaseTest
from crc.api.workflow_sync import get_all_spec_state, get_changed_workflows
from crc.models.workflow import WorkflowSpecModel
import json
pip
from datetime import datetime
from crc.services.file_service import FileService
class TestWorkflowSync(BaseTest):
@patch('crc.api.workflow_sync.requests.get')
@patch('crc.services.workflow_sync.WorkflowSyncService.get_all_remote_workflows')
def test_get_no_changes(self, mock_get):
self.load_example_data()
othersys = get_all_spec_state()
mock_get.return_value.ok = True
mock_get.return_value.text = json.dumps(othersys)
response = get_changed_workflows('somewhere over the rainbow')
mock_get.return_value = othersys
response = get_changed_workflows('localhost:0000') # not actually used due to mock
self.assertIsNotNone(response)
self.assertEqual(response,[])
@patch('crc.services.workflow_sync.WorkflowSyncService.get_all_remote_workflows')
def test_remote_workflow_change(self, mock_get):
self.load_example_data()
othersys = get_all_spec_state()
othersys[1]['date_created'] = str(datetime.now())
othersys[1]['md5_hash'] = '12345'
mock_get.return_value = othersys
response = get_changed_workflows('localhost:0000') #endpoint is not used due to mock
self.assertIsNotNone(response)
self.assertEqual(len(response),1)
self.assertEqual(response[0]['workflow_spec_id'], 'random_fact')
self.assertEqual(response[0]['location'], 'remote')
self.assertEqual(response[0]['new'], False)
@patch('crc.services.workflow_sync.WorkflowSyncService.get_all_remote_workflows')
def test_remote_workflow_has_new(self, mock_get):
self.load_example_data()
othersys = get_all_spec_state()
othersys.append({'workflow_spec_id':'my_new_workflow',
'date_created':str(datetime.now()),
'md5_hash': '12345'})
mock_get.return_value = othersys
response = get_changed_workflows('localhost:0000') #endpoint is not used due to mock
self.assertIsNotNone(response)
self.assertEqual(len(response),1)
self.assertEqual(response[0]['workflow_spec_id'],'my_new_workflow')
self.assertEqual(response[0]['location'], 'remote')
self.assertEqual(response[0]['new'], True)
@patch('crc.services.workflow_sync.WorkflowSyncService.get_all_remote_workflows')
def test_local_workflow_has_new(self, mock_get):
self.load_example_data()
othersys = get_all_spec_state()
mock_get.return_value = othersys
wf_spec = WorkflowSpecModel()
wf_spec.id = 'abcdefg'
wf_spec.display_name = 'New Workflow - Yum!!'
wf_spec.name = 'my_new_workflow'
wf_spec.description = 'yep - its a new workflow'
wf_spec.category_id = 0
wf_spec.display_order = 0
db.session.add(wf_spec)
db.session.commit()
FileService.add_workflow_spec_file(wf_spec,'dummyfile.txt','text',b'this is a test')
# after setting up the test - I realized that this doesn't return anything for
# a workflow that is new locally - it just returns nothing
response = get_changed_workflows('localhost:0000') #endpoint is not used due to mock
self.assertIsNotNone(response)
self.assertEqual(response,[])