sartography
/
cr-connect-workflow
mirror of https://github.com/sartography/cr-connect-workflow.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add a robust way of adding an API key, update examples and documentation for swagger API and add the ability to completely sync the local system from the remote system.

This commit is contained in:
Kelly McDonald 2020-12-09 12:13:17 -05:00
parent 0e1aa59fa1
commit c57b17df1e
3 changed files with 204 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
config/default.py
View File

@ -6,6 +6,14 @@ basedir = os.path.abspath(os.path.dirname(__file__))
JSON_SORT_KEYS = False # CRITICAL. Do not sort the data when returning values to the front end.
# The API_TOKEN is used to ensure that the
# workflow synch can work without a lot of
# back and forth.
# you may want to change this to something simple for testing!!
# NB, if you change this in the local endpoint,
# it needs to be changed in the remote endpoint as well
API_TOKEN = 'af95596f327c9ecc007b60414fc84b61'
NAME = "CR Connect Workflow"
FLASK_PORT = environ.get('PORT0') or environ.get('FLASK_PORT', default="5000")
CORS_ALLOW_ORIGINS = re.split(r',\s*', environ.get('CORS_ALLOW_ORIGINS', default="localhost:4200, localhost:5002"))

177
crc/api.yml
View File

@ -100,11 +100,12 @@ paths:
type: array
items:
$ref: "#/components/schemas/Study"
/workflow_spec/diff:
/workflow_spec/pullall:
get:
operationId: crc.api.workflow.get_changed_workflows
summary: Provides a list of workflow specs and their signature
security: [] # Disable security for this endpoint only - we'll sanity check
operationId: crc.api.workflow.sync_all_changed_workflows
summary: Sync all workflows that have changed on the remote side and provide a list of the results
security:
- ApiKeyAuth : []
# in the endpoint
parameters:
- name: remote
@ -117,21 +118,51 @@ paths:
- Workflow Spec States
responses:
'200':
description: An array of workflow specs, with last touched date and file signature.
description: An array of workflow specs that were synced from remote.
content:
application/json:
schema:
type: array
items:
$ref: "#/components/schemas/Study"
type: string
example : ['top_level_workflow','3b495037-f7d4-4509-bf58-cee41c0c6b0e']
/workflow_spec/diff:
get:
operationId: crc.api.workflow.get_changed_workflows
summary: Provides a list of workflow that differ from remote and if it is new or not
security :
- ApiKeyAuth : []
# in the endpoint
parameters:
- name: remote
in: query
required: true
description: The remote endpoint
schema:
type: string
tags:
- Workflow Spec States
responses:
'200':
description: An array of workflow specs, with last touched date and which one is most recent.
content:
application/json:
schema:
type: array
items:
$ref: "#/components/schemas/WorkflowSpecDiffList"
/workflow_spec/{workflow_spec_id}/files:
get:
operationId: crc.api.workflow.get_workflow_spec_files
summary: Provides a list of workflow specs and their signature
security: [] # Disable security for this endpoint only - we'll sanity check
# in the endpoint
summary: Provides a list of files for a workflow spec on this machine.
security :
- ApiKeyAuth : []
parameters:
- name: workflow_spec_id
in: path
@ -144,20 +175,20 @@ paths:
- Workflow Spec States
responses:
'200':
description: An array of workflow specs, with last touched date and file signature.
description: An array of files for a workflow spec on the local system, with details.
content:
application/json:
schema:
type: array
items:
$ref: "#/components/schemas/Study"
$ref: "#/components/schemas/WorkflowSpecFilesList"
/workflow_spec/{workflow_spec_id}/files/sync:
get:
operationId: crc.api.workflow.sync_changed_files
summary: Provides a list of files that were updated
security: [] # Disable security for this endpoint only - we'll sanity check
# in the endpoint
summary: Syncs files from a workflow on a remote system and provides a list of files that were updated
security :
- ApiKeyAuth : []
parameters:
- name: workflow_spec_id
in: path
@ -176,20 +207,23 @@ paths:
- Workflow Spec States
responses:
'200':
description: An array of workflow specs, with last touched date and file signature.
description: A list of files that were synced for the workflow.
content:
application/json:
schema:
type: array
items:
$ref: "#/components/schemas/Study"
type : string
example : ["data_security_plan.dmn",'some_other_file.xml']
/workflow_spec/{workflow_spec_id}/files/diff:
get:
operationId: crc.api.workflow.get_changed_files
summary: Provides a list of workflow specs and their signature
security: [] # Disable security for this endpoint only - we'll sanity check
# in the endpoint
summary: Provides a list of files for a workflow specs that differ from remote and their signature
security :
- ApiKeyAuth : []
parameters:
- name: workflow_spec_id
in: path
@ -208,21 +242,22 @@ paths:
- Workflow Spec States
responses:
'200':
description: An array of workflow specs, with last touched date and file signature.
description: An array of files that are different from remote, with last touched date and file signature.
content:
application/json:
schema:
type: array
items:
$ref: "#/components/schemas/Study"
$ref: "#/components/schemas/WorkflowSpecFilesDiff"
/workflow_spec/all:
get:
operationId: crc.api.workflow.get_all_spec_state
summary: Provides a list of files for a workflow spec
security: [] # Disable security for this endpoint only - we'll sanity check
# in the endpoint
summary: Provides a list of workflow specs, last update date and thumbprint
security:
- ApiKeyAuth : []
tags:
- Workflow Spec States
responses:
@ -233,7 +268,7 @@ paths:
schema:
type: array
items:
$ref: "#/components/schemas/Study"
$ref: "#/components/schemas/WorkflowSpecAll"
/study/all:
@ -1314,6 +1349,12 @@ components:
scheme: bearer
bearerFormat: JWT
x-bearerInfoFunc: crc.api.user.verify_token_admin
ApiKeyAuth :
type : apiKey
in : header
name : X-CR-API-KEY
x-apikeyInfoFunc: crc.api.workflow.verify_token
schemas:
User:
properties:
@ -1337,6 +1378,92 @@ components:
properties:
id:
type: string
WorkflowSpecDiffList:
properties:
workflow_spec_id:
type: string
example : top_level_workflow
date_created :
type: string
example : 2020-12-09 16:55:12.951500+00:00
location :
type : string
example : remote
new :
type : boolean
example : false
WorkflowSpecFilesList:
properties:
file_model_id:
type : integer
example : 171
workflow_spec_id :
type: string
example : top_level_workflow
filename :
type: string
example : data_security_plan.dmn
date_created :
type: string
example : 2020-12-01 13:58:12.420333+00:00
type:
type : string
example : dmn
primary :
type : boolean
example : false
content_type:
type: string
example : text/xml
primary_process_id:
type : string
example : null
md5_hash:
type: string
example: f12e2bbd-a20c-673b-ccb8-a8a1ea9c5b7b
WorkflowSpecFilesDiff:
properties:
filename :
type: string
example : data_security_plan.dmn
date_created :
type: string
example : 2020-12-01 13:58:12.420333+00:00
type:
type : string
example : dmn
primary :
type : boolean
example : false
content_type:
type: string
example : text/xml
primary_process_id:
type : string
example : null
md5_hash:
type: string
example: f12e2bbd-a20c-673b-ccb8-a8a1ea9c5b7b
location:
type : string
example : remote
new:
type: boolean
example : false
WorkflowSpecAll:
properties:
workflow_spec_id :
type: string
example : acaf1258-43b4-437e-8846-f612afa66811
date_created :
type: string
example : 2020-12-01 13:58:12.420333+00:00
md5_hash:
type: string
example: c30fd597f21715018eab12f97f9d4956
Study:
properties:
id:

56
crc/api/workflow.py
View File

@ -7,7 +7,7 @@ from hashlib import md5
import pandas as pd
from SpiffWorkflow.util.deep_merge import DeepMerge
from flask import g
from crc import session, db
from crc import session, db, app
from crc.api.common import ApiError, ApiErrorSchema
from crc.models.api_models import WorkflowApi, WorkflowApiSchema, NavigationItem, NavigationItemSchema
from crc.models.file import FileModel, LookupDataSchema, FileDataModel
@ -269,12 +269,19 @@ def join_uuids(uuids):
# in the same order
return hashlib.md5(combined_uuids.encode('utf8')).hexdigest() # make a hash of the hashes
def get_changed_workflows(remote):
def verify_token(token, required_scopes):
if token == app.config['API_TOKEN']:
return {'scope':['any']}
else:
raise ApiError("permission_denied","API Token information is not correct")
def get_changed_workflows(remote,as_df=False):
"""
gets a remote endpoint - gets the workflows and then
determines what workflows are different from the remote endpoint
"""
response = requests.get('http://'+remote+'/v1.0/workflow_spec/all')
response = requests.get('http://'+remote+'/v1.0/workflow_spec/all',headers={'X-CR-API-KEY':app.config['API_TOKEN']})
# This is probably very and may allow cross site attacks - fix later
remote = pd.DataFrame(json.loads(response.text))
@ -318,14 +325,25 @@ def get_changed_workflows(remote):
output = changedfiles[~changedfiles.index.isin(right['workflow_spec_id'])]
# return the list as a dict, let swagger convert it to json
return output.reset_index().to_dict(orient='records')
if as_df:
return output
else:
return output.reset_index().to_dict(orient='records')
def sync_all_changed_workflows(remote):
workflowsdf = get_changed_workflows(remote,as_df=True)
workflows = workflowsdf.reset_index().to_dict(orient='records')
for workflow in workflows:
sync_changed_files(remote,workflow['workflow_spec_id'])
return [x['workflow_spec_id'] for x in workflows]
def sync_all_changed_files(remote):
pass
def sync_changed_files(remote,workflow_spec_id):
# make sure that spec is local before syncing files
remotespectext = requests.get('http://'+remote+'/v1.0/workflow-specification/'+workflow_spec_id)
remotespectext = requests.get('http://'+remote+'/v1.0/workflow-specification/'+workflow_spec_id,
headers={'X-CR-API-KEY': app.config['API_TOKEN']})
specdict = json.loads(remotespectext.text)
localspec = session.query(WorkflowSpecModel).filter(WorkflowSpecModel.id == workflow_spec_id).first()
if localspec is None:
@ -343,13 +361,20 @@ def sync_changed_files(remote,workflow_spec_id):
session.add(localspec)
changedfiles = get_changed_files(remote,workflow_spec_id,as_df=True)
if len(changedfiles)==0:
return []
updatefiles = changedfiles[~((changedfiles['new']==True) & (changedfiles['location']=='local'))]
updatefiles = updatefiles.reset_index().to_dict(orient='records')
deletefiles = changedfiles[((changedfiles['new']==True) & (changedfiles['location']=='local'))]
for delfile in deletefiles.reset_index().to_dict(orient='records'):
deletefiles = deletefiles.reset_index().to_dict(orient='records')
for delfile in deletefiles:
currentfile = session.query(FileModel).filter(FileModel.workflow_spec_id==workflow_spec_id,
FileModel.name == delfile['filename']).first()
FileService.delete_file(currentfile.id)
for updatefile in updatefiles.reset_index().to_dict(orient='records'):
for updatefile in updatefiles:
currentfile = session.query(FileModel).filter(FileModel.workflow_spec_id==workflow_spec_id,
FileModel.name == updatefile['filename']).first()
if not currentfile:
@ -364,9 +389,11 @@ def sync_changed_files(remote,workflow_spec_id):
currentfile.primary_process_id = updatefile['primary_process_id']
session.add(currentfile)
response = requests.get('http://'+remote+'/v1.0/file/'+updatefile['md5_hash']+'/hash_data')
response = requests.get('http://'+remote+'/v1.0/file/'+updatefile['md5_hash']+'/hash_data',
headers={'X-CR-API-KEY': app.config['API_TOKEN']})
FileService.update_file(currentfile,response.content,updatefile['type'])
session.commit()
return [x['filename'] for x in updatefiles]
def get_changed_files(remote,workflow_spec_id,as_df=False):
@ -375,7 +402,8 @@ def get_changed_files(remote,workflow_spec_id,as_df=False):
local and remote and determines what files have been change and returns a list of those
files
"""
response = requests.get('http://'+remote+'/v1.0/workflow_spec/'+workflow_spec_id+'/files')
response = requests.get('http://'+remote+'/v1.0/workflow_spec/'+workflow_spec_id+'/files',
headers={'X-CR-API-KEY':app.config['API_TOKEN']})
# This is probably very and may allow cross site attacks - fix later
remote = pd.DataFrame(json.loads(response.text))
# get the local thumbprints & make sure that 'workflow_spec_id' is a column, not an index
@ -386,7 +414,11 @@ def get_changed_files(remote,workflow_spec_id,as_df=False):
left_on=['filename','md5_hash'],
how = 'outer' ,
indicator=True).loc[lambda x : x['_merge']!='both']
if len(different) == 0:
if as_df:
return different
else:
return []
# each line has a tag on it - if was in the left or the right,
# label it so we know if that was on the remote or local machine
different.loc[different['_merge']=='left_only','location'] = 'remote'