sartography
/
cr-connect-workflow
mirror of https://github.com/sartography/cr-connect-workflow.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Move data store base into its own file.

This commit is contained in:
Dan 2020-11-18 15:34:50 -05:00
parent 66693d9f3a
commit b83ab1bcd1
8 changed files with 136 additions and 155 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

79
crc/api/data_store.py
View File

@ -1,15 +1,13 @@
import json
from datetime import datetime from datetime import datetime
from flask import g
from sqlalchemy.exc import IntegrityError
import json
from crc import session from crc import session
from crc.scripts.script import DataStoreBase from crc.api.common import ApiError
from crc.api.common import ApiError, ApiErrorSchema from crc.models.data_store import DataStoreModel, DataStoreSchema
from crc.models.data_store import DataStoreModel,DataStoreSchema from crc.scripts.data_store_base import DataStoreBase
def study_data_set(study_id,key,value): def study_data_set(study_id, key, value):
"""Set a study data value in the data_store, mimic the script endpoint""" """Set a study data value in the data_store, mimic the script endpoint"""
if study_id is None: if study_id is None:
raise ApiError('unknown_study', 'Please provide a valid Study ID.') raise ApiError('unknown_study', 'Please provide a valid Study ID.')
@ -17,11 +15,12 @@ def study_data_set(study_id,key,value):
if key is None: if key is None:
raise ApiError('invalid_key', 'Please provide a valid key') raise ApiError('invalid_key', 'Please provide a valid key')
dsb = DataStoreBase() dsb = DataStoreBase()
retval=dsb.set_data_common('api',study_id,None,None,None,'api_study_data_set',key,value) retval = dsb.set_data_common('api', study_id, None, None, None, 'api_study_data_set', key, value)
json_value = json.dumps(retval, ensure_ascii=False, indent=2) json_value = json.dumps(retval, ensure_ascii=False, indent=2)
return json_value return json_value
def study_data_get(study_id,key,default=None):
def study_data_get(study_id, key, default=None):
"""Get a study data value in the data_store, mimic the script endpoint""" """Get a study data value in the data_store, mimic the script endpoint"""
if study_id is None: if study_id is None:
raise ApiError('unknown_study', 'Please provide a valid Study ID.') raise ApiError('unknown_study', 'Please provide a valid Study ID.')
@ -29,24 +28,23 @@ def study_data_get(study_id,key,default=None):
if key is None: if key is None:
raise ApiError('invalid_key', 'Please provide a valid key') raise ApiError('invalid_key', 'Please provide a valid key')
dsb = DataStoreBase() dsb = DataStoreBase()
retval = dsb.get_data_common(study_id,None,'api_study_data_get',key,default) retval = dsb.get_data_common(study_id, None, 'api_study_data_get', key, default)
#json_value = json.dumps(retval, ensure_ascii=False, indent=2) # just return raw text # json_value = json.dumps(retval, ensure_ascii=False, indent=2) # just return raw text
return retval return retval
def study_multi_get(study_id): def study_multi_get(study_id):
"""Get all data_store values for a given study_id study""" """Get all data_store values for a given study_id study"""
if study_id is None: if study_id is None:
raise ApiError('unknown_study', 'Please provide a valid Study ID.') raise ApiError('unknown_study', 'Please provide a valid Study ID.')
dsb = DataStoreBase() dsb = DataStoreBase()
retval=dsb.get_multi_common(study_id,None) retval = dsb.get_multi_common(study_id, None)
results = DataStoreSchema(many=True).dump(retval) results = DataStoreSchema(many=True).dump(retval)
return results return results
def study_data_del(study_id, key):
def study_data_del(study_id,key):
"""Delete a study data value in the data store""" """Delete a study data value in the data store"""
if study_id is None: if study_id is None:
raise ApiError('unknown_study', 'Please provide a valid Study ID.') raise ApiError('unknown_study', 'Please provide a valid Study ID.')
@ -54,12 +52,12 @@ def study_data_del(study_id,key):
if key is None: if key is None:
raise ApiError('invalid_key', 'Please provide a valid key') raise ApiError('invalid_key', 'Please provide a valid key')
dsb = DataStoreBase() dsb = DataStoreBase()
dsb.del_data_common(study_id,None,'api_study_data_get',key) dsb.del_data_common(study_id, None, 'api_study_data_get', key)
json_value = json.dumps('deleted', ensure_ascii=False, indent=2) json_value = json.dumps('deleted', ensure_ascii=False, indent=2)
return json_value return json_value
def user_data_set(user_id,key,value): def user_data_set(user_id, key, value):
"""Set a user data value in the data_store, mimic the script endpoint""" """Set a user data value in the data_store, mimic the script endpoint"""
if user_id is None: if user_id is None:
raise ApiError('unknown_study', 'Please provide a valid UserID.') raise ApiError('unknown_study', 'Please provide a valid UserID.')
@ -68,19 +66,19 @@ def user_data_set(user_id,key,value):
raise ApiError('invalid_key', 'Please provide a valid key') raise ApiError('invalid_key', 'Please provide a valid key')
dsb = DataStoreBase() dsb = DataStoreBase()
retval=dsb.set_data_common('api', retval = dsb.set_data_common('api',
None, None,
user_id, user_id,
None, None,
None, None,
'api_user_data_set', 'api_user_data_set',
key,value) key, value)
json_value = json.dumps(retval, ensure_ascii=False, indent=2) json_value = json.dumps(retval, ensure_ascii=False, indent=2)
return json_value return json_value
def user_data_get(user_id,key,default=None): def user_data_get(user_id, key, default=None):
"""Get a user data value from the data_store, mimic the script endpoint""" """Get a user data value from the data_store, mimic the script endpoint"""
if user_id is None: if user_id is None:
raise ApiError('unknown_study', 'Please provide a valid UserID.') raise ApiError('unknown_study', 'Please provide a valid UserID.')
@ -88,27 +86,27 @@ def user_data_get(user_id,key,default=None):
if key is None: if key is None:
raise ApiError('invalid_key', 'Please provide a valid key') raise ApiError('invalid_key', 'Please provide a valid key')
dsb = DataStoreBase() dsb = DataStoreBase()
retval=dsb.get_data_common(None, retval = dsb.get_data_common(None,
user_id, user_id,
'api_user_data_get', 'api_user_data_get',
key,default) key, default)
#json_value = json.dumps(retval, ensure_ascii=False, indent=2) # just return raw text # json_value = json.dumps(retval, ensure_ascii=False, indent=2) # just return raw text
return retval return retval
def user_multi_get(user_id): def user_multi_get(user_id):
"""Get all data values in the data_store for a userid""" """Get all data values in the data_store for a userid"""
if user_id is None: if user_id is None:
raise ApiError('unknown_study', 'Please provide a valid UserID.') raise ApiError('unknown_study', 'Please provide a valid UserID.')
dsb = DataStoreBase() dsb = DataStoreBase()
retval=dsb.get_multi_common(None, retval = dsb.get_multi_common(None,
user_id) user_id)
results = DataStoreSchema(many=True).dump(retval) results = DataStoreSchema(many=True).dump(retval)
return results return results
def datastore_del(id): def datastore_del(id):
"""Delete a data store item for a user_id and a key""" """Delete a data store item for a user_id and a key"""
session.query(DataStoreModel).filter_by(id=id).delete() session.query(DataStoreModel).filter_by(id=id).delete()
@ -116,12 +114,14 @@ def datastore_del(id):
json_value = json.dumps('deleted', ensure_ascii=False, indent=2) json_value = json.dumps('deleted', ensure_ascii=False, indent=2)
return json_value return json_value
def datastore_get(id): def datastore_get(id):
"""Delete a data store item for a user_id and a key""" """Delete a data store item for a user_id and a key"""
item = session.query(DataStoreModel).filter_by(id=id).first() item = session.query(DataStoreModel).filter_by(id=id).first()
results = DataStoreSchema(many=False).dump(item) results = DataStoreSchema(many=False).dump(item)
return results return results
def update_datastore(id, body): def update_datastore(id, body):
"""allow a modification to a datastore item """ """allow a modification to a datastore item """
if id is None: if id is None:
@ -134,8 +134,8 @@ def update_datastore(id, body):
# I'm not sure if there is a generic way to use the # I'm not sure if there is a generic way to use the
# schema to both parse the body and update the SQLAlchemy record # schema to both parse the body and update the SQLAlchemy record
for key in body: for key in body:
if hasattr(item,key): if hasattr(item, key):
setattr(item,key,body[key]) setattr(item, key, body[key])
item.last_updated = datetime.now() item.last_updated = datetime.now()
session.add(item) session.add(item)
session.commit() session.commit()
@ -146,7 +146,7 @@ def add_datastore(body):
""" add a new datastore item """ """ add a new datastore item """
print(body) print(body)
if body.get(id,None): if body.get(id, None):
raise ApiError('id_specified', 'You may not specify an id for a new datastore item') raise ApiError('id_specified', 'You may not specify an id for a new datastore item')
if 'key' not in body: if 'key' not in body:
@ -158,17 +158,16 @@ def add_datastore(body):
if (not 'user_id' in body) and (not 'study_id' in body): if (not 'user_id' in body) and (not 'study_id' in body):
raise ApiError('conflicting_values', 'A datastore item should have either a study_id or a user_id') raise ApiError('conflicting_values', 'A datastore item should have either a study_id or a user_id')
if 'user_id' in body and 'study_id' in body: if 'user_id' in body and 'study_id' in body:
raise ApiError('conflicting_values', 'A datastore item should have either a study_id or a user_id, ' raise ApiError('conflicting_values', 'A datastore item should have either a study_id or a user_id, '
'but not both') 'but not both')
item = DataStoreModel(key=body['key'],value=body['value']) item = DataStoreModel(key=body['key'], value=body['value'])
# I'm not sure if there is a generic way to use the # I'm not sure if there is a generic way to use the
# schema to both parse the body and update the SQLAlchemy record # schema to both parse the body and update the SQLAlchemy record
for key in body: for key in body:
if hasattr(item,key): if hasattr(item, key):
setattr(item,key,body[key]) setattr(item, key, body[key])
item.last_updated = datetime.now() item.last_updated = datetime.now()
session.add(item) session.add(item)
session.commit() session.commit()

80
crc/scripts/data_store_base.py Normal file
View File

@ -0,0 +1,80 @@
import importlib
import os
import pkgutil
from crc import session
from crc.api.common import ApiError
from crc.models.data_store import DataStoreModel
from crc.models.workflow import WorkflowModel
from datetime import datetime
class DataStoreBase(object):
def overwritten(self, value, prev_value):
if prev_value is None:
overwritten = False
else:
if prev_value == value:
overwritten = False
else:
overwritten = True
return overwritten
def set_validate_common(self, study_id, workflow_id, user_id, script_name, *args):
self.check_args_2(args, script_name)
workflow = session.query(WorkflowModel).filter(WorkflowModel.id == workflow_id).first()
self.get_prev_value(study_id=study_id, user_id=user_id, key=args[0])
def check_args(self, args, maxlen=1, script_name='study_data_get'):
if len(args) < 1 or len(args) > maxlen:
raise ApiError(code="missing_argument",
message=f"The {script_name} script takes either one or two arguments, "
f"starting with the key and an optional default")
def check_args_2(self, args, script_name='study_data_set'):
if len(args) != 2:
raise ApiError(code="missing_argument",
message=f"The {script_name} script takes two arguments, starting with the key and a "
"value for the key")
def get_prev_value(self, study_id, user_id, key):
study = session.query(DataStoreModel).filter_by(study_id=study_id, user_id=user_id, key=key).first()
return study
def set_data_common(self, task_id, study_id, user_id, workflow_id, workflow_spec_id, script_name, *args, **kwargs):
self.check_args_2(args, script_name=script_name)
study = self.get_prev_value(study_id=study_id, user_id=user_id, key=args[0])
if workflow_spec_id is None and workflow_id is not None:
workflow = session.query(WorkflowModel).filter(WorkflowModel.id == workflow_id).first()
workflow_spec_id = workflow.workflow_spec_id
if study is not None:
prev_value = study.value
else:
prev_value = None
study = DataStoreModel(key=args[0], value=args[1],
study_id=study_id,
task_id=task_id,
user_id=user_id, # Make this available to any User
workflow_id=workflow_id,
spec_id=workflow_spec_id)
study.value = args[1]
study.last_updated = datetime.now()
overwritten = self.overwritten(study.value, prev_value)
session.add(study)
session.commit()
return {'new_value': study.value,
'old_value': prev_value,
'overwritten': overwritten}
def get_data_common(self, study_id, user_id, script_name, *args):
self.check_args(args, 2, script_name)
study = session.query(DataStoreModel).filter_by(study_id=study_id, user_id=user_id, key=args[0]).first()
if study:
return study.value
else:
return args[1]
def get_multi_common(self, study_id, user_id):
study = session.query(DataStoreModel).filter_by(study_id=study_id, user_id=user_id)
return study

81
crc/scripts/script.py
View File

@ -1,11 +1,9 @@
import importlib import importlib
import os import os
import pkgutil import pkgutil
from crc import session
from crc.api.common import ApiError from crc.api.common import ApiError
from crc.models.data_store import DataStoreModel
from crc.models.workflow import WorkflowModel
from datetime import datetime
class Script(object): class Script(object):
""" Provides an abstract class that defines how scripts should work, this """ Provides an abstract class that defines how scripts should work, this
@ -70,9 +68,6 @@ class Script(object):
workflow_id) workflow_id)
return execlist return execlist
@staticmethod @staticmethod
def get_all_subclasses(): def get_all_subclasses():
return Script._get_all_subclasses(Script) return Script._get_all_subclasses(Script)
@ -112,77 +107,5 @@ class ScriptValidationError:
def from_api_error(cls, api_error: ApiError): def from_api_error(cls, api_error: ApiError):
return cls(api_error.code, api_error.message) return cls(api_error.code, api_error.message)
class DataStoreBase():
def overwritten(self,value,prev_value):
if prev_value == None:
overwritten = False
else:
if prev_value == value:
overwritten = False
else:
overwritten = True
return overwritten
def set_validate_common(self, study_id, workflow_id, user_id, script_name, *args):
self.check_args_2(args,script_name)
workflow = session.query(WorkflowModel).filter(WorkflowModel.id == workflow_id).first()
self.get_prev_value(study_id=study_id,user_id=user_id, key=args[0])
def check_args(self, args, maxlen=1, script_name='study_data_get'):
if len(args) < 1 or len(args) > maxlen :
raise ApiError(code="missing_argument",
message="The %s script takes either one or two arguments, starting with the key and an "%script_name + \
"optional default")
def check_args_2(self, args, script_name='study_data_set'):
if len(args) != 2:
raise ApiError(code="missing_argument",
message="The %s script takes two arguments, starting with the key and a "%script_name +\
"value for the key")
def get_prev_value(self,study_id,user_id,key):
study = session.query(DataStoreModel).filter_by(study_id=study_id,user_id=user_id,key=key).first()
return study
def set_data_common(self, task_id, study_id, user_id, workflow_id, workflow_spec_id, script_name, *args, **kwargs):
self.check_args_2(args,script_name=script_name)
study = self.get_prev_value(study_id=study_id,user_id=user_id, key=args[0])
if workflow_spec_id is None and workflow_id is not None:
workflow = session.query(WorkflowModel).filter(WorkflowModel.id == workflow_id).first()
workflow_spec_id = workflow.workflow_spec_id
if study is not None:
prev_value = study.value
else:
prev_value = None
study = DataStoreModel(key=args[0],value=args[1],
study_id=study_id,
task_id=task_id,
user_id=user_id, # Make this available to any User
workflow_id= workflow_id,
spec_id=workflow_spec_id)
study.value = args[1]
study.last_updated = datetime.now()
overwritten = self.overwritten(study.value,prev_value)
session.add(study)
session.commit()
return {'new_value':study.value,
'old_value':prev_value,
'overwritten':overwritten}
def get_data_common(self, study_id, user_id, script_name, *args):
self.check_args(args,2,script_name)
study = session.query(DataStoreModel).filter_by(study_id=study_id,user_id=user_id,key=args[0]).first()
if study:
return study.value
else:
return args[1]
def get_multi_common(self, study_id, user_id):
study = session.query(DataStoreModel).filter_by(study_id=study_id,user_id=user_id)
return (study)

7
crc/scripts/study_data_get.py
View File

@ -1,14 +1,11 @@
import requests from crc.scripts.data_store_base import DataStoreBase
from crc.scripts.script import Script
from crc.scripts.script import Script, DataStoreBase
from crc import session
from crc.models.data_store import DataStoreModel
class StudyDataGet(Script,DataStoreBase): class StudyDataGet(Script,DataStoreBase):
def get_description(self): def get_description(self):
return """Gets study data from the data store.""" return """Gets study data from the data store."""
def do_task_validate_only(self, task, study_id, workflow_id, *args, **kwargs): def do_task_validate_only(self, task, study_id, workflow_id, *args, **kwargs):
self.do_task(task, study_id, workflow_id, *args, **kwargs) self.do_task(task, study_id, workflow_id, *args, **kwargs)

9
crc/scripts/study_data_set.py
View File

@ -1,15 +1,11 @@
import requests from crc.scripts.data_store_base import DataStoreBase
from crc.scripts.script import Script
from crc.scripts.script import Script, DataStoreBase
from crc import session
from crc.models.workflow import WorkflowModel
from crc.models.data_store import DataStoreModel
class StudyDataSet(Script,DataStoreBase): class StudyDataSet(Script,DataStoreBase):
def get_description(self): def get_description(self):
return """Sets study data from the data store.""" return """Sets study data from the data store."""
def do_task_validate_only(self, task, study_id, workflow_id, *args, **kwargs): def do_task_validate_only(self, task, study_id, workflow_id, *args, **kwargs):
self.set_validate_common(study_id, self.set_validate_common(study_id,
workflow_id, workflow_id,
@ -17,7 +13,6 @@ class StudyDataSet(Script,DataStoreBase):
'study_data_set', 'study_data_set',
*args) *args)
def do_task(self, task, study_id, workflow_id, *args, **kwargs): def do_task(self, task, study_id, workflow_id, *args, **kwargs):
return self.set_data_common(task.id, return self.set_data_common(task.id,
study_id, study_id,

12
crc/scripts/user_data_get.py
View File

@ -1,15 +1,13 @@
import requests
from crc.scripts.script import Script, DataStoreBase
from crc import session
from crc.models.data_store import DataStoreModel
from flask import g from flask import g
class UserDataGet(Script,DataStoreBase): from crc.scripts.data_store_base import DataStoreBase
from crc.scripts.script import Script
class UserDataGet(Script, DataStoreBase):
def get_description(self): def get_description(self):
return """Gets user data from the data store.""" return """Gets user data from the data store."""
def do_task_validate_only(self, task, study_id, workflow_id, *args, **kwargs): def do_task_validate_only(self, task, study_id, workflow_id, *args, **kwargs):
self.do_task(task, study_id, workflow_id, *args, **kwargs) self.do_task(task, study_id, workflow_id, *args, **kwargs)

13
crc/scripts/user_data_set.py
View File

@ -1,17 +1,13 @@
import requests
from crc.scripts.script import Script, DataStoreBase
from crc import session
from crc.models.workflow import WorkflowModel
from crc.models.data_store import DataStoreModel
from flask import g from flask import g
from crc.scripts.data_store_base import DataStoreBase
from crc.scripts.script import Script
class UserDataSet(Script,DataStoreBase): class UserDataSet(Script,DataStoreBase):
def get_description(self): def get_description(self):
return """Sets user data to the data store.""" return """Sets user data to the data store."""
def do_task_validate_only(self, task, study_id, workflow_id, *args, **kwargs): def do_task_validate_only(self, task, study_id, workflow_id, *args, **kwargs):
self.set_validate_common(None, self.set_validate_common(None,
workflow_id, workflow_id,
@ -19,7 +15,6 @@ class UserDataSet(Script,DataStoreBase):
'user_data_set', 'user_data_set',
*args) *args)
def do_task(self, task, study_id, workflow_id, *args, **kwargs): def do_task(self, task, study_id, workflow_id, *args, **kwargs):
return self.set_data_common(task.id, return self.set_data_common(task.id,
None, None,

10
tests/emails/test_email_script.py
View File

@ -1,12 +1,6 @@
from tests.base_test import BaseTest from crc import mail
from crc.models.email import EmailModel from crc.models.email import EmailModel
from crc.services.file_service import FileService from tests.base_test import BaseTest
from crc.scripts.email import Email
from crc.services.workflow_processor import WorkflowProcessor
from crc.api.common import ApiError
from crc import db, mail
class TestEmailScript(BaseTest): class TestEmailScript(BaseTest):