From b83ab1bcd12ec580d7ec9e4bb265591780cfa475 Mon Sep 17 00:00:00 2001 From: Dan Date: Wed, 18 Nov 2020 15:34:50 -0500 Subject: [PATCH] Move data store base into its own file. --- crc/api/data_store.py | 79 +++++++++++++++--------------- crc/scripts/data_store_base.py | 80 ++++++++++++++++++++++++++++++ crc/scripts/script.py | 81 +------------------------------ crc/scripts/study_data_get.py | 7 +-- crc/scripts/study_data_set.py | 9 +--- crc/scripts/user_data_get.py | 12 ++--- crc/scripts/user_data_set.py | 13 ++--- tests/emails/test_email_script.py | 10 +--- 8 files changed, 136 insertions(+), 155 deletions(-) create mode 100644 crc/scripts/data_store_base.py diff --git a/crc/api/data_store.py b/crc/api/data_store.py index 4c094a3f..03c088e9 100644 --- a/crc/api/data_store.py +++ b/crc/api/data_store.py @@ -1,15 +1,13 @@ +import json from datetime import datetime -from flask import g -from sqlalchemy.exc import IntegrityError -import json from crc import session -from crc.scripts.script import DataStoreBase -from crc.api.common import ApiError, ApiErrorSchema -from crc.models.data_store import DataStoreModel,DataStoreSchema +from crc.api.common import ApiError +from crc.models.data_store import DataStoreModel, DataStoreSchema +from crc.scripts.data_store_base import DataStoreBase -def study_data_set(study_id,key,value): +def study_data_set(study_id, key, value): """Set a study data value in the data_store, mimic the script endpoint""" if study_id is None: raise ApiError('unknown_study', 'Please provide a valid Study ID.') @@ -17,11 +15,12 @@ def study_data_set(study_id,key,value): if key is None: raise ApiError('invalid_key', 'Please provide a valid key') dsb = DataStoreBase() - retval=dsb.set_data_common('api',study_id,None,None,None,'api_study_data_set',key,value) + retval = dsb.set_data_common('api', study_id, None, None, None, 'api_study_data_set', key, value) json_value = json.dumps(retval, ensure_ascii=False, indent=2) return json_value -def study_data_get(study_id,key,default=None): + +def study_data_get(study_id, key, default=None): """Get a study data value in the data_store, mimic the script endpoint""" if study_id is None: raise ApiError('unknown_study', 'Please provide a valid Study ID.') @@ -29,24 +28,23 @@ def study_data_get(study_id,key,default=None): if key is None: raise ApiError('invalid_key', 'Please provide a valid key') dsb = DataStoreBase() - retval = dsb.get_data_common(study_id,None,'api_study_data_get',key,default) - #json_value = json.dumps(retval, ensure_ascii=False, indent=2) # just return raw text + retval = dsb.get_data_common(study_id, None, 'api_study_data_get', key, default) + # json_value = json.dumps(retval, ensure_ascii=False, indent=2) # just return raw text return retval + def study_multi_get(study_id): """Get all data_store values for a given study_id study""" if study_id is None: raise ApiError('unknown_study', 'Please provide a valid Study ID.') dsb = DataStoreBase() - retval=dsb.get_multi_common(study_id,None) + retval = dsb.get_multi_common(study_id, None) results = DataStoreSchema(many=True).dump(retval) return results - - -def study_data_del(study_id,key): +def study_data_del(study_id, key): """Delete a study data value in the data store""" if study_id is None: raise ApiError('unknown_study', 'Please provide a valid Study ID.') @@ -54,12 +52,12 @@ def study_data_del(study_id,key): if key is None: raise ApiError('invalid_key', 'Please provide a valid key') dsb = DataStoreBase() - dsb.del_data_common(study_id,None,'api_study_data_get',key) + dsb.del_data_common(study_id, None, 'api_study_data_get', key) json_value = json.dumps('deleted', ensure_ascii=False, indent=2) return json_value -def user_data_set(user_id,key,value): +def user_data_set(user_id, key, value): """Set a user data value in the data_store, mimic the script endpoint""" if user_id is None: raise ApiError('unknown_study', 'Please provide a valid UserID.') @@ -68,19 +66,19 @@ def user_data_set(user_id,key,value): raise ApiError('invalid_key', 'Please provide a valid key') dsb = DataStoreBase() - retval=dsb.set_data_common('api', - None, - user_id, - None, - None, - 'api_user_data_set', - key,value) + retval = dsb.set_data_common('api', + None, + user_id, + None, + None, + 'api_user_data_set', + key, value) json_value = json.dumps(retval, ensure_ascii=False, indent=2) return json_value -def user_data_get(user_id,key,default=None): +def user_data_get(user_id, key, default=None): """Get a user data value from the data_store, mimic the script endpoint""" if user_id is None: raise ApiError('unknown_study', 'Please provide a valid UserID.') @@ -88,27 +86,27 @@ def user_data_get(user_id,key,default=None): if key is None: raise ApiError('invalid_key', 'Please provide a valid key') dsb = DataStoreBase() - retval=dsb.get_data_common(None, - user_id, - 'api_user_data_get', - key,default) + retval = dsb.get_data_common(None, + user_id, + 'api_user_data_get', + key, default) - #json_value = json.dumps(retval, ensure_ascii=False, indent=2) # just return raw text + # json_value = json.dumps(retval, ensure_ascii=False, indent=2) # just return raw text return retval + def user_multi_get(user_id): """Get all data values in the data_store for a userid""" if user_id is None: raise ApiError('unknown_study', 'Please provide a valid UserID.') dsb = DataStoreBase() - retval=dsb.get_multi_common(None, - user_id) + retval = dsb.get_multi_common(None, + user_id) results = DataStoreSchema(many=True).dump(retval) return results - def datastore_del(id): """Delete a data store item for a user_id and a key""" session.query(DataStoreModel).filter_by(id=id).delete() @@ -116,12 +114,14 @@ def datastore_del(id): json_value = json.dumps('deleted', ensure_ascii=False, indent=2) return json_value + def datastore_get(id): """Delete a data store item for a user_id and a key""" item = session.query(DataStoreModel).filter_by(id=id).first() results = DataStoreSchema(many=False).dump(item) return results + def update_datastore(id, body): """allow a modification to a datastore item """ if id is None: @@ -134,8 +134,8 @@ def update_datastore(id, body): # I'm not sure if there is a generic way to use the # schema to both parse the body and update the SQLAlchemy record for key in body: - if hasattr(item,key): - setattr(item,key,body[key]) + if hasattr(item, key): + setattr(item, key, body[key]) item.last_updated = datetime.now() session.add(item) session.commit() @@ -146,7 +146,7 @@ def add_datastore(body): """ add a new datastore item """ print(body) - if body.get(id,None): + if body.get(id, None): raise ApiError('id_specified', 'You may not specify an id for a new datastore item') if 'key' not in body: @@ -158,17 +158,16 @@ def add_datastore(body): if (not 'user_id' in body) and (not 'study_id' in body): raise ApiError('conflicting_values', 'A datastore item should have either a study_id or a user_id') - if 'user_id' in body and 'study_id' in body: raise ApiError('conflicting_values', 'A datastore item should have either a study_id or a user_id, ' 'but not both') - item = DataStoreModel(key=body['key'],value=body['value']) + item = DataStoreModel(key=body['key'], value=body['value']) # I'm not sure if there is a generic way to use the # schema to both parse the body and update the SQLAlchemy record for key in body: - if hasattr(item,key): - setattr(item,key,body[key]) + if hasattr(item, key): + setattr(item, key, body[key]) item.last_updated = datetime.now() session.add(item) session.commit() diff --git a/crc/scripts/data_store_base.py b/crc/scripts/data_store_base.py new file mode 100644 index 00000000..05934052 --- /dev/null +++ b/crc/scripts/data_store_base.py @@ -0,0 +1,80 @@ +import importlib +import os +import pkgutil +from crc import session +from crc.api.common import ApiError +from crc.models.data_store import DataStoreModel +from crc.models.workflow import WorkflowModel +from datetime import datetime + + +class DataStoreBase(object): + + def overwritten(self, value, prev_value): + if prev_value is None: + overwritten = False + else: + if prev_value == value: + overwritten = False + else: + overwritten = True + return overwritten + + def set_validate_common(self, study_id, workflow_id, user_id, script_name, *args): + self.check_args_2(args, script_name) + workflow = session.query(WorkflowModel).filter(WorkflowModel.id == workflow_id).first() + self.get_prev_value(study_id=study_id, user_id=user_id, key=args[0]) + + def check_args(self, args, maxlen=1, script_name='study_data_get'): + if len(args) < 1 or len(args) > maxlen: + raise ApiError(code="missing_argument", + message=f"The {script_name} script takes either one or two arguments, " + f"starting with the key and an optional default") + + def check_args_2(self, args, script_name='study_data_set'): + if len(args) != 2: + raise ApiError(code="missing_argument", + message=f"The {script_name} script takes two arguments, starting with the key and a " + "value for the key") + + def get_prev_value(self, study_id, user_id, key): + study = session.query(DataStoreModel).filter_by(study_id=study_id, user_id=user_id, key=key).first() + return study + + def set_data_common(self, task_id, study_id, user_id, workflow_id, workflow_spec_id, script_name, *args, **kwargs): + + self.check_args_2(args, script_name=script_name) + study = self.get_prev_value(study_id=study_id, user_id=user_id, key=args[0]) + if workflow_spec_id is None and workflow_id is not None: + workflow = session.query(WorkflowModel).filter(WorkflowModel.id == workflow_id).first() + workflow_spec_id = workflow.workflow_spec_id + if study is not None: + prev_value = study.value + else: + prev_value = None + study = DataStoreModel(key=args[0], value=args[1], + study_id=study_id, + task_id=task_id, + user_id=user_id, # Make this available to any User + workflow_id=workflow_id, + spec_id=workflow_spec_id) + study.value = args[1] + study.last_updated = datetime.now() + overwritten = self.overwritten(study.value, prev_value) + session.add(study) + session.commit() + return {'new_value': study.value, + 'old_value': prev_value, + 'overwritten': overwritten} + + def get_data_common(self, study_id, user_id, script_name, *args): + self.check_args(args, 2, script_name) + study = session.query(DataStoreModel).filter_by(study_id=study_id, user_id=user_id, key=args[0]).first() + if study: + return study.value + else: + return args[1] + + def get_multi_common(self, study_id, user_id): + study = session.query(DataStoreModel).filter_by(study_id=study_id, user_id=user_id) + return study diff --git a/crc/scripts/script.py b/crc/scripts/script.py index 9ccfea12..45c4c221 100644 --- a/crc/scripts/script.py +++ b/crc/scripts/script.py @@ -1,11 +1,9 @@ import importlib import os import pkgutil -from crc import session + from crc.api.common import ApiError -from crc.models.data_store import DataStoreModel -from crc.models.workflow import WorkflowModel -from datetime import datetime + class Script(object): """ Provides an abstract class that defines how scripts should work, this @@ -70,9 +68,6 @@ class Script(object): workflow_id) return execlist - - - @staticmethod def get_all_subclasses(): return Script._get_all_subclasses(Script) @@ -112,77 +107,5 @@ class ScriptValidationError: def from_api_error(cls, api_error: ApiError): return cls(api_error.code, api_error.message) -class DataStoreBase(): - def overwritten(self,value,prev_value): - if prev_value == None: - overwritten = False - else: - if prev_value == value: - overwritten = False - else: - overwritten = True - return overwritten - - def set_validate_common(self, study_id, workflow_id, user_id, script_name, *args): - self.check_args_2(args,script_name) - workflow = session.query(WorkflowModel).filter(WorkflowModel.id == workflow_id).first() - self.get_prev_value(study_id=study_id,user_id=user_id, key=args[0]) - - def check_args(self, args, maxlen=1, script_name='study_data_get'): - if len(args) < 1 or len(args) > maxlen : - raise ApiError(code="missing_argument", - - message="The %s script takes either one or two arguments, starting with the key and an "%script_name + \ - "optional default") - - def check_args_2(self, args, script_name='study_data_set'): - if len(args) != 2: - raise ApiError(code="missing_argument", - message="The %s script takes two arguments, starting with the key and a "%script_name +\ - "value for the key") - - def get_prev_value(self,study_id,user_id,key): - study = session.query(DataStoreModel).filter_by(study_id=study_id,user_id=user_id,key=key).first() - return study - - def set_data_common(self, task_id, study_id, user_id, workflow_id, workflow_spec_id, script_name, *args, **kwargs): - - self.check_args_2(args,script_name=script_name) - study = self.get_prev_value(study_id=study_id,user_id=user_id, key=args[0]) - if workflow_spec_id is None and workflow_id is not None: - workflow = session.query(WorkflowModel).filter(WorkflowModel.id == workflow_id).first() - workflow_spec_id = workflow.workflow_spec_id - if study is not None: - prev_value = study.value - else: - prev_value = None - study = DataStoreModel(key=args[0],value=args[1], - study_id=study_id, - task_id=task_id, - user_id=user_id, # Make this available to any User - workflow_id= workflow_id, - spec_id=workflow_spec_id) - study.value = args[1] - study.last_updated = datetime.now() - overwritten = self.overwritten(study.value,prev_value) - session.add(study) - session.commit() - return {'new_value':study.value, - 'old_value':prev_value, - 'overwritten':overwritten} - - - def get_data_common(self, study_id, user_id, script_name, *args): - self.check_args(args,2,script_name) - study = session.query(DataStoreModel).filter_by(study_id=study_id,user_id=user_id,key=args[0]).first() - if study: - return study.value - else: - return args[1] - - def get_multi_common(self, study_id, user_id): - study = session.query(DataStoreModel).filter_by(study_id=study_id,user_id=user_id) - return (study) - diff --git a/crc/scripts/study_data_get.py b/crc/scripts/study_data_get.py index 13a55bbe..4b109dff 100644 --- a/crc/scripts/study_data_get.py +++ b/crc/scripts/study_data_get.py @@ -1,14 +1,11 @@ -import requests +from crc.scripts.data_store_base import DataStoreBase +from crc.scripts.script import Script -from crc.scripts.script import Script, DataStoreBase -from crc import session -from crc.models.data_store import DataStoreModel class StudyDataGet(Script,DataStoreBase): def get_description(self): return """Gets study data from the data store.""" - def do_task_validate_only(self, task, study_id, workflow_id, *args, **kwargs): self.do_task(task, study_id, workflow_id, *args, **kwargs) diff --git a/crc/scripts/study_data_set.py b/crc/scripts/study_data_set.py index 891e1b2b..dffc1bd0 100644 --- a/crc/scripts/study_data_set.py +++ b/crc/scripts/study_data_set.py @@ -1,15 +1,11 @@ -import requests +from crc.scripts.data_store_base import DataStoreBase +from crc.scripts.script import Script -from crc.scripts.script import Script, DataStoreBase -from crc import session -from crc.models.workflow import WorkflowModel -from crc.models.data_store import DataStoreModel class StudyDataSet(Script,DataStoreBase): def get_description(self): return """Sets study data from the data store.""" - def do_task_validate_only(self, task, study_id, workflow_id, *args, **kwargs): self.set_validate_common(study_id, workflow_id, @@ -17,7 +13,6 @@ class StudyDataSet(Script,DataStoreBase): 'study_data_set', *args) - def do_task(self, task, study_id, workflow_id, *args, **kwargs): return self.set_data_common(task.id, study_id, diff --git a/crc/scripts/user_data_get.py b/crc/scripts/user_data_get.py index 61f6c3c8..037ae300 100644 --- a/crc/scripts/user_data_get.py +++ b/crc/scripts/user_data_get.py @@ -1,15 +1,13 @@ -import requests - -from crc.scripts.script import Script, DataStoreBase -from crc import session -from crc.models.data_store import DataStoreModel from flask import g -class UserDataGet(Script,DataStoreBase): +from crc.scripts.data_store_base import DataStoreBase +from crc.scripts.script import Script + + +class UserDataGet(Script, DataStoreBase): def get_description(self): return """Gets user data from the data store.""" - def do_task_validate_only(self, task, study_id, workflow_id, *args, **kwargs): self.do_task(task, study_id, workflow_id, *args, **kwargs) diff --git a/crc/scripts/user_data_set.py b/crc/scripts/user_data_set.py index 0f1f9a0a..3e7b4cae 100644 --- a/crc/scripts/user_data_set.py +++ b/crc/scripts/user_data_set.py @@ -1,17 +1,13 @@ -import requests - -from crc.scripts.script import Script, DataStoreBase -from crc import session -from crc.models.workflow import WorkflowModel -from crc.models.data_store import DataStoreModel - from flask import g +from crc.scripts.data_store_base import DataStoreBase +from crc.scripts.script import Script + + class UserDataSet(Script,DataStoreBase): def get_description(self): return """Sets user data to the data store.""" - def do_task_validate_only(self, task, study_id, workflow_id, *args, **kwargs): self.set_validate_common(None, workflow_id, @@ -19,7 +15,6 @@ class UserDataSet(Script,DataStoreBase): 'user_data_set', *args) - def do_task(self, task, study_id, workflow_id, *args, **kwargs): return self.set_data_common(task.id, None, diff --git a/tests/emails/test_email_script.py b/tests/emails/test_email_script.py index 12a00fac..1385609b 100644 --- a/tests/emails/test_email_script.py +++ b/tests/emails/test_email_script.py @@ -1,12 +1,6 @@ -from tests.base_test import BaseTest - +from crc import mail from crc.models.email import EmailModel -from crc.services.file_service import FileService -from crc.scripts.email import Email -from crc.services.workflow_processor import WorkflowProcessor -from crc.api.common import ApiError - -from crc import db, mail +from tests.base_test import BaseTest class TestEmailScript(BaseTest):