Merge remote-tracking branch 'origin/feature/approval_request_script' into feature/approval_request_script

crc/api.yml

@@ -82,7 +82,7 @@ paths:
   # /v1.0/study
   /study:
     get:
-      operationId: crc.api.study.all_studies
+      operationId: crc.api.study.user_studies
       summary: Provides a list of studies related to the current user.
       tags:
         - Studies
@@ -109,11 +109,13 @@ paths:
           content:
             application/json:
               schema:
-                $ref: "#/components/schemas/Study"
-  /study-files:
+                type: array
+                items:
+                  $ref: "#/components/schemas/Study"
+  /study/all:
     get:
-      operationId: crc.api.study.all_studies_and_files
-      summary: Provides a list of studies with submitted files
+      operationId: crc.api.study.all_studies
+      summary: Provides a list of studies
       tags:
         - Studies
       responses:
@@ -122,7 +124,9 @@ paths:
           content:
             application/json:
               schema:
-                $ref: "#/components/schemas/Study"
+                type: array
+                items:
+                  $ref: "#/components/schemas/Study"
   /study/{study_id}:
     parameters:
       - name: study_id
@@ -353,24 +357,12 @@ paths:
         description: The unique id of a workflow specification
         schema:
           type: string
-      - name: study_id
-        in: query
-        required: false
-        description: The unique id of a study
-        schema:
-          type: integer
       - name: workflow_id
         in: query
         required: false
         description: The unique id of a workflow instance
         schema:
           type: integer
-      - name: task_id
-        in: query
-        required: false
-        description: The unique id of a workflow task
-        schema:
-          type: string
       - name: form_field_key
         in: query
         required: false
@@ -680,7 +672,7 @@ paths:
             application/json:
               schema:
                 $ref: "#/components/schemas/Workflow"
-  /workflow/{workflow_id}/task/{task_id}/lookup/{field_id}:
+  /workflow/{workflow_id}/lookup/{field_id}:
     parameters:
       - name: workflow_id
         in: path
@@ -689,13 +681,6 @@ paths:
         schema:
           type: integer
           format: int32
-      - name: task_id
-        in: path
-        required: true
-        description: The id of the task
-        schema:
-          type: string
-          format: uuid
       - name: field_id
         in: path
         required: true

crc/api/file.py

@@ -1,51 +1,58 @@
 import io
+from typing import List
 
 import connexion
 from flask import send_file
 
 from crc import session
 from crc.api.common import ApiError
-from crc.models.file import FileModelSchema, FileModel, FileDataModel
+from crc.models.file import FileSchema, FileModel, File, FileModelSchema
 from crc.models.workflow import WorkflowSpecModel
 from crc.services.file_service import FileService
 
 
-def get_files(workflow_spec_id=None, study_id=None, workflow_id=None, task_id=None, form_field_key=None):
-    if all(v is None for v in [workflow_spec_id, study_id, workflow_id, task_id, form_field_key]):
-        raise ApiError('missing_parameter',
-                       'Please specify at least one of workflow_spec_id, study_id, '
-                       'workflow_id, and task_id for this file in the HTTP parameters')
+def to_file_api(file_model):
+    """Converts a FileModel object to something we can return via the aip"""
+    return File.from_models(file_model, FileService.get_file_data(file_model.id))
 
-    results = FileService.get_files(workflow_spec_id, study_id, workflow_id, task_id, form_field_key)
-    return FileModelSchema(many=True).dump(results)
+
+def get_files(workflow_spec_id=None, workflow_id=None, form_field_key=None):
+    if all(v is None for v in [workflow_spec_id, workflow_id, form_field_key]):
+        raise ApiError('missing_parameter',
+                       'Please specify either a workflow_spec_id or a '
+                       'workflow_id with an optional form_field_key')
+
+    file_models = FileService.get_files(workflow_spec_id=workflow_spec_id,
+                                        workflow_id=workflow_id,
+                                        irb_doc_code=form_field_key)
+
+    files = (to_file_api(model) for model in file_models)
+    return FileSchema(many=True).dump(files)
 
 
 def get_reference_files():
     results = FileService.get_files(is_reference=True)
-    return FileModelSchema(many=True).dump(results)
+    files = (to_file_api(model) for model in results)
+    return FileSchema(many=True).dump(files)
 
 
-def add_file(workflow_spec_id=None, study_id=None, workflow_id=None, task_id=None, form_field_key=None):
-    all_none = all(v is None for v in [workflow_spec_id, study_id, workflow_id, task_id, form_field_key])
-    missing_some = (workflow_spec_id is None) and (None in [study_id, workflow_id, form_field_key])
-    if all_none or missing_some:
-        raise ApiError('missing_parameter',
-                       'Please specify either a workflow_spec_id or all 3 of study_id, '
-                       'workflow_id, and field_id for this file in the HTTP parameters')
-    if 'file' not in connexion.request.files:
-        raise ApiError('invalid_file',
-                       'Expected a file named "file" in the multipart form request')
-
+def add_file(workflow_spec_id=None, workflow_id=None, form_field_key=None):
     file = connexion.request.files['file']
-    if workflow_spec_id:
+    if workflow_id:
+        if form_field_key is None:
+            raise ApiError('invalid_workflow_file',
+                           'When adding a workflow related file, you must specify a form_field_key')
+        file_model = FileService.add_workflow_file(workflow_id=workflow_id, irb_doc_code=form_field_key,
+                                                   name=file.filename, content_type=file.content_type,
+                                                   binary_data=file.stream.read())
+    elif workflow_spec_id:
         workflow_spec = session.query(WorkflowSpecModel).filter_by(id=workflow_spec_id).first()
         file_model = FileService.add_workflow_spec_file(workflow_spec, file.filename, file.content_type,
                                                         file.stream.read())
     else:
-        file_model = FileService.add_form_field_file(study_id, workflow_id, task_id, form_field_key, file.filename,
-                                                     file.content_type, file.stream.read())
+        raise ApiError("invalid_file", "You must supply either a workflow spec id or a workflow_id and form_field_key.")
 
-    return FileModelSchema().dump(file_model)
+    return FileSchema().dump(to_file_api(file_model))
 
 
 def get_reference_file(name):
@@ -80,7 +87,7 @@ def set_reference_file(name):
         file_model = file_models[0]
         FileService.update_file(file_models[0], file.stream.read(), file.content_type)
 
-    return FileModelSchema().dump(file_model)
+    return FileSchema().dump(to_file_api(file_model))
 
 
 def update_file_data(file_id):
@@ -89,7 +96,7 @@ def update_file_data(file_id):
     if file_model is None:
         raise ApiError('no_such_file', 'The file id you provided does not exist')
     file_model = FileService.update_file(file_model, file.stream.read(), file.content_type)
-    return FileModelSchema().dump(file_model)
+    return FileSchema().dump(to_file_api(file_model))
 
 
 def get_file_data(file_id, version=None):
@@ -101,7 +108,7 @@ def get_file_data(file_id, version=None):
         attachment_filename=file_data.file_model.name,
         mimetype=file_data.file_model.content_type,
         cache_timeout=-1,  # Don't cache these files on the browser.
-        last_modified=file_data.last_updated
+        last_modified=file_data.date_created
     )
 
 
@@ -109,7 +116,7 @@ def get_file_info(file_id):
     file_model = session.query(FileModel).filter_by(id=file_id).with_for_update().first()
     if file_model is None:
         raise ApiError('no_such_file', 'The file id you provided does not exist', status_code=404)
-    return FileModelSchema().dump(file_model)
+    return FileSchema().dump(to_file_api(file_model))
 
 
 def update_file_info(file_id, body):
@@ -124,7 +131,7 @@ def update_file_info(file_id, body):
     file_model = FileModelSchema().load(body, session=session)
     session.add(file_model)
     session.commit()
-    return FileModelSchema().dump(file_model)
+    return FileSchema().dump(to_file_api(file_model))
 
 
 def delete_file(file_id):

crc/api/study.py

@@ -6,7 +6,7 @@ from sqlalchemy.exc import IntegrityError
 from crc import session
 from crc.api.common import ApiError, ApiErrorSchema
 from crc.models.protocol_builder import ProtocolBuilderStatus
-from crc.models.study import StudySchema, StudyFilesSchema, StudyModel, Study
+from crc.models.study import StudySchema, StudyModel, Study
 from crc.services.study_service import StudyService
 
 
@@ -65,7 +65,7 @@ def delete_study(study_id):
         raise ApiError(code="study_integrity_error", message=message)
 
 
-def all_studies():
+def user_studies():
     """Returns all the studies associated with the current user. """
     StudyService.synch_with_protocol_builder_if_enabled(g.user)
     studies = StudyService.get_studies_for_user(g.user)
@@ -73,8 +73,8 @@ def all_studies():
     return results
 
 
-def all_studies_and_files():
-    """Returns all studies with submitted files"""
-    studies = StudyService.get_studies_with_files()
-    results = StudyFilesSchema(many=True).dump(studies)
+def all_studies():
+    """Returns all studies (regardless of user) with submitted files"""
+    studies = StudyService.get_all_studies_with_files()
+    results = StudySchema(many=True).dump(studies)
     return results

crc/api/user.py

@@ -1,6 +1,7 @@
 import json
 
 import connexion
+import flask
 from flask import redirect, g, request
 
 from crc import app, db
@@ -109,8 +110,11 @@ def _handle_login(user_info: LdapUserInfo, redirect_url=app.config['FRONTEND_AUT
     # Return the frontend auth callback URL, with auth token appended.
     auth_token = user.encode_auth_token().decode()
     if redirect_url is not None:
-        app.logger.info("SSO_LOGIN: REDIRECTING TO: " + redirect_url)
-        return redirect('%s/%s' % (redirect_url, auth_token))
+        if redirect_url.find("http://") != 0 and redirect_url.find("https://") != 0:
+            redirect_url = "http://" + redirect_url
+        url = '%s?token=%s' % (redirect_url, auth_token)
+        app.logger.info("SSO_LOGIN: REDIRECTING TO: " + url)
+        return flask.redirect(url, code=302)
     else:
         app.logger.info("SSO_LOGIN:  NO REDIRECT, JUST RETURNING AUTH TOKEN.")
         return auth_token

crc/api/workflow.py

@@ -118,8 +118,8 @@ def __get_workflow_api_model(processor: WorkflowProcessor, next_task = None):
         next_task=None,
         navigation=navigation,
         workflow_spec_id=processor.workflow_spec_id,
-        spec_version=processor.get_spec_version(),
-        is_latest_spec=processor.get_spec_version() == processor.get_latest_version_string(processor.workflow_spec_id),
+        spec_version=processor.get_version_string(),
+        is_latest_spec=processor.is_latest_spec,
         total_tasks=processor.workflow_model.total_tasks,
         completed_tasks=processor.workflow_model.completed_tasks,
         last_updated=processor.workflow_model.last_updated
@@ -219,26 +219,13 @@ def delete_workflow_spec_category(cat_id):
     session.commit()
 
 
-def lookup(workflow_id, task_id, field_id, query, limit):
+def lookup(workflow_id, field_id, query, limit):
     """
     given a field in a task, attempts to find the lookup table or function associated
     with that field and runs a full-text query against it to locate the values and
     labels that would be returned to a type-ahead box.
+    Tries to be fast, but first runs will be very slow.
     """
-    workflow_model = session.query(WorkflowModel).filter_by(id=workflow_id).first()
-    if not workflow_model:
-        raise ApiError("unknown_workflow", "No workflow found with id: %i" % workflow_id)
-    processor = WorkflowProcessor(workflow_model)
-    task_id = uuid.UUID(task_id)
-    spiff_task = processor.bpmn_workflow.get_task(task_id)
-    if not spiff_task:
-        raise ApiError("unknown_task", "No task with %s found in workflow: %i" % (task_id, workflow_id))
-    field = None
-    for f in spiff_task.task_spec.form.fields:
-        if f.id == field_id:
-            field = f
-    if not field:
-        raise ApiError("unknown_field", "No field named %s in task %s" % (task_id, spiff_task.task_spec.name))
-
-    lookup_data = LookupService.lookup(spiff_task, field, query, limit)
+    workflow = session.query(WorkflowModel).filter(WorkflowModel.id == workflow_id).first()
+    lookup_data = LookupService.lookup(workflow, field_id, query, limit)
     return LookupDataSchema(many=True).dump(lookup_data)

crc/models/approval.py

@@ -1,14 +1,13 @@
 import enum
-import marshmallow
 
+import marshmallow
+from ldap3.core.exceptions import LDAPSocketOpenError
 from marshmallow import INCLUDE
 from sqlalchemy import func
 
-from ldap3.core.exceptions import LDAPSocketOpenError
-
 from crc import db, ma
 from crc.api.common import ApiError
-from crc.models.file import FileModel
+from crc.models.file import FileDataModel
 from crc.models.study import StudyModel
 from crc.models.workflow import WorkflowModel
 from crc.services.ldap_service import LdapService
@@ -22,13 +21,11 @@ class ApprovalStatus(enum.Enum):
 
 
 class ApprovalFile(db.Model):
-    id = db.Column(db.Integer, primary_key=True)
-    file_id = db.Column(db.Integer, db.ForeignKey(FileModel.id), nullable=False)
-    approval_id = db.Column(db.Integer, db.ForeignKey("approval.id"), nullable=False)
-    file_version = db.Column(db.Integer, nullable=False)
+    file_data_id = db.Column(db.Integer, db.ForeignKey(FileDataModel.id), primary_key=True)
+    approval_id = db.Column(db.Integer, db.ForeignKey("approval.id"), primary_key=True)
 
     approval = db.relationship("ApprovalModel")
-    file = db.relationship(FileModel)
+    file_data = db.relationship(FileDataModel)
 
 
 class ApprovalModel(db.Model):
@@ -43,9 +40,9 @@ class ApprovalModel(db.Model):
     message = db.Column(db.String, default='')
     date_created = db.Column(db.DateTime(timezone=True), default=func.now())
     version = db.Column(db.Integer) # Incremented integer, so 1,2,3 as requests are made.
-    workflow_hash = db.Column(db.String) # A hash of the workflow at the moment the approval was created.
-
-    approval_files = db.relationship(ApprovalFile, back_populates="approval")
+    approval_files = db.relationship(ApprovalFile, back_populates="approval",
+                                     cascade="all, delete, delete-orphan",
+                                     order_by=ApprovalFile.file_data_id)
 
 
 class Approval(object):
@@ -66,7 +63,6 @@ class Approval(object):
         instance.message = model.message
         instance.date_created = model.date_created
         instance.version = model.version
-        instance.workflow_hash = model.workflow_hash
         instance.title = ''
         if model.study:
             instance.title = model.study.title
@@ -91,9 +87,9 @@ class Approval(object):
         instance.associated_files = []
         for approval_file in model.approval_files:
             associated_file = {}
-            associated_file['id'] = approval_file.file.id
-            associated_file['name'] = approval_file.file.name
-            associated_file['content_type'] = approval_file.file.content_type
+            associated_file['id'] = approval_file.file_data.file_model.id
+            associated_file['name'] = approval_file.file_data.file_model.name
+            associated_file['content_type'] = approval_file.file_data.file_model.content_type
             instance.associated_files.append(associated_file)
 
         return instance

crc/models/file.py

@@ -1,16 +1,18 @@
 import enum
 from typing import cast
 
+from marshmallow import INCLUDE, EXCLUDE
 from marshmallow_enum import EnumField
 from marshmallow_sqlalchemy import SQLAlchemyAutoSchema
 from sqlalchemy import func, Index
 from sqlalchemy.dialects.postgresql import UUID
+from sqlalchemy.orm import deferred
 
 from crc import db, ma
 
 
 class FileType(enum.Enum):
-    bpmn = "bpmm"
+    bpmn = "bpmn"
     csv = 'csv'
     dmn = "dmn"
     doc = "doc"
@@ -55,15 +57,16 @@ CONTENT_TYPES = {
     "zip": "application/zip"
 }
 
+
 class FileDataModel(db.Model):
     __tablename__ = 'file_data'
     id = db.Column(db.Integer, primary_key=True)
     md5_hash = db.Column(UUID(as_uuid=True), unique=False, nullable=False)
-    data = db.Column(db.LargeBinary)
+    data = deferred(db.Column(db.LargeBinary))  # Don't load it unless you have to.
     version = db.Column(db.Integer, default=0)
-    last_updated = db.Column(db.DateTime(timezone=True), default=func.now())
+    date_created = db.Column(db.DateTime(timezone=True), default=func.now())
     file_model_id = db.Column(db.Integer, db.ForeignKey('file.id'))
-    file_model = db.relationship("FileModel")
+    file_model = db.relationship("FileModel", foreign_keys=[file_model_id])
 
 
 class FileModel(db.Model):
@@ -78,13 +81,31 @@ class FileModel(db.Model):
     primary_process_id = db.Column(db.String, nullable=True) # An id in the xml of BPMN documents, critical for primary BPMN.
     workflow_spec_id = db.Column(db.String, db.ForeignKey('workflow_spec.id'), nullable=True)
     workflow_id = db.Column(db.Integer, db.ForeignKey('workflow.id'), nullable=True)
-    study_id = db.Column(db.Integer, db.ForeignKey('study.id'), nullable=True)
-    task_id = db.Column(db.String, nullable=True)
     irb_doc_code = db.Column(db.String, nullable=True) # Code reference to the irb_documents.xlsx reference file.
-    form_field_key = db.Column(db.String, nullable=True)
-    latest_version = db.Column(db.Integer, default=0)
 
 
+class File(object):
+    @classmethod
+    def from_models(cls, model: FileModel, data_model: FileDataModel):
+        instance = cls()
+        instance.id = model.id
+        instance.name = model.name
+        instance.is_status = model.is_status
+        instance.is_reference = model.is_reference
+        instance.content_type = model.content_type
+        instance.primary = model.primary
+        instance.primary_process_id = model.primary_process_id
+        instance.workflow_spec_id = model.workflow_spec_id
+        instance.workflow_id = model.workflow_id
+        instance.irb_doc_code = model.irb_doc_code
+        instance.type = model.type
+        if data_model:
+            instance.last_modified = data_model.date_created
+            instance.latest_version = data_model.version
+        else:
+            instance.last_modified = None
+            instance.latest_version = None
+        return instance
 
 class FileModelSchema(SQLAlchemyAutoSchema):
     class Meta:
@@ -92,29 +113,37 @@ class FileModelSchema(SQLAlchemyAutoSchema):
         load_instance = True
         include_relationships = True
         include_fk = True  # Includes foreign keys
+        unknown = EXCLUDE
+    type = EnumField(FileType)
+
+
+class FileSchema(ma.Schema):
+    class Meta:
+        model = File
+        fields = ["id", "name", "is_status", "is_reference", "content_type",
+                  "primary", "primary_process_id", "workflow_spec_id", "workflow_id",
+                  "irb_doc_code", "last_modified", "latest_version", "type"]
+        unknown = INCLUDE
     type = EnumField(FileType)
 
 
 class LookupFileModel(db.Model):
-    """Takes the content of a file (like a xlsx, or csv file) and creates a key/value
-    store that can be used for lookups and searches. This table contains the metadata,
-    so we know the version of the file that was used, and what key column, and value column
-    were used to generate this lookup table.  ie, the same xls file might have multiple
-    lookup file models, if different keys and labels are used - or someone decides to
-    make a change.  We need to handle full text search over the label and value columns,
-    and not every column, because we don't know how much information will be in there. """
+    """Gives us a quick way to tell what kind of lookup is set on a form field.
+    Connected to the file data model, so that if a new version of the same file is
+    created, we can update the listing."""
+    #fixme: What happens if they change the file associated with a lookup field?
     __tablename__ = 'lookup_file'
     id = db.Column(db.Integer, primary_key=True)
-    label_column = db.Column(db.String)
-    value_column = db.Column(db.String)
+    workflow_spec_id = db.Column(db.String)
+    field_id = db.Column(db.String)
+    is_ldap = db.Column(db.Boolean)  # Allows us to run an ldap query instead of a db lookup.
     file_data_model_id = db.Column(db.Integer, db.ForeignKey('file_data.id'))
-
+    dependencies = db.relationship("LookupDataModel", lazy="select", backref="lookup_file_model", cascade="all, delete, delete-orphan")
 
 class LookupDataModel(db.Model):
     __tablename__ = 'lookup_data'
     id = db.Column(db.Integer, primary_key=True)
     lookup_file_model_id = db.Column(db.Integer, db.ForeignKey('lookup_file.id'))
-    lookup_file_model = db.relationship(LookupFileModel)
     value = db.Column(db.String)
     label = db.Column(db.String)
     # In the future, we might allow adding an additional "search" column if we want to search things not in label.

crc/models/study.py

@@ -40,10 +40,6 @@ class StudyModel(db.Model):
         if self.on_hold:
             self.protocol_builder_status = ProtocolBuilderStatus.HOLD
 
-    def files(self):
-        _files = FileModel.query.filter_by(workflow_id=self.workflow[0].id)
-        return _files
-
 
 class WorkflowMetadata(object):
     def __init__(self, id, name, display_name, description, spec_version, category_id, state: WorkflowState, status: WorkflowStatus,
@@ -68,7 +64,7 @@ class WorkflowMetadata(object):
             name=workflow.workflow_spec.name,
             display_name=workflow.workflow_spec.display_name,
             description=workflow.workflow_spec.description,
-            spec_version=workflow.spec_version,
+            spec_version=workflow.spec_version(),
             category_id=workflow.workflow_spec.category_id,
             state=WorkflowState.optional,
             status=workflow.status,
@@ -122,7 +118,7 @@ class Study(object):
         self.ind_number = ind_number
         self.categories = categories
         self.warnings = []
-
+        self.files = []
 
     @classmethod
     def from_model(cls, study_model: StudyModel):
@@ -153,6 +149,7 @@ class StudySchema(ma.Schema):
     hsr_number = fields.String(allow_none=True)
     sponsor = fields.String(allow_none=True)
     ind_number = fields.String(allow_none=True)
+    files = fields.List(fields.Nested(SimpleFileSchema), dump_only=True)
 
     class Meta:
         model = Study
@@ -165,14 +162,3 @@ class StudySchema(ma.Schema):
         """Can load the basic study data for updates to the database, but categories are write only"""
         return Study(**data)
 
-
-class StudyFilesSchema(ma.Schema):
-
-    files = fields.Method('_files')
-
-    class Meta:
-        model = Study
-        additional = ["id", "title", "last_updated", "primary_investigator_id"]
-
-    def _files(self, obj):
-        return [file.name for file in obj.files()]

crc/models/workflow.py

@@ -5,6 +5,7 @@ from marshmallow import EXCLUDE
 from marshmallow_sqlalchemy import SQLAlchemyAutoSchema
 
 from crc import db
+from crc.models.file import FileModel, FileDataModel
 
 
 class WorkflowSpecCategoryModel(db.Model):
@@ -67,6 +68,14 @@ class WorkflowStatus(enum.Enum):
     complete = "complete"
 
 
+class WorkflowSpecDependencyFile(db.Model):
+    """Connects a workflow to the version of the specification files it depends on to execute"""
+    file_data_id = db.Column(db.Integer, db.ForeignKey(FileDataModel.id), primary_key=True)
+    workflow_id = db.Column(db.Integer, db.ForeignKey("workflow.id"), primary_key=True)
+
+    file_data = db.relationship(FileDataModel)
+
+
 class WorkflowModel(db.Model):
     __tablename__ = 'workflow'
     id = db.Column(db.Integer, primary_key=True)
@@ -76,7 +85,13 @@ class WorkflowModel(db.Model):
     study = db.relationship("StudyModel", backref='workflow')
     workflow_spec_id = db.Column(db.String, db.ForeignKey('workflow_spec.id'))
     workflow_spec = db.relationship("WorkflowSpecModel")
-    spec_version = db.Column(db.String)
     total_tasks = db.Column(db.Integer, default=0)
     completed_tasks = db.Column(db.Integer, default=0)
     last_updated = db.Column(db.DateTime)
+    # Order By is important or generating hashes on reviews.
+    dependencies = db.relationship(WorkflowSpecDependencyFile, cascade="all, delete, delete-orphan",
+                                   order_by="WorkflowSpecDependencyFile.file_data_id")
+
+    def spec_version(self):
+        dep_ids = list(dep.file_data_id for dep in self.dependencies)
+        return "-".join(str(dep_ids))

crc/scripts/complete_template.py

@@ -36,14 +36,11 @@ Takes two arguments:
         final_document_stream = self.process_template(task, study_id, workflow, *args, **kwargs)
         file_name = args[0]
         irb_doc_code = args[1]
-        FileService.add_task_file(study_id=study_id,
-                                  workflow_id=workflow_id,
-                                  workflow_spec_id=workflow.workflow_spec_id,
-                                  task_id=task.id,
-                                  name=file_name,
-                                  content_type=CONTENT_TYPES['docx'],
-                                  binary_data=final_document_stream.read(),
-                                  irb_doc_code=irb_doc_code)
+        FileService.add_workflow_file(workflow_id=workflow_id,
+                                      name=file_name,
+                                      content_type=CONTENT_TYPES['docx'],
+                                      binary_data=final_document_stream.read(),
+                                      irb_doc_code=irb_doc_code)
 
     def process_template(self, task, study_id, workflow=None, *args, **kwargs):
         """Entry point, mostly worried about wiring it all up."""
@@ -62,13 +59,13 @@ Takes two arguments:
 
         file_data_model = None
         if workflow is not None:
-            # Get the workflow's latest files
-            joined_file_data_models = WorkflowProcessor\
-                .get_file_models_for_version(workflow.workflow_spec_id, workflow.spec_version)
-
-            for joined_file_data in joined_file_data_models:
-                if joined_file_data.file_model.name == file_name:
-                    file_data_model = session.query(FileDataModel).filter_by(id=joined_file_data.id).first()
+            # Get the workflow specification file with the given name.
+            file_data_models = FileService.get_spec_data_files(
+                workflow_spec_id=workflow.workflow_spec_id,
+                workflow_id=workflow.id)
+            for file_data in file_data_models:
+                if file_data.file_model.name == file_name:
+                    file_data_model = file_data
 
         if workflow is None or file_data_model is None:
             file_data_model = FileService.get_workflow_file_data(task.workflow, file_name)

crc/services/approval_service.py

@@ -3,8 +3,10 @@ from datetime import datetime
 from sqlalchemy import desc
 
 from crc import db, session
+from crc.api.common import ApiError
 
 from crc.models.approval import ApprovalModel, ApprovalStatus, ApprovalFile
+from crc.models.workflow import WorkflowModel
 from crc.services.file_service import FileService
 
 
@@ -51,15 +53,22 @@ class ApprovalService(object):
 
         # Construct as hash of the latest files to see if things have changed since
         # the last approval.
-        latest_files = FileService.get_workflow_files(workflow_id)
-        current_workflow_hash = ApprovalService._generate_workflow_hash(latest_files)
+        workflow = db.session.query(WorkflowModel).filter(WorkflowModel.id == workflow_id).first()
+        workflow_data_files = FileService.get_workflow_data_files(workflow_id)
+        current_data_file_ids = list(data_file.id for data_file in workflow_data_files)
+
+        if len(current_data_file_ids) == 0:
+            raise ApiError("invalid_workflow_approval", "You can't create an approval for a workflow that has"
+                                                        "no files to approve in it.")
 
         # If an existing approval request exists and no changes were made, do nothing.
         # If there is an existing approval request for a previous version of the workflow
         # then add a new request, and cancel any waiting/pending requests.
         if latest_approval_request:
-            # We could just compare the ApprovalFile lists here and do away with this hash.
-            if latest_approval_request.workflow_hash == current_workflow_hash:
+            request_file_ids = list(file.file_data_id for file in latest_approval_request.approval_files)
+            current_data_file_ids.sort()
+            request_file_ids.sort()
+            if current_data_file_ids == request_file_ids:
                 return  # This approval already exists.
             else:
                 latest_approval_request.status = ApprovalStatus.CANCELED.value
@@ -71,27 +80,18 @@ class ApprovalService(object):
         model = ApprovalModel(study_id=study_id, workflow_id=workflow_id,
                               approver_uid=approver_uid, status=ApprovalStatus.WAITING.value,
                               message="", date_created=datetime.now(),
-                              version=version, workflow_hash=current_workflow_hash)
-        approval_files = ApprovalService._create_approval_files(latest_files, model)
+                              version=version)
+        approval_files = ApprovalService._create_approval_files(workflow_data_files, model)
         db.session.add(model)
         db.session.add_all(approval_files)
         db.session.commit()
 
     @staticmethod
-    def _create_approval_files(files, approval):
+    def _create_approval_files(workflow_data_files, approval):
         """Currently based exclusively on the status of files associated with a workflow."""
         file_approval_models = []
-        for file in files:
-            file_approval_models.append(ApprovalFile(file_id=file.id,
-                                                     approval=approval,
-                                                     file_version=file.latest_version))
+        for file_data in workflow_data_files:
+            file_approval_models.append(ApprovalFile(file_data_id=file_data.id,
+                                                     approval=approval))
         return file_approval_models
 
-    @staticmethod
-    def _generate_workflow_hash(files):
-        """Currently based exclusively on the status of files associated with a workflow."""
-        version_array = []
-        for file in files:
-            version_array.append(str(file.id) + "[" + str(file.latest_version) + "]")
-        full_version = "-".join(version_array)
-        return full_version

crc/services/file_service.py

@@ -5,13 +5,14 @@ from datetime import datetime
 from uuid import UUID
 from xml.etree import ElementTree
 
+from SpiffWorkflow.bpmn.parser.ValidationException import ValidationException
 from pandas import ExcelFile
+from sqlalchemy import desc
 
 from crc import session
 from crc.api.common import ApiError
 from crc.models.file import FileType, FileDataModel, FileModel, LookupFileModel, LookupDataModel
-from crc.models.workflow import WorkflowSpecModel
-from crc.services.workflow_processor import WorkflowProcessor
+from crc.models.workflow import WorkflowSpecModel, WorkflowModel, WorkflowSpecDependencyFile
 
 
 class FileService(object):
@@ -40,31 +41,27 @@ class FileService(object):
         return code in df['code'].values
 
     @staticmethod
-    def add_form_field_file(study_id, workflow_id, task_id, form_field_key, name, content_type, binary_data):
-        """Create a new file and associate it with a user task form field within a workflow.
-        Please note that the form_field_key MUST be a known file in the irb_documents.xslx reference document."""
-        if not FileService.is_allowed_document(form_field_key):
+    def add_workflow_file(workflow_id, irb_doc_code, name, content_type, binary_data):
+        """Create a new file and associate it with the workflow
+        Please note that the irb_doc_code MUST be a known file in the irb_documents.xslx reference document."""
+        if not FileService.is_allowed_document(irb_doc_code):
             raise ApiError("invalid_form_field_key",
                            "When uploading files, the form field id must match a known document in the "
-                           "irb_docunents.xslx reference file.  This code is not found in that file '%s'" % form_field_key)
+                           "irb_docunents.xslx reference file.  This code is not found in that file '%s'" % irb_doc_code)
 
         """Assure this is unique to the workflow, task, and document code AND the Name
            Because we will allow users to upload multiple files for the same form field 
             in some cases """
         file_model = session.query(FileModel)\
             .filter(FileModel.workflow_id == workflow_id)\
-            .filter(FileModel.task_id == str(task_id))\
             .filter(FileModel.name == name)\
-            .filter(FileModel.irb_doc_code == form_field_key).first()
+            .filter(FileModel.irb_doc_code == irb_doc_code).first()
 
         if not file_model:
             file_model = FileModel(
-                study_id=study_id,
                 workflow_id=workflow_id,
-                task_id=task_id,
                 name=name,
-                form_field_key=form_field_key,
-                irb_doc_code=form_field_key
+                irb_doc_code=irb_doc_code
             )
         return FileService.update_file(file_model, binary_data, content_type)
 
@@ -85,28 +82,6 @@ class FileService(object):
         df = df.set_index(index_column)
         return json.loads(df.to_json(orient='index'))
 
-    @staticmethod
-    def add_task_file(study_id, workflow_id, workflow_spec_id, task_id, name, content_type, binary_data,
-                      irb_doc_code=None):
-
-        """Assure this is unique to the workflow, task, and document code.  Disregard name."""
-        file_model = session.query(FileModel)\
-            .filter(FileModel.workflow_id == workflow_id)\
-            .filter(FileModel.task_id == str(task_id))\
-            .filter(FileModel.irb_doc_code == irb_doc_code).first()
-
-        if not file_model:
-            """Create a new file and associate it with an executing task within a workflow."""
-            file_model = FileModel(
-                study_id=study_id,
-                workflow_id=workflow_id,
-                workflow_spec_id=workflow_spec_id,
-                task_id=task_id,
-                name=name,
-                irb_doc_code=irb_doc_code
-            )
-        return FileService.update_file(file_model, binary_data, content_type)
-
     @staticmethod
     def get_workflow_files(workflow_id):
         """Returns all the file models associated with a running workflow."""
@@ -136,12 +111,12 @@ class FileService(object):
     def update_file(file_model, binary_data, content_type):
         session.flush()  # Assure the database is up-to-date before running this.
 
-        file_data_model = session.query(FileDataModel). \
-            filter_by(file_model_id=file_model.id,
-                      version=file_model.latest_version
-                      ).with_for_update().first()
+        latest_data_model = session.query(FileDataModel). \
+            filter(FileDataModel.file_model_id == file_model.id).\
+            order_by(desc(FileDataModel.date_created)).first()
+
         md5_checksum = UUID(hashlib.md5(binary_data).hexdigest())
-        if (file_data_model is not None) and (md5_checksum == file_data_model.md5_hash):
+        if (latest_data_model is not None) and (md5_checksum == latest_data_model.md5_hash):
             # This file does not need to be updated, it's the same file.
             return file_model
 
@@ -155,22 +130,20 @@ class FileService(object):
             file_model.type = FileType[file_extension]
             file_model.content_type = content_type
 
-        if file_data_model is None:
+        if latest_data_model is None:
             version = 1
         else:
-            version = file_data_model.version + 1
+            version = latest_data_model.version + 1
 
         # If this is a BPMN, extract the process id.
         if file_model.type == FileType.bpmn:
             bpmn: ElementTree.Element = ElementTree.fromstring(binary_data)
-            file_model.primary_process_id = WorkflowProcessor.get_process_id(bpmn)
+            file_model.primary_process_id = FileService.get_process_id(bpmn)
 
-        file_model.latest_version = version
         new_file_data_model = FileDataModel(
             data=binary_data, file_model_id=file_model.id, file_model=file_model,
-            version=version, md5_hash=md5_checksum, last_updated=datetime.now()
+            version=version, md5_hash=md5_checksum, date_created=datetime.now()
         )
-
         session.add_all([file_model, new_file_data_model])
         session.commit()
         session.flush()  # Assure the id is set on the model before returning it.
@@ -178,49 +151,103 @@ class FileService(object):
         return file_model
 
     @staticmethod
-    def get_files(workflow_spec_id=None,
-                  study_id=None, workflow_id=None, task_id=None, form_field_key=None,
+    def get_process_id(et_root: ElementTree.Element):
+        process_elements = []
+        for child in et_root:
+            if child.tag.endswith('process') and child.attrib.get('isExecutable', False):
+                process_elements.append(child)
+
+        if len(process_elements) == 0:
+            raise ValidationException('No executable process tag found')
+
+        # There are multiple root elements
+        if len(process_elements) > 1:
+
+            # Look for the element that has the startEvent in it
+            for e in process_elements:
+                this_element: ElementTree.Element = e
+                for child_element in list(this_element):
+                    if child_element.tag.endswith('startEvent'):
+                        return this_element.attrib['id']
+
+            raise ValidationException('No start event found in %s' % et_root.attrib['id'])
+
+        return process_elements[0].attrib['id']
+
+    @staticmethod
+    def get_files_for_study(study_id, irb_doc_code=None):
+        query = session.query(FileModel).\
+                join(WorkflowModel).\
+                filter(WorkflowModel.study_id == study_id)
+        if irb_doc_code:
+            query = query.filter(FileModel.irb_doc_code == irb_doc_code)
+        return query.all()
+
+    @staticmethod
+    def get_files(workflow_spec_id=None, workflow_id=None,
                   name=None, is_reference=False, irb_doc_code=None):
         query = session.query(FileModel).filter_by(is_reference=is_reference)
         if workflow_spec_id:
             query = query.filter_by(workflow_spec_id=workflow_spec_id)
-        if all(v is None for v in [study_id, workflow_id, task_id, form_field_key]):
-            query = query.filter_by(
-                study_id=None,
-                workflow_id=None,
-                task_id=None,
-                form_field_key=None,
-            )
-        else:
-            if study_id:
-                query = query.filter_by(study_id=study_id)
-            if workflow_id:
-                query = query.filter_by(workflow_id=workflow_id)
-            if task_id:
-                query = query.filter_by(task_id=str(task_id))
-            if form_field_key:
-                query = query.filter_by(form_field_key=form_field_key)
-            if name:
-                query = query.filter_by(name=name)
+        elif workflow_id:
+            query = query.filter_by(workflow_id=workflow_id)
             if irb_doc_code:
                 query = query.filter_by(irb_doc_code=irb_doc_code)
+        elif is_reference:
+            query = query.filter_by(is_reference=True)
+
+        if name:
+            query = query.filter_by(name=name)
+        query = query.order_by(FileModel.id)
 
         results = query.all()
         return results
 
     @staticmethod
-    def get_file_data(file_id, file_model=None, version=None):
+    def get_spec_data_files(workflow_spec_id, workflow_id=None, name=None):
+        """Returns all the FileDataModels related to a workflow specification.
+        If a workflow is specified, returns the version of the spec relatted
+        to that workflow, otherwise, returns the lastes files."""
+        if workflow_id:
+            query = session.query(FileDataModel) \
+                    .join(WorkflowSpecDependencyFile) \
+                    .filter(WorkflowSpecDependencyFile.workflow_id == workflow_id) \
+                    .order_by(FileDataModel.id)
+            if name:
+                query = query.join(FileModel).filter(FileModel.name == name)
+            return query.all()
+        else:
+            """Returns all the latest files related to a workflow specification"""
+            file_models = FileService.get_files(workflow_spec_id=workflow_spec_id)
+            latest_data_files = []
+            for file_model in file_models:
+                if name and file_model.name == name:
+                    latest_data_files.append(FileService.get_file_data(file_model.id))
+                elif not name:
+                    latest_data_files.append(FileService.get_file_data(file_model.id))
+            return latest_data_files
 
-        """Returns the file_data that is associated with the file model id, if an actual file_model
-        is provided, uses that rather than looking it up again."""
-        if file_model is None:
-            file_model = session.query(FileModel).filter(FileModel.id == file_id).first()
-        if version is None:
-            version = file_model.latest_version
-        return session.query(FileDataModel) \
-            .filter(FileDataModel.file_model_id == file_id) \
-            .filter(FileDataModel.version == version) \
-            .first()
+    @staticmethod
+    def get_workflow_data_files(workflow_id=None):
+        """Returns all the FileDataModels related to a running workflow -
+        So these are the latest data files that were uploaded or generated
+        that go along with this workflow.  Not related to the spec in any way"""
+        file_models = FileService.get_files(workflow_id=workflow_id)
+        latest_data_files = []
+        for file_model in file_models:
+            latest_data_files.append(FileService.get_file_data(file_model.id))
+        return latest_data_files
+
+    @staticmethod
+    def get_file_data(file_id: int, version: int = None):
+        """Returns the file data with the given version, or the lastest file, if version isn't provided."""
+        query = session.query(FileDataModel) \
+            .filter(FileDataModel.file_model_id == file_id)
+        if version:
+            query = query.filter(FileDataModel.version == version)
+        else:
+            query = query.order_by(desc(FileDataModel.date_created))
+        return query.first()
 
     @staticmethod
     def get_reference_file_data(file_name):
@@ -229,7 +256,7 @@ class FileService(object):
             filter(FileModel.name == file_name).first()
         if not file_model:
             raise ApiError("file_not_found", "There is no reference file with the name '%s'" % file_name)
-        return FileService.get_file_data(file_model.id, file_model)
+        return FileService.get_file_data(file_model.id)
 
     @staticmethod
     def get_workflow_file_data(workflow, file_name):

crc/services/lookup_service.py

@@ -1,4 +1,5 @@
 import logging
+import re
 
 from pandas import ExcelFile
 from sqlalchemy import func, desc
@@ -8,8 +9,11 @@ from crc import db
 from crc.api.common import ApiError
 from crc.models.api_models import Task
 from crc.models.file import FileDataModel, LookupFileModel, LookupDataModel
+from crc.models.workflow import WorkflowModel, WorkflowSpecDependencyFile
 from crc.services.file_service import FileService
 from crc.services.ldap_service import LdapService
+from crc.services.workflow_processor import WorkflowProcessor
+
 
 class TSRank(GenericFunction):
     package = 'full_text'
@@ -31,33 +35,56 @@ class LookupService(object):
     """
 
     @staticmethod
-    def lookup(spiff_task, field, query, limit):
-        """Executes the lookup for the given field."""
-        if field.type != Task.FIELD_TYPE_AUTO_COMPLETE:
-            raise ApiError.from_task("invalid_field_type",
-                                          "Field '%s' must be an autocomplete field to use lookups." % field.label,
-                                          task=spiff_task)
-
-        # If this field has an associated options file, then do the lookup against that field.
-        if field.has_property(Task.PROP_OPTIONS_FILE):
-            lookup_table = LookupService.get_lookup_table(spiff_task, field)
-            return LookupService._run_lookup_query(lookup_table, query, limit)
-        # If this is a ldap lookup, use the ldap service to provide the fields to return.
-        elif field.has_property(Task.PROP_LDAP_LOOKUP):
-            return LookupService._run_ldap_query(query, limit)
-        else:
-            raise ApiError.from_task("unknown_lookup_option",
-                                     "Lookup supports using spreadsheet options or ldap options, and neither was"
-                                     "provided.")
+    def get_lookup_model(spiff_task, field):
+        workflow_id = spiff_task.workflow.data[WorkflowProcessor.WORKFLOW_ID_KEY]
+        workflow = db.session.query(WorkflowModel).filter(WorkflowModel.id == workflow_id).first()
+        return LookupService.__get_lookup_model(workflow, field.id)
 
     @staticmethod
-    def get_lookup_table(spiff_task, field):
-        """ Checks to see if the options are provided in a separate lookup table associated with the
+    def __get_lookup_model(workflow, field_id):
+        lookup_model = db.session.query(LookupFileModel) \
+            .filter(LookupFileModel.workflow_spec_id == workflow.workflow_spec_id) \
+            .filter(LookupFileModel.field_id == field_id).first()
+
+        # one more quick query, to see if the lookup file is still related to this workflow.
+        # if not, we need to rebuild the lookup table.
+        is_current = False
+        if lookup_model:
+            is_current = db.session.query(WorkflowSpecDependencyFile).\
+                filter(WorkflowSpecDependencyFile.file_data_id == lookup_model.file_data_model_id).count()
+
+        if not is_current:
+            if lookup_model:
+                db.session.delete(lookup_model)
+            # Very very very expensive, but we don't know need this till we do.
+            lookup_model = LookupService.create_lookup_model(workflow, field_id)
+
+        return lookup_model
+
+    @staticmethod
+    def lookup(workflow, field_id, query, limit):
+
+        lookup_model = LookupService.__get_lookup_model(workflow, field_id)
+
+        if lookup_model.is_ldap:
+            return LookupService._run_ldap_query(query, limit)
+        else:
+            return LookupService._run_lookup_query(lookup_model, query, limit)
+
+
+
+    @staticmethod
+    def create_lookup_model(workflow_model, field_id):
+        """
+         This is all really expensive, but should happen just once (per file change).
+         Checks to see if the options are provided in a separate lookup table associated with the
         workflow, and if so, assures that data exists in the database, and return a model than can be used
         to locate that data.
-
         Returns:  an array of LookupData, suitable for returning to the api.
         """
+        processor = WorkflowProcessor(workflow_model)  # VERY expensive, Ludicrous for lookup / type ahead
+        spiff_task, field = processor.find_task_and_field_by_field_id(field_id)
+
         if field.has_property(Task.PROP_OPTIONS_FILE):
             if not field.has_property(Task.PROP_OPTIONS_VALUE_COLUMN) or \
                     not field.has_property(Task.PROP_OPTIONS_LABEL_COL):
@@ -72,52 +99,67 @@ class LookupService(object):
             file_name = field.get_property(Task.PROP_OPTIONS_FILE)
             value_column = field.get_property(Task.PROP_OPTIONS_VALUE_COLUMN)
             label_column = field.get_property(Task.PROP_OPTIONS_LABEL_COL)
-            data_model = FileService.get_workflow_file_data(spiff_task.workflow, file_name)
-            lookup_model = LookupService.get_lookup_table_from_data_model(data_model, value_column, label_column)
-            return lookup_model
+            latest_files = FileService.get_spec_data_files(workflow_spec_id=workflow_model.workflow_spec_id,
+                                                           workflow_id=workflow_model.id,
+                                                           name=file_name)
+            if len(latest_files) < 1:
+                raise ApiError("missing_file", "Unable to locate the lookup data file '%s'" % file_name)
+            else:
+                data_model = latest_files[0]
+
+            lookup_model = LookupService.build_lookup_table(data_model, value_column, label_column,
+                                                            workflow_model.workflow_spec_id, field_id)
+
+        elif field.has_property(Task.PROP_LDAP_LOOKUP):
+            lookup_model = LookupFileModel(workflow_spec_id=workflow_model.workflow_spec_id,
+                                           field_id=field_id,
+                                           is_ldap=True)
+        else:
+            raise ApiError("unknown_lookup_option",
+                            "Lookup supports using spreadsheet options or ldap options, and neither "
+                            "was provided.")
+        db.session.add(lookup_model)
+        db.session.commit()
+        return lookup_model
 
     @staticmethod
-    def get_lookup_table_from_data_model(data_model: FileDataModel, value_column, label_column):
+    def build_lookup_table(data_model: FileDataModel, value_column, label_column, workflow_spec_id, field_id):
         """ In some cases the lookup table can be very large.  This method will add all values to the database
          in a way that can be searched and returned via an api call - rather than sending the full set of
           options along with the form.  It will only open the file and process the options if something has
           changed.  """
+        xls = ExcelFile(data_model.data)
+        df = xls.parse(xls.sheet_names[0])  # Currently we only look at the fist sheet.
+        if value_column not in df:
+            raise ApiError("invalid_emum",
+                           "The file %s does not contain a column named % s" % (data_model.file_model.name,
+                                                                                value_column))
+        if label_column not in df:
+            raise ApiError("invalid_emum",
+                           "The file %s does not contain a column named % s" % (data_model.file_model.name,
+                                                                                label_column))
 
-        lookup_model = db.session.query(LookupFileModel) \
-            .filter(LookupFileModel.file_data_model_id == data_model.id) \
-            .filter(LookupFileModel.value_column == value_column) \
-            .filter(LookupFileModel.label_column == label_column).first()
-
-        if not lookup_model:
-            xls = ExcelFile(data_model.data)
-            df = xls.parse(xls.sheet_names[0])  # Currently we only look at the fist sheet.
-            if value_column not in df:
-                raise ApiError("invalid_emum",
-                               "The file %s does not contain a column named % s" % (data_model.file_model.name,
-                                                                                    value_column))
-            if label_column not in df:
-                raise ApiError("invalid_emum",
-                               "The file %s does not contain a column named % s" % (data_model.file_model.name,
-                                                                                    label_column))
-
-            lookup_model = LookupFileModel(label_column=label_column, value_column=value_column,
-                                           file_data_model_id=data_model.id)
-
-            db.session.add(lookup_model)
-            for index, row in df.iterrows():
-                lookup_data = LookupDataModel(lookup_file_model=lookup_model,
-                                              value=row[value_column],
-                                              label=row[label_column],
-                                              data=row.to_json())
-                db.session.add(lookup_data)
-            db.session.commit()
+        lookup_model = LookupFileModel(workflow_spec_id=workflow_spec_id,
+                                       field_id=field_id,
+                                       file_data_model_id=data_model.id,
+                                       is_ldap=False)
 
+        db.session.add(lookup_model)
+        for index, row in df.iterrows():
+            lookup_data = LookupDataModel(lookup_file_model=lookup_model,
+                                          value=row[value_column],
+                                          label=row[label_column],
+                                          data=row.to_json())
+            db.session.add(lookup_data)
+        db.session.commit()
         return lookup_model
 
     @staticmethod
     def _run_lookup_query(lookup_file_model, query, limit):
         db_query = LookupDataModel.query.filter(LookupDataModel.lookup_file_model == lookup_file_model)
 
+        query = re.sub('[^A-Za-z0-9 ]+', '', query)
+        print("Query: " + query)
         query = query.strip()
         if len(query) > 0:
             if ' ' in query:

crc/services/study_service.py

@@ -33,10 +33,15 @@ class StudyService(object):
         return studies
 
     @staticmethod
-    def get_studies_with_files():
+    def get_all_studies_with_files():
         """Returns a list of all studies"""
         db_studies = session.query(StudyModel).all()
-        return db_studies
+        studies = []
+        for s in db_studies:
+            study = Study.from_model(s)
+            study.files = FileService.get_files_for_study(study.id)
+            studies.append(study)
+        return studies
 
     @staticmethod
     def get_study(study_id, study_model: StudyModel = None):
@@ -48,6 +53,7 @@ class StudyService(object):
         study = Study.from_model(study_model)
         study.categories = StudyService.get_categories()
         workflow_metas = StudyService.__get_workflow_metas(study_id)
+        study.files = FileService.get_files_for_study(study.id)
 
         # Calling this line repeatedly is very very slow.  It creates the
         # master spec and runs it.
@@ -72,6 +78,8 @@ class StudyService(object):
     def delete_workflow(workflow):
         for file in session.query(FileModel).filter_by(workflow_id=workflow.id).all():
             FileService.delete_file(file.id)
+        for deb in workflow.dependencies:
+            session.delete(deb)
         session.query(TaskEventModel).filter_by(workflow_id=workflow.id).delete()
         session.query(WorkflowModel).filter_by(id=workflow.id).delete()
 
@@ -150,17 +158,15 @@ class StudyService(object):
             doc['display_name'] = ' / '.join(name_list)
 
             # For each file, get associated workflow status
-            doc_files = FileService.get_files(study_id=study_id, irb_doc_code=code)
+            doc_files = FileService.get_files_for_study(study_id=study_id, irb_doc_code=code)
             doc['count'] = len(doc_files)
             doc['files'] = []
             for file in doc_files:
                 doc['files'].append({'file_id': file.id,
-                                     'task_id': file.task_id,
-                                     'workflow_id': file.workflow_id,
-                                     'workflow_spec_id': file.workflow_spec_id})
+                                     'workflow_id': file.workflow_id})
 
                 # update the document status to match the status of the workflow it is in.
-                if not 'status' in doc or doc['status'] is None:
+                if 'status' not in doc or doc['status'] is None:
                     workflow: WorkflowModel = session.query(WorkflowModel).filter_by(id=file.workflow_id).first()
                     doc['status'] = workflow.status.value
 

crc/services/workflow_processor.py

@@ -1,8 +1,7 @@
-import random
 import re
-import string
 import xml.etree.ElementTree as ElementTree
 from datetime import datetime
+from typing import List
 
 from SpiffWorkflow import Task as SpiffTask, WorkflowException
 from SpiffWorkflow.bpmn.BpmnScriptEngine import BpmnScriptEngine
@@ -13,14 +12,15 @@ from SpiffWorkflow.bpmn.workflow import BpmnWorkflow
 from SpiffWorkflow.camunda.parser.CamundaParser import CamundaParser
 from SpiffWorkflow.dmn.parser.BpmnDmnParser import BpmnDmnParser
 from SpiffWorkflow.exceptions import WorkflowTaskExecException
-from SpiffWorkflow.operators import Operator
 from SpiffWorkflow.specs import WorkflowSpec
+from sqlalchemy import desc
 
 from crc import session
 from crc.api.common import ApiError
 from crc.models.file import FileDataModel, FileModel, FileType
-from crc.models.workflow import WorkflowStatus, WorkflowModel
+from crc.models.workflow import WorkflowStatus, WorkflowModel, WorkflowSpecDependencyFile
 from crc.scripts.script import Script
+from crc.services.file_service import FileService
 
 
 class CustomBpmnScriptEngine(BpmnScriptEngine):
@@ -48,7 +48,7 @@ class CustomBpmnScriptEngine(BpmnScriptEngine):
             mod = __import__(module_name, fromlist=[class_name])
             klass = getattr(mod, class_name)
             study_id = task.workflow.data[WorkflowProcessor.STUDY_ID_KEY]
-            if(WorkflowProcessor.WORKFLOW_ID_KEY in task.workflow.data):
+            if WorkflowProcessor.WORKFLOW_ID_KEY in task.workflow.data:
                 workflow_id = task.workflow.data[WorkflowProcessor.WORKFLOW_ID_KEY]
             else:
                 workflow_id = None
@@ -75,7 +75,7 @@ class CustomBpmnScriptEngine(BpmnScriptEngine):
         Evaluate the given expression, within the context of the given task and
         return the result.
         """
-        exp,valid = self.validateExpression(expression)
+        exp, valid = self.validateExpression(expression)
         return self._eval(exp, **task.data)
 
     @staticmethod
@@ -100,7 +100,7 @@ class WorkflowProcessor(object):
     STUDY_ID_KEY = "study_id"
     VALIDATION_PROCESS_KEY = "validate_only"
 
-    def __init__(self, workflow_model: WorkflowModel, soft_reset=False, hard_reset=False):
+    def __init__(self, workflow_model: WorkflowModel, soft_reset=False, hard_reset=False, validate_only=False):
         """Create a Workflow Processor based on the serialized information available in the workflow model.
         If soft_reset is set to true, it will try to use the latest version of the workflow specification.
         If hard_reset is set to true, it will create a new Workflow, but embed the data from the last
@@ -108,18 +108,22 @@ class WorkflowProcessor(object):
         If neither flag is set, it will use the same version of the specification that was used to originally
         create the workflow model. """
         self.workflow_model = workflow_model
-        orig_version = workflow_model.spec_version
-        if soft_reset or workflow_model.spec_version is None:
-            self.workflow_model.spec_version = WorkflowProcessor.get_latest_version_string(
-                workflow_model.workflow_spec_id)
 
-        spec = self.get_spec(workflow_model.workflow_spec_id, workflow_model.spec_version)
+        if soft_reset or len(workflow_model.dependencies) == 0:
+            self.spec_data_files = FileService.get_spec_data_files(
+                workflow_spec_id=workflow_model.workflow_spec_id)
+        else:
+            self.spec_data_files = FileService.get_spec_data_files(
+                workflow_spec_id=workflow_model.workflow_spec_id,
+                workflow_id=workflow_model.id)
+
+        spec = self.get_spec(self.spec_data_files, workflow_model.workflow_spec_id)
         self.workflow_spec_id = workflow_model.workflow_spec_id
         try:
-            self.bpmn_workflow = self.__get_bpmn_workflow(workflow_model, spec)
+            self.bpmn_workflow = self.__get_bpmn_workflow(workflow_model, spec, validate_only)
             self.bpmn_workflow.script_engine = self._script_engine
 
-            if not self.WORKFLOW_ID_KEY in self.bpmn_workflow.data:
+            if self.WORKFLOW_ID_KEY not in self.bpmn_workflow.data:
                 if not workflow_model.id:
                     session.add(workflow_model)
                     session.commit()
@@ -132,71 +136,63 @@ class WorkflowProcessor(object):
                 self.save()
 
         except KeyError as ke:
-            if soft_reset:
-                # Undo the soft-reset.
-                workflow_model.spec_version = orig_version
             raise ApiError(code="unexpected_workflow_structure",
                            message="Failed to deserialize workflow"
                                    " '%s' version %s, due to a mis-placed or missing task '%s'" %
-                                   (self.workflow_spec_id, workflow_model.spec_version, str(ke)) +
-                           " This is very likely due to a soft reset where there was a structural change.")
+                                   (self.workflow_spec_id, self.get_version_string(), str(ke)) +
+                                   " This is very likely due to a soft reset where there was a structural change.")
         if hard_reset:
             # Now that the spec is loaded, get the data and rebuild the bpmn with the new details
-            workflow_model.spec_version = self.hard_reset()
+            self.hard_reset()
             workflow_model.bpmn_workflow_json = WorkflowProcessor._serializer.serialize_workflow(self.bpmn_workflow)
             self.save()
+        if soft_reset:
+            self.save()
 
-    def __get_bpmn_workflow(self, workflow_model: WorkflowModel, spec: WorkflowSpec):
+        # set whether this is the latest spec file.
+        if self.spec_data_files == FileService.get_spec_data_files(workflow_spec_id=workflow_model.workflow_spec_id):
+            self.is_latest_spec = True
+        else:
+            self.is_latest_spec = False
 
+    def __get_bpmn_workflow(self, workflow_model: WorkflowModel, spec: WorkflowSpec, validate_only=False):
         if workflow_model.bpmn_workflow_json:
             bpmn_workflow = self._serializer.deserialize_workflow(workflow_model.bpmn_workflow_json, workflow_spec=spec)
         else:
             bpmn_workflow = BpmnWorkflow(spec, script_engine=self._script_engine)
             bpmn_workflow.data[WorkflowProcessor.STUDY_ID_KEY] = workflow_model.study_id
-            bpmn_workflow.data[WorkflowProcessor.VALIDATION_PROCESS_KEY] = False
+            bpmn_workflow.data[WorkflowProcessor.VALIDATION_PROCESS_KEY] = validate_only
             bpmn_workflow.do_engine_steps()
         return bpmn_workflow
 
     def save(self):
         """Saves the current state of this processor to the database """
-        workflow_model = self.workflow_model
-        workflow_model.bpmn_workflow_json = self.serialize()
+        self.workflow_model.bpmn_workflow_json = self.serialize()
         complete_states = [SpiffTask.CANCELLED, SpiffTask.COMPLETED]
         tasks = list(self.get_all_user_tasks())
-        workflow_model.status = self.get_status()
-        workflow_model.total_tasks = len(tasks)
-        workflow_model.completed_tasks = sum(1 for t in tasks if t.state in complete_states)
-        workflow_model.last_updated = datetime.now()
-        session.add(workflow_model)
+        self.workflow_model.status = self.get_status()
+        self.workflow_model.total_tasks = len(tasks)
+        self.workflow_model.completed_tasks = sum(1 for t in tasks if t.state in complete_states)
+        self.workflow_model.last_updated = datetime.now()
+        self.update_dependencies(self.spec_data_files)
+        session.add(self.workflow_model)
         session.commit()
 
-    @staticmethod
-    def run_master_spec(spec_model, study):
-        """Executes a BPMN specification for the given study, without recording any information to the database
-        Useful for running the master specification, which should not persist. """
-        version = WorkflowProcessor.get_latest_version_string(spec_model.id)
-        spec = WorkflowProcessor.get_spec(spec_model.id, version)
-        try:
-            bpmn_workflow = BpmnWorkflow(spec, script_engine=WorkflowProcessor._script_engine)
-            bpmn_workflow.data[WorkflowProcessor.STUDY_ID_KEY] = study.id
-            bpmn_workflow.data[WorkflowProcessor.VALIDATION_PROCESS_KEY] = False
-            bpmn_workflow.do_engine_steps()
-        except WorkflowException as we:
-            raise ApiError.from_task_spec("error_running_master_spec",  str(we), we.sender)
-
-        if not bpmn_workflow.is_completed():
-            raise ApiError("master_spec_not_automatic",
-                           "The master spec should only contain fully automated tasks, it failed to complete.")
-
-        return bpmn_workflow.last_task.data
+    def get_version_string(self):
+        # this could potentially become expensive to load all the data in the data models.
+        # in which case we might consider using a deferred loader for the actual data, but
+        # trying not to pre-optimize.
+        file_data_models = FileService.get_spec_data_files(self.workflow_model.workflow_spec_id,
+                                                           self.workflow_model.id)
+        return WorkflowProcessor.__get_version_string_for_data_models(file_data_models)
 
     @staticmethod
-    def get_parser():
-        parser = MyCustomParser()
-        return parser
+    def get_latest_version_string_for_spec(spec_id):
+        file_data_models = FileService.get_spec_data_files(spec_id)
+        return WorkflowProcessor.__get_version_string_for_data_models(file_data_models)
 
     @staticmethod
-    def get_latest_version_string(workflow_spec_id):
+    def __get_version_string_for_data_models(file_data_models):
         """Version is in the format v[VERSION] (FILE_ID_LIST)
          For example, a single bpmn file with only one version would be
          v1 (12)  Where 12 is the id of the file data model that is used to create the
@@ -205,10 +201,6 @@ class WorkflowProcessor(object):
          a Spec that includes a BPMN, DMN, an a Word file all on the first
          version would be v1.1.1 (12.45.21)"""
 
-        # this could potentially become expensive to load all the data in the data models.
-        # in which case we might consider using a deferred loader for the actual data, but
-        # trying not to pre-optimize.
-        file_data_models = WorkflowProcessor.__get_latest_file_models(workflow_spec_id)
         major_version = 0  # The version of the primary file.
         minor_version = []  # The versions of the minor files if any.
         file_ids = []
@@ -224,60 +216,72 @@ class WorkflowProcessor(object):
         full_version = "v%s (%s)" % (version, files)
         return full_version
 
-    @staticmethod
-    def get_file_models_for_version(workflow_spec_id, version):
-        file_id_strings = re.findall('\((.*)\)', version)[0].split(".")
-        file_ids = [int(i) for i in file_id_strings]
-        files = session.query(FileDataModel)\
-            .join(FileModel) \
-            .filter(FileModel.workflow_spec_id == workflow_spec_id)\
-            .filter(FileDataModel.id.in_(file_ids)).all()
-        if len(files) != len(file_ids):
-            raise ApiError("invalid_version",
-                           "The version '%s' of workflow specification '%s' is invalid. " %
-                           (version, workflow_spec_id) +
-                           " Unable to locate the correct files to recreate it.")
-        return files
+
+
+    def update_dependencies(self, spec_data_files):
+        existing_dependencies = FileService.get_spec_data_files(
+            workflow_spec_id=self.workflow_model.workflow_spec_id,
+            workflow_id=self.workflow_model.id)
+
+        # Don't save the dependencies if they haven't changed.
+        if existing_dependencies == spec_data_files:
+            return
+
+        # Remove all existing dependencies, and replace them.
+        self.workflow_model.dependencies = []
+        for file_data in spec_data_files:
+            self.workflow_model.dependencies.append(WorkflowSpecDependencyFile(file_data_id=file_data.id))
 
     @staticmethod
-    def __get_latest_file_models(workflow_spec_id):
-        """Returns all the latest files related to a workflow specification"""
-        return session.query(FileDataModel) \
-            .join(FileModel) \
-            .filter(FileModel.workflow_spec_id == workflow_spec_id)\
-            .filter(FileDataModel.version == FileModel.latest_version)\
-            .order_by(FileModel.id)\
-            .all()
+    def run_master_spec(spec_model, study):
+        """Executes a BPMN specification for the given study, without recording any information to the database
+        Useful for running the master specification, which should not persist. """
+        spec_data_files = FileService.get_spec_data_files(spec_model.id)
+        spec = WorkflowProcessor.get_spec(spec_data_files, spec_model.id)
+        try:
+            bpmn_workflow = BpmnWorkflow(spec, script_engine=WorkflowProcessor._script_engine)
+            bpmn_workflow.data[WorkflowProcessor.STUDY_ID_KEY] = study.id
+            bpmn_workflow.data[WorkflowProcessor.VALIDATION_PROCESS_KEY] = False
+            bpmn_workflow.do_engine_steps()
+        except WorkflowException as we:
+            raise ApiError.from_task_spec("error_running_master_spec", str(we), we.sender)
+
+        if not bpmn_workflow.is_completed():
+            raise ApiError("master_spec_not_automatic",
+                           "The master spec should only contain fully automated tasks, it failed to complete.")
+
+        return bpmn_workflow.last_task.data
 
     @staticmethod
-    def get_spec(workflow_spec_id, version=None):
-        """Returns the requested version of the specification,
-        or the latest version if none is specified."""
+    def get_parser():
+        parser = MyCustomParser()
+        return parser
+
+    @staticmethod
+    def get_spec(file_data_models: List[FileDataModel], workflow_spec_id):
+        """Returns a SpiffWorkflow specification for the given workflow spec,
+        using the files provided.  The Workflow_spec_id is only used to generate
+        better error messages."""
         parser = WorkflowProcessor.get_parser()
         process_id = None
 
-        if version is None:
-            file_data_models = WorkflowProcessor.__get_latest_file_models(workflow_spec_id)
-        else:
-            file_data_models = WorkflowProcessor.get_file_models_for_version(workflow_spec_id, version)
-
         for file_data in file_data_models:
             if file_data.file_model.type == FileType.bpmn:
                 bpmn: ElementTree.Element = ElementTree.fromstring(file_data.data)
                 if file_data.file_model.primary:
-                    process_id = WorkflowProcessor.get_process_id(bpmn)
+                    process_id = FileService.get_process_id(bpmn)
                 parser.add_bpmn_xml(bpmn, filename=file_data.file_model.name)
             elif file_data.file_model.type == FileType.dmn:
                 dmn: ElementTree.Element = ElementTree.fromstring(file_data.data)
                 parser.add_dmn_xml(dmn, filename=file_data.file_model.name)
         if process_id is None:
-            raise(ApiError(code="no_primary_bpmn_error",
-                           message="There is no primary BPMN model defined for workflow %s" % workflow_spec_id))
+            raise (ApiError(code="no_primary_bpmn_error",
+                            message="There is no primary BPMN model defined for workflow %s" % workflow_spec_id))
         try:
             spec = parser.get_spec(process_id)
         except ValidationException as ve:
             raise ApiError(code="workflow_validation_error",
-                           message="Failed to parse Workflow Specification '%s' %s." % (workflow_spec_id, version) +
+                           message="Failed to parse Workflow Specification '%s'" % workflow_spec_id +
                                    "Error is %s" % str(ve),
                            file_name=ve.filename,
                            task_id=ve.id,
@@ -301,8 +305,8 @@ class WorkflowProcessor(object):
 
          Returns the new version.
          """
-        version = WorkflowProcessor.get_latest_version_string(self.workflow_spec_id)
-        spec = WorkflowProcessor.get_spec(self.workflow_spec_id)  # Force latest version by NOT specifying version
+        self.spec_data_files = FileService.get_spec_data_files(workflow_spec_id=self.workflow_spec_id)
+        spec = WorkflowProcessor.get_spec(self.spec_data_files, self.workflow_spec_id)
         # spec = WorkflowProcessor.get_spec(self.workflow_spec_id, version)
         bpmn_workflow = BpmnWorkflow(spec, script_engine=self._script_engine)
         bpmn_workflow.data = self.bpmn_workflow.data
@@ -310,14 +314,10 @@ class WorkflowProcessor(object):
             task.data = self.bpmn_workflow.last_task.data
         bpmn_workflow.do_engine_steps()
         self.bpmn_workflow = bpmn_workflow
-        return version
 
     def get_status(self):
         return self.status_of(self.bpmn_workflow)
 
-    def get_spec_version(self):
-        return self.workflow_model.spec_version
-
     def do_engine_steps(self):
         try:
             self.bpmn_workflow.do_engine_steps()
@@ -398,32 +398,18 @@ class WorkflowProcessor(object):
         return [t for t in all_tasks
                 if not self.bpmn_workflow._is_engine_task(t.task_spec) and t.state in [t.COMPLETED, t.CANCELLED]]
 
-    @staticmethod
-    def get_process_id(et_root: ElementTree.Element):
-        process_elements = []
-        for child in et_root:
-            if child.tag.endswith('process') and child.attrib.get('isExecutable', False):
-                process_elements.append(child)
-
-        if len(process_elements) == 0:
-            raise ValidationException('No executable process tag found')
-
-        # There are multiple root elements
-        if len(process_elements) > 1:
-
-            # Look for the element that has the startEvent in it
-            for e in process_elements:
-                this_element: ElementTree.Element = e
-                for child_element in list(this_element):
-                    if child_element.tag.endswith('startEvent'):
-                        return this_element.attrib['id']
-
-            raise ValidationException('No start event found in %s' % et_root.attrib['id'])
-
-        return process_elements[0].attrib['id']
-
     def get_nav_item(self, task):
         for nav_item in self.bpmn_workflow.get_nav_list():
             if nav_item['task_id'] == task.id:
                 return nav_item
 
+    def find_task_and_field_by_field_id(self, field_id):
+        """Tracks down a form field by name in the workflow spec,
+         only looks at ready tasks. Returns a tuple of the task, and form"""
+        for spiff_task in self.bpmn_workflow.get_tasks():
+            if hasattr(spiff_task.task_spec, "form"):
+                for field in spiff_task.task_spec.form.fields:
+                    if field.id == field_id:
+                        return spiff_task, field
+        raise ApiError("invalid_field",
+                       "Unable to find a task in the workflow with a lookup field called: %s" % field_id)

crc/services/workflow_service.py

@@ -17,9 +17,14 @@ from crc import db, app
 from crc.api.common import ApiError
 from crc.models.api_models import Task, MultiInstanceType
 from crc.models.file import LookupDataModel
+from crc.models.protocol_builder import ProtocolBuilderStatus
 from crc.models.stats import TaskEventModel
+from crc.models.study import StudyModel
+from crc.models.user import UserModel
+from crc.models.workflow import WorkflowModel, WorkflowStatus
 from crc.services.file_service import FileService
 from crc.services.lookup_service import LookupService
+from crc.services.study_service import StudyService
 from crc.services.workflow_processor import WorkflowProcessor, CustomBpmnScriptEngine
 
 
@@ -36,21 +41,46 @@ class WorkflowService(object):
      But for now, this contains tools for converting spiff-workflow models into our
      own API models with additional information and capabilities."""
 
-    @classmethod
-    def test_spec(cls, spec_id):
+    @staticmethod
+    def make_test_workflow(spec_id):
+        user = db.session.query(UserModel).filter_by(uid="test").first()
+        if not user:
+            db.session.add(UserModel(uid="test"))
+        study = db.session.query(StudyModel).filter_by(user_uid="test").first()
+        if not study:
+            db.session.add(StudyModel(user_uid="test", title="test"))
+            db.session.commit()
+        workflow_model = WorkflowModel(status=WorkflowStatus.not_started,
+                                       workflow_spec_id=spec_id,
+                                       last_updated=datetime.now(),
+                                       study=study)
+        return workflow_model
+
+    @staticmethod
+    def delete_test_data():
+        for study in db.session.query(StudyModel).filter(StudyModel.user_uid=="test"):
+            StudyService.delete_study(study.id)
+            db.session.commit()
+        db.session.query(UserModel).filter_by(uid="test").delete()
+
+    @staticmethod
+    def test_spec(spec_id):
         """Runs a spec through it's paces to see if it results in any errors.  Not fool-proof, but a good
         sanity check."""
-        version = WorkflowProcessor.get_latest_version_string(spec_id)
-        spec = WorkflowProcessor.get_spec(spec_id, version)
-        bpmn_workflow = BpmnWorkflow(spec, script_engine=CustomBpmnScriptEngine())
-        bpmn_workflow.data[WorkflowProcessor.STUDY_ID_KEY] = 1
-        bpmn_workflow.data[WorkflowProcessor.WORKFLOW_ID_KEY] = spec_id
-        bpmn_workflow.data[WorkflowProcessor.VALIDATION_PROCESS_KEY] = True
 
-        while not bpmn_workflow.is_completed():
+        workflow_model = WorkflowService.make_test_workflow(spec_id)
+
+        try:
+            processor = WorkflowProcessor(workflow_model, validate_only=True)
+        except WorkflowException as we:
+            WorkflowService.delete_test_data()
+            raise ApiError.from_task_spec("workflow_execution_exception", str(we),
+                                          we.sender)
+
+        while not processor.bpmn_workflow.is_completed():
             try:
-                bpmn_workflow.do_engine_steps()
-                tasks = bpmn_workflow.get_tasks(SpiffTask.READY)
+                processor.bpmn_workflow.do_engine_steps()
+                tasks = processor.bpmn_workflow.get_tasks(SpiffTask.READY)
                 for task in tasks:
                     task_api = WorkflowService.spiff_task_to_api_task(
                         task,
@@ -58,8 +88,10 @@ class WorkflowService(object):
                     WorkflowService.populate_form_with_random_data(task, task_api)
                     task.complete()
             except WorkflowException as we:
+                WorkflowService.delete_test_data()
                 raise ApiError.from_task_spec("workflow_execution_exception", str(we),
                                               we.sender)
+        WorkflowService.delete_test_data()
 
     @staticmethod
     def populate_form_with_random_data(task, task_api):
@@ -82,7 +114,7 @@ class WorkflowService(object):
                                                              " with no options" % field.id,
                                              task)
             elif field.type == "autocomplete":
-                lookup_model = LookupService.get_lookup_table(task, field)
+                lookup_model = LookupService.get_lookup_model(task, field)
                 if field.has_property(Task.PROP_LDAP_LOOKUP):
                     form_data[field.id] = {
                         "label": "dhf8r",
@@ -248,12 +280,12 @@ class WorkflowService(object):
 
     @staticmethod
     def process_options(spiff_task, field):
-        lookup_model = LookupService.get_lookup_table(spiff_task, field)
 
         # If this is an auto-complete field, do not populate options, a lookup will happen later.
         if field.type == Task.FIELD_TYPE_AUTO_COMPLETE:
             pass
-        else:
+        elif field.has_property(Task.PROP_OPTIONS_FILE):
+            lookup_model = LookupService.get_lookup_model(spiff_task, field)
             data = db.session.query(LookupDataModel).filter(LookupDataModel.lookup_file_model == lookup_model).all()
             if not hasattr(field, 'options'):
                 field.options = []
@@ -269,7 +301,7 @@ class WorkflowService(object):
             user_uid=g.user.uid,
             workflow_id=workflow_model.id,
             workflow_spec_id=workflow_model.workflow_spec_id,
-            spec_version=workflow_model.spec_version,
+            spec_version=processor.get_version_string(),
             action=action,
             task_id=task.id,
             task_name=task.name,
@@ -284,3 +316,4 @@ class WorkflowService(object):
         )
         db.session.add(task_event)
         db.session.commit()
+

migrations/versions/23c62c933848_.py

@@ -0,0 +1,34 @@
+"""empty message
+
+Revision ID: 23c62c933848
+Revises: 9b43e725f39c
+Create Date: 2020-05-28 10:30:49.409760
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '23c62c933848'
+down_revision = '9b43e725f39c'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint('file_study_id_fkey', 'file', type_='foreignkey')
+    op.drop_column('file', 'task_id')
+    op.drop_column('file', 'study_id')
+    op.drop_column('file', 'form_field_key')
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('file', sa.Column('form_field_key', sa.VARCHAR(), autoincrement=False, nullable=True))
+    op.add_column('file', sa.Column('study_id', sa.INTEGER(), autoincrement=False, nullable=True))
+    op.add_column('file', sa.Column('task_id', sa.VARCHAR(), autoincrement=False, nullable=True))
+    op.create_foreign_key('file_study_id_fkey', 'file', 'study', ['study_id'], ['id'])
+    # ### end Alembic commands ###

migrations/versions/5064b72284b7_.py

@@ -0,0 +1,36 @@
+"""empty message
+
+Revision ID: 5064b72284b7
+Revises: bec71f7dc652
+Create Date: 2020-05-28 23:54:45.623361
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '5064b72284b7'
+down_revision = 'bec71f7dc652'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('lookup_file', sa.Column('field_id', sa.String(), nullable=True))
+    op.add_column('lookup_file', sa.Column('is_ldap', sa.Boolean(), nullable=True))
+    op.add_column('lookup_file', sa.Column('workflow_spec_id', sa.String(), nullable=True))
+    op.drop_column('lookup_file', 'value_column')
+    op.drop_column('lookup_file', 'label_column')
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('lookup_file', sa.Column('label_column', sa.VARCHAR(), autoincrement=False, nullable=True))
+    op.add_column('lookup_file', sa.Column('value_column', sa.VARCHAR(), autoincrement=False, nullable=True))
+    op.drop_column('lookup_file', 'workflow_spec_id')
+    op.drop_column('lookup_file', 'is_ldap')
+    op.drop_column('lookup_file', 'field_id')
+    # ### end Alembic commands ###

migrations/versions/bec71f7dc652_.py

@@ -0,0 +1,63 @@
+"""empty message
+
+Revision ID: bec71f7dc652
+Revises: 23c62c933848
+Create Date: 2020-05-28 20:08:45.891406
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = 'bec71f7dc652'
+down_revision = '23c62c933848'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+
+    op.create_table('workflow_spec_dependency_file',
+    sa.Column('file_data_id', sa.Integer(), nullable=False),
+    sa.Column('workflow_id', sa.Integer(), nullable=False),
+    sa.ForeignKeyConstraint(['file_data_id'], ['file_data.id'], ),
+    sa.ForeignKeyConstraint(['workflow_id'], ['workflow.id'], ),
+    sa.PrimaryKeyConstraint('file_data_id', 'workflow_id')
+    )
+    op.drop_column('approval', 'workflow_hash')
+    op.execute(
+        """
+        delete from approval_file;
+        delete from approval;
+        """
+    )
+    op.add_column('approval_file', sa.Column('file_data_id', sa.Integer(), nullable=False))
+    op.drop_constraint('approval_file_file_id_fkey', 'approval_file', type_='foreignkey')
+    op.create_foreign_key(None, 'approval_file', 'file_data', ['file_data_id'], ['id'])
+    op.drop_column('approval_file', 'id')
+    op.drop_column('approval_file', 'file_version')
+    op.drop_column('approval_file', 'file_id')
+    op.drop_column('file', 'latest_version')
+    op.add_column('file_data', sa.Column('date_created', sa.DateTime(timezone=True), nullable=True))
+    op.drop_column('file_data', 'last_updated')
+    op.drop_column('workflow', 'spec_version')
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('workflow', sa.Column('spec_version', sa.VARCHAR(), autoincrement=False, nullable=True))
+    op.add_column('file_data', sa.Column('last_updated', postgresql.TIMESTAMP(timezone=True), autoincrement=False, nullable=True))
+    op.drop_column('file_data', 'date_created')
+    op.add_column('file', sa.Column('latest_version', sa.INTEGER(), autoincrement=False, nullable=True))
+    op.add_column('approval_file', sa.Column('file_id', sa.INTEGER(), autoincrement=False, nullable=False))
+    op.add_column('approval_file', sa.Column('file_version', sa.INTEGER(), autoincrement=False, nullable=False))
+    op.add_column('approval_file', sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False))
+    op.drop_constraint(None, 'approval_file', type_='foreignkey')
+    op.create_foreign_key('approval_file_file_id_fkey', 'approval_file', 'file', ['file_id'], ['id'])
+    op.drop_column('approval_file', 'file_data_id')
+    op.add_column('approval', sa.Column('workflow_hash', sa.VARCHAR(), autoincrement=False, nullable=True))
+    op.drop_table('workflow_spec_dependency_file')
+    # ### end Alembic commands ###

tests/data/docx/docx.bpmn

@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<bpmn:definitions xmlns:bpmn="http://www.omg.org/spec/BPMN/20100524/MODEL" xmlns:bpmndi="http://www.omg.org/spec/BPMN/20100524/DI" xmlns:dc="http://www.omg.org/spec/DD/20100524/DC" xmlns:camunda="http://camunda.org/schema/1.0/bpmn" xmlns:di="http://www.omg.org/spec/DD/20100524/DI" id="Definitions_96a17d9" targetNamespace="http://bpmn.io/schema/bpmn" exporter="Camunda Modeler" exporterVersion="3.4.1">
+<bpmn:definitions xmlns:bpmn="http://www.omg.org/spec/BPMN/20100524/MODEL" xmlns:bpmndi="http://www.omg.org/spec/BPMN/20100524/DI" xmlns:dc="http://www.omg.org/spec/DD/20100524/DC" xmlns:camunda="http://camunda.org/schema/1.0/bpmn" xmlns:di="http://www.omg.org/spec/DD/20100524/DI" id="Definitions_96a17d9" targetNamespace="http://bpmn.io/schema/bpmn" exporter="Camunda Modeler" exporterVersion="3.7.3">
   <bpmn:process id="Process_93a29b3" isExecutable="true">
     <bpmn:startEvent id="StartEvent_1">
       <bpmn:outgoing>SequenceFlow_0637d8i</bpmn:outgoing>
@@ -27,7 +27,7 @@
       </bpmn:extensionElements>
       <bpmn:incoming>SequenceFlow_1i7hk1a</bpmn:incoming>
       <bpmn:outgoing>SequenceFlow_11c35oq</bpmn:outgoing>
-      <bpmn:script>CompleteTemplate Letter.docx AncillaryDocument.CoCApplication</bpmn:script>
+      <bpmn:script>CompleteTemplate Letter.docx AD_CoCApp</bpmn:script>
     </bpmn:scriptTask>
     <bpmn:endEvent id="EndEvent_0evb22x">
       <bpmn:incoming>SequenceFlow_11c35oq</bpmn:incoming>
@@ -36,30 +36,30 @@
   </bpmn:process>
   <bpmndi:BPMNDiagram id="BPMNDiagram_1">
     <bpmndi:BPMNPlane id="BPMNPlane_1" bpmnElement="Process_93a29b3">
-      <bpmndi:BPMNShape id="_BPMNShape_StartEvent_2" bpmnElement="StartEvent_1">
-        <dc:Bounds x="179" y="99" width="36" height="36" />
-      </bpmndi:BPMNShape>
-      <bpmndi:BPMNEdge id="SequenceFlow_0637d8i_di" bpmnElement="SequenceFlow_0637d8i">
-        <di:waypoint x="215" y="117" />
-        <di:waypoint x="265" y="117" />
+      <bpmndi:BPMNEdge id="SequenceFlow_11c35oq_di" bpmnElement="SequenceFlow_11c35oq">
+        <di:waypoint x="565" y="117" />
+        <di:waypoint x="665" y="117" />
       </bpmndi:BPMNEdge>
-      <bpmndi:BPMNShape id="UserTask_02o51o8_di" bpmnElement="task_gather_information">
-        <dc:Bounds x="265" y="77" width="100" height="80" />
-      </bpmndi:BPMNShape>
       <bpmndi:BPMNEdge id="SequenceFlow_1i7hk1a_di" bpmnElement="SequenceFlow_1i7hk1a">
         <di:waypoint x="365" y="117" />
         <di:waypoint x="465" y="117" />
       </bpmndi:BPMNEdge>
+      <bpmndi:BPMNEdge id="SequenceFlow_0637d8i_di" bpmnElement="SequenceFlow_0637d8i">
+        <di:waypoint x="215" y="117" />
+        <di:waypoint x="265" y="117" />
+      </bpmndi:BPMNEdge>
+      <bpmndi:BPMNShape id="_BPMNShape_StartEvent_2" bpmnElement="StartEvent_1">
+        <dc:Bounds x="179" y="99" width="36" height="36" />
+      </bpmndi:BPMNShape>
+      <bpmndi:BPMNShape id="UserTask_02o51o8_di" bpmnElement="task_gather_information">
+        <dc:Bounds x="265" y="77" width="100" height="80" />
+      </bpmndi:BPMNShape>
       <bpmndi:BPMNShape id="ScriptTask_0xjh8x4_di" bpmnElement="task_generate_document">
         <dc:Bounds x="465" y="77" width="100" height="80" />
       </bpmndi:BPMNShape>
       <bpmndi:BPMNShape id="EndEvent_0evb22x_di" bpmnElement="EndEvent_0evb22x">
         <dc:Bounds x="665" y="99" width="36" height="36" />
       </bpmndi:BPMNShape>
-      <bpmndi:BPMNEdge id="SequenceFlow_11c35oq_di" bpmnElement="SequenceFlow_11c35oq">
-        <di:waypoint x="565" y="117" />
-        <di:waypoint x="665" y="117" />
-      </bpmndi:BPMNEdge>
     </bpmndi:BPMNPlane>
   </bpmndi:BPMNDiagram>
 </bpmn:definitions>

tests/test_approvals_service.py

@@ -9,7 +9,12 @@ from crc.services.workflow_processor import WorkflowProcessor
 class TestApprovalsService(BaseTest):
 
     def test_create_approval_record(self):
+        self.create_reference_document()
         workflow = self.create_workflow("empty_workflow")
+        FileService.add_workflow_file(workflow_id=workflow.id,
+                                      name="anything.png", content_type="text",
+                                      binary_data=b'5678', irb_doc_code="UVACompl_PRCAppr" )
+
         ApprovalService.add_approval(study_id=workflow.study_id, workflow_id=workflow.id, approver_uid="dhf8r")
         self.assertEquals(1, db.session.query(ApprovalModel).count())
         model = db.session.query(ApprovalModel).first()
@@ -17,10 +22,14 @@ class TestApprovalsService(BaseTest):
         self.assertEquals(workflow.id, model.workflow_id)
         self.assertEquals("dhf8r", model.approver_uid)
         self.assertEquals(1, model.version)
-        self.assertIsNotNone(model.workflow_hash)
 
     def test_new_requests_dont_add_if_approval_exists_for_current_workflow(self):
+        self.create_reference_document()
         workflow = self.create_workflow("empty_workflow")
+        FileService.add_workflow_file(workflow_id=workflow.id,
+                                      name="anything.png", content_type="text",
+                                      binary_data=b'5678', irb_doc_code="UVACompl_PRCAppr" )
+
         ApprovalService.add_approval(study_id=workflow.study_id, workflow_id=workflow.id, approver_uid="dhf8r")
         ApprovalService.add_approval(study_id=workflow.study_id, workflow_id=workflow.id, approver_uid="dhf8r")
         self.assertEquals(1, db.session.query(ApprovalModel).count())
@@ -31,17 +40,15 @@ class TestApprovalsService(BaseTest):
         self.load_example_data()
         self.create_reference_document()
         workflow = self.create_workflow('empty_workflow')
-        processor = WorkflowProcessor(workflow)
-        task = processor.next_task()
+        FileService.add_workflow_file(workflow_id=workflow.id,
+                                      name="anything.png", content_type="text",
+                                      binary_data=b'5678', irb_doc_code="AD_CoCAppr")
 
         ApprovalService.add_approval(study_id=workflow.study_id, workflow_id=workflow.id, approver_uid="dhf8r")
 
-        irb_code_1 = "UVACompl_PRCAppr"  # The first file referenced in pb required docs.
-        FileService.add_task_file(study_id=workflow.study_id, workflow_id=workflow.id,
-                                  workflow_spec_id=workflow.workflow_spec_id,
-                                  task_id=task.id,
-                                  name="anything.png", content_type="text",
-                                  binary_data=b'5678', irb_doc_code=irb_code_1)
+        FileService.add_workflow_file(workflow_id=workflow.id,
+                                      name="anything.png", content_type="text",
+                                      binary_data=b'5678', irb_doc_code="UVACompl_PRCAppr")
 
         ApprovalService.add_approval(study_id=workflow.study_id, workflow_id=workflow.id, approver_uid="dhf8r")
         self.assertEquals(2, db.session.query(ApprovalModel).count())
@@ -50,45 +57,3 @@ class TestApprovalsService(BaseTest):
         self.assertEquals(2, models[1].version)
 
 
-    def test_generate_workflow_hash_and_version(self):
-        self.load_example_data()
-        self.create_reference_document()
-        workflow = self.create_workflow('empty_workflow')
-        processor = WorkflowProcessor(workflow)
-        task = processor.next_task()
-        irb_code_1 = "UVACompl_PRCAppr"  # The first file referenced in pb required docs.
-        irb_code_2 = "NonUVAIRB_AssuranceForm"  # The second file in above.
-        # Add a task file to the workflow.
-        FileService.add_task_file(study_id=workflow.study_id, workflow_id=workflow.id,
-                                  workflow_spec_id=workflow.workflow_spec_id,
-                                  task_id=task.id,
-                                  name="anything.png", content_type="text",
-                                  binary_data=b'5678', irb_doc_code=irb_code_1)
-        # Add a two form field files with the same irb_code, but
-        FileService.add_form_field_file(study_id=workflow.study_id, workflow_id=workflow.id,
-                                  task_id=task.id,
-                                  form_field_key=irb_code_2,
-                                  name="anything.png", content_type="text",
-                                  binary_data=b'1234')
-        FileService.add_form_field_file(study_id=workflow.study_id, workflow_id=workflow.id,
-                                  form_field_key=irb_code_2,
-                                  task_id=task.id,
-                                  name="another_anything.png", content_type="text",
-                                  binary_data=b'5678')
-
-
-        # Workflow hash should look be id[1]-id[1]-id[1]
-        # Sould be three files, each with a version of 1.
-        # where id is the file id, which we don't know, thus the regex.
-        latest_files = FileService.get_workflow_files(workflow.id)
-        self.assertRegexpMatches(ApprovalService._generate_workflow_hash(latest_files), "\d+\[1\]-\d+\[1\]-\d+\[1\]")
-
-        # Replace last file
-        # should now be id[1]-id[1]-id[2]
-        FileService.add_form_field_file(study_id=workflow.study_id, workflow_id=workflow.id,
-                                  form_field_key=irb_code_2,
-                                  task_id=task.id,
-                                  name="another_anything.png", content_type="text",
-                                  binary_data=b'9999')
-        self.assertRegexpMatches(ApprovalService._generate_workflow_hash(latest_files), "\d+\[1\]-\d+\[1\]-\d+\[2\]")
-

tests/test_file_service.py

@@ -13,21 +13,21 @@ class TestFileService(BaseTest):
         processor = WorkflowProcessor(workflow)
         task = processor.next_task()
         irb_code = "UVACompl_PRCAppr"  # The first file referenced in pb required docs.
-        FileService.add_task_file(study_id=workflow.study_id, workflow_id=workflow.id,
-                                  workflow_spec_id=workflow.workflow_spec_id,
-                                  task_id=task.id,
-                                  name="anything.png", content_type="text",
-                                  binary_data=b'1234', irb_doc_code=irb_code)
+        FileService.add_workflow_file(workflow_id=workflow.id,
+                                      name="anything.png", content_type="text",
+                                      binary_data=b'1234', irb_doc_code=irb_code)
         # Add the file again with different data
-        FileService.add_task_file(study_id=workflow.study_id, workflow_id=workflow.id,
-                                  workflow_spec_id=workflow.workflow_spec_id,
-                                  task_id=task.id,
-                                  name="anything.png", content_type="text",
-                                  binary_data=b'5678', irb_doc_code=irb_code)
+        FileService.add_workflow_file(workflow_id=workflow.id,
+                                      name="anything.png", content_type="text",
+                                      binary_data=b'5678', irb_doc_code=irb_code)
 
         file_models = FileService.get_workflow_files(workflow_id=workflow.id)
         self.assertEquals(1, len(file_models))
-        self.assertEquals(2, file_models[0].latest_version)
+
+        file_data = FileService.get_workflow_data_files(workflow_id=workflow.id)
+        self.assertEquals(1, len(file_data))
+        self.assertEquals(2, file_data[0].version)
+
 
     def test_add_file_from_form_increments_version_and_replaces_on_subsequent_add_with_same_name(self):
         self.load_example_data()
@@ -36,21 +36,22 @@ class TestFileService(BaseTest):
         processor = WorkflowProcessor(workflow)
         task = processor.next_task()
         irb_code = "UVACompl_PRCAppr"  # The first file referenced in pb required docs.
-        FileService.add_form_field_file(study_id=workflow.study_id, workflow_id=workflow.id,
-                                  task_id=task.id,
-                                  form_field_key=irb_code,
-                                  name="anything.png", content_type="text",
-                                  binary_data=b'1234')
+        FileService.add_workflow_file(workflow_id=workflow.id,
+                                      irb_doc_code=irb_code,
+                                      name="anything.png", content_type="text",
+                                      binary_data=b'1234')
         # Add the file again with different data
-        FileService.add_form_field_file(study_id=workflow.study_id, workflow_id=workflow.id,
-                                  form_field_key=irb_code,
-                                  task_id=task.id,
-                                  name="anything.png", content_type="text",
-                                  binary_data=b'5678')
+        FileService.add_workflow_file(workflow_id=workflow.id,
+                                      irb_doc_code=irb_code,
+                                      name="anything.png", content_type="text",
+                                      binary_data=b'5678')
 
         file_models = FileService.get_workflow_files(workflow_id=workflow.id)
         self.assertEquals(1, len(file_models))
-        self.assertEquals(2, file_models[0].latest_version)
+
+        file_data = FileService.get_workflow_data_files(workflow_id=workflow.id)
+        self.assertEquals(1, len(file_data))
+        self.assertEquals(2, file_data[0].version)
 
     def test_add_file_from_form_allows_multiple_files_with_different_names(self):
         self.load_example_data()
@@ -59,18 +60,14 @@ class TestFileService(BaseTest):
         processor = WorkflowProcessor(workflow)
         task = processor.next_task()
         irb_code = "UVACompl_PRCAppr"  # The first file referenced in pb required docs.
-        FileService.add_form_field_file(study_id=workflow.study_id, workflow_id=workflow.id,
-                                  task_id=task.id,
-                                  form_field_key=irb_code,
-                                  name="anything.png", content_type="text",
-                                  binary_data=b'1234')
+        FileService.add_workflow_file(workflow_id=workflow.id,
+                                      irb_doc_code=irb_code,
+                                      name="anything.png", content_type="text",
+                                      binary_data=b'1234')
         # Add the file again with different data
-        FileService.add_form_field_file(study_id=workflow.study_id, workflow_id=workflow.id,
-                                  form_field_key=irb_code,
-                                  task_id=task.id,
-                                  name="a_different_thing.png", content_type="text",
-                                  binary_data=b'5678')
+        FileService.add_workflow_file(workflow_id=workflow.id,
+                                      irb_doc_code=irb_code,
+                                      name="a_different_thing.png", content_type="text",
+                                      binary_data=b'5678')
         file_models = FileService.get_workflow_files(workflow_id=workflow.id)
         self.assertEquals(2, len(file_models))
-        self.assertEquals(1, file_models[0].latest_version)
-        self.assertEquals(1, file_models[1].latest_version)

tests/test_files_api.py

@@ -1,15 +1,14 @@
 import io
 import json
-from datetime import datetime
-from unittest.mock import patch
+
+from tests.base_test import BaseTest
 
 from crc import session
-from crc.models.file import FileModel, FileType, FileModelSchema, FileDataModel
+from crc.models.file import FileModel, FileType, FileSchema, FileModelSchema
 from crc.models.workflow import WorkflowSpecModel
 from crc.services.file_service import FileService
 from crc.services.workflow_processor import WorkflowProcessor
 from example_data import ExampleDataLoader
-from tests.base_test import BaseTest
 
 
 class TestFilesApi(BaseTest):
@@ -166,16 +165,16 @@ class TestFilesApi(BaseTest):
                           content_type='multipart/form-data', headers=self.logged_in_headers())
         self.assert_success(rv)
         self.assertIsNotNone(rv.get_data())
-        json_data = json.loads(rv.get_data(as_text=True))
-        file = FileModelSchema().load(json_data, session=session)
-        self.assertEqual(2, file.latest_version)
-        self.assertEqual(FileType.bpmn, file.type)
-        self.assertEqual("application/octet-stream", file.content_type)
+        file_json = json.loads(rv.get_data(as_text=True))
+        self.assertEqual(2, file_json['latest_version'])
+        self.assertEqual(FileType.bpmn.value, file_json['type'])
+        self.assertEqual("application/octet-stream", file_json['content_type'])
         self.assertEqual(spec.id, file.workflow_spec_id)
 
         # Assure it is updated in the database and properly persisted.
         file_model = session.query(FileModel).filter(FileModel.id == file.id).first()
-        self.assertEqual(2, file_model.latest_version)
+        file_data = FileService.get_file_data(file_model.id)
+        self.assertEqual(2, file_data.version)
 
         rv = self.app.get('/v1.0/file/%i/data' % file.id, headers=self.logged_in_headers())
         self.assert_success(rv)
@@ -192,15 +191,13 @@ class TestFilesApi(BaseTest):
                            content_type='multipart/form-data', headers=self.logged_in_headers())
         self.assertIsNotNone(rv.get_data())
         json_data = json.loads(rv.get_data(as_text=True))
-        file = FileModelSchema().load(json_data, session=session)
-        self.assertEqual(1, file.latest_version)
+        self.assertEqual(1, json_data['latest_version'])
         data['file'] = io.BytesIO(self.minimal_bpmn("abcdef")), 'my_new_file.bpmn'
-        rv = self.app.put('/v1.0/file/%i/data' % file.id, data=data, follow_redirects=True,
+        rv = self.app.put('/v1.0/file/%i/data' % json_data['id'], data=data, follow_redirects=True,
                           content_type='multipart/form-data', headers=self.logged_in_headers())
         self.assertIsNotNone(rv.get_data())
         json_data = json.loads(rv.get_data(as_text=True))
-        file = FileModelSchema().load(json_data, session=session)
-        self.assertEqual(1, file.latest_version)
+        self.assertEqual(1, json_data['latest_version'])
 
     def test_get_file(self):
         self.load_example_data()

tests/test_lookup_service.py

@@ -1,90 +1,119 @@
+import os
 
 from tests.base_test import BaseTest
 
-from crc import session
-from crc.models.file import FileDataModel, FileModel, LookupFileModel, LookupDataModel
 from crc.services.file_service import FileService
+from crc.api.common import ApiError
+from crc import session, app
+from crc.models.file import FileDataModel, FileModel, LookupFileModel, LookupDataModel, CONTENT_TYPES
 from crc.services.lookup_service import LookupService
+from crc.services.workflow_processor import WorkflowProcessor
 
 
 class TestLookupService(BaseTest):
 
-
-    def test_create_lookup_file_multiple_times_does_not_update_database(self):
-        spec = BaseTest.load_test_spec('enum_options_from_file')
+    def test_lookup_returns_good_error_on_bad_field(self):
+        spec = BaseTest.load_test_spec('enum_options_with_search')
+        workflow = self.create_workflow('enum_options_with_search')
         file_model = session.query(FileModel).filter(FileModel.name == "customer_list.xls").first()
         file_data_model = session.query(FileDataModel).filter(FileDataModel.file_model == file_model).first()
+        with self.assertRaises(ApiError):
+            LookupService.lookup(workflow, "not_the_right_field", "sam", limit=10)
 
-        LookupService.get_lookup_table_from_data_model(file_data_model, "CUSTOMER_NUMBER", "CUSTOMER_NAME")
-        LookupService.get_lookup_table_from_data_model(file_data_model, "CUSTOMER_NUMBER", "CUSTOMER_NAME")
-        LookupService.get_lookup_table_from_data_model(file_data_model, "CUSTOMER_NUMBER", "CUSTOMER_NAME")
+    def test_lookup_table_is_not_created_more_than_once(self):
+        spec = BaseTest.load_test_spec('enum_options_with_search')
+        workflow = self.create_workflow('enum_options_with_search')
+        LookupService.lookup(workflow, "sponsor", "sam", limit=10)
+        LookupService.lookup(workflow, "sponsor", "something", limit=10)
+        LookupService.lookup(workflow, "sponsor", "blah", limit=10)
         lookup_records = session.query(LookupFileModel).all()
         self.assertIsNotNone(lookup_records)
         self.assertEqual(1, len(lookup_records))
         lookup_record = lookup_records[0]
         lookup_data = session.query(LookupDataModel).filter(LookupDataModel.lookup_file_model == lookup_record).all()
         self.assertEquals(28, len(lookup_data))
-        # Using the same table with different lookup lable or value, does create additional records.
-        LookupService.get_lookup_table_from_data_model(file_data_model, "CUSTOMER_NAME", "CUSTOMER_NUMBER")
+
+    def test_updates_to_file_cause_lookup_rebuild(self):
+        spec = BaseTest.load_test_spec('enum_options_with_search')
+        workflow = self.create_workflow('enum_options_with_search')
+        file_model = session.query(FileModel).filter(FileModel.name == "sponsors.xls").first()
+        LookupService.lookup(workflow, "sponsor", "sam", limit=10)
         lookup_records = session.query(LookupFileModel).all()
         self.assertIsNotNone(lookup_records)
-        self.assertEqual(2, len(lookup_records))
+        self.assertEqual(1, len(lookup_records))
+        lookup_record = lookup_records[0]
+        lookup_data = session.query(LookupDataModel).filter(LookupDataModel.lookup_file_model == lookup_record).all()
+        self.assertEquals(28, len(lookup_data))
+
+        # Update the workflow specification file.
+        file_path = os.path.join(app.root_path, '..', 'tests', 'data',
+                                 'enum_options_with_search', 'sponsors_modified.xls')
+        file = open(file_path, 'rb')
+        FileService.update_file(file_model, file.read(), CONTENT_TYPES['xls'])
+        file.close()
+
+        # restart the workflow, so it can pick up the changes.
+        WorkflowProcessor(workflow, soft_reset=True)
+
+        LookupService.lookup(workflow, "sponsor", "sam", limit=10)
+        lookup_records = session.query(LookupFileModel).all()
+        lookup_record = lookup_records[0]
+        lookup_data = session.query(LookupDataModel).filter(LookupDataModel.lookup_file_model == lookup_record).all()
+        self.assertEquals(4, len(lookup_data))
+
+
 
     def test_some_full_text_queries(self):
         spec = BaseTest.load_test_spec('enum_options_from_file')
-        file_model = session.query(FileModel).filter(FileModel.name == "customer_list.xls").first()
-        file_data_model = session.query(FileDataModel).filter(FileDataModel.file_model == file_model).first()
-        lookup_table = LookupService.get_lookup_table_from_data_model(file_data_model, "CUSTOMER_NUMBER", "CUSTOMER_NAME")
+        workflow = self.create_workflow('enum_options_from_file')
+        processor = WorkflowProcessor(workflow)
+        processor.do_engine_steps()
 
-        results = LookupService._run_lookup_query(lookup_table, "medicines", limit=10)
+        results = LookupService.lookup(workflow, "AllTheNames", "", limit=10)
+        self.assertEquals(10, len(results), "Blank queries return everything, to the limit")
+
+        results = LookupService.lookup(workflow, "AllTheNames", "medicines", limit=10)
         self.assertEquals(1, len(results), "words in the middle of label are detected.")
         self.assertEquals("The Medicines Company", results[0].label)
 
-        results = LookupService._run_lookup_query(lookup_table, "", limit=10)
-        self.assertEquals(10, len(results), "Blank queries return everything, to the limit")
-
-        results = LookupService._run_lookup_query(lookup_table, "UVA", limit=10)
+        results = LookupService.lookup(workflow, "AllTheNames", "UVA", limit=10)
         self.assertEquals(1, len(results), "Beginning of label is found.")
         self.assertEquals("UVA - INTERNAL - GM USE ONLY", results[0].label)
 
-        results = LookupService._run_lookup_query(lookup_table, "uva", limit=10)
+        results = LookupService.lookup(workflow, "AllTheNames", "uva", limit=10)
         self.assertEquals(1, len(results), "case does not matter.")
         self.assertEquals("UVA - INTERNAL - GM USE ONLY", results[0].label)
 
-        results = LookupService._run_lookup_query(lookup_table, "medici", limit=10)
+        results = LookupService.lookup(workflow, "AllTheNames", "medici", limit=10)
         self.assertEquals(1, len(results), "partial words are picked up.")
         self.assertEquals("The Medicines Company", results[0].label)
 
-        results = LookupService._run_lookup_query(lookup_table, "Genetics Savings", limit=10)
+        results = LookupService.lookup(workflow, "AllTheNames", "Genetics Savings", limit=10)
         self.assertEquals(1, len(results), "multiple terms are picked up..")
         self.assertEquals("Genetics Savings & Clone, Inc.", results[0].label)
 
-        results = LookupService._run_lookup_query(lookup_table, "Genetics Sav", limit=10)
+        results = LookupService.lookup(workflow, "AllTheNames", "Genetics Sav", limit=10)
         self.assertEquals(1, len(results), "prefix queries still work with partial terms")
         self.assertEquals("Genetics Savings & Clone, Inc.", results[0].label)
 
-        results = LookupService._run_lookup_query(lookup_table, "Gen Sav", limit=10)
+        results = LookupService.lookup(workflow, "AllTheNames", "Gen Sav", limit=10)
         self.assertEquals(1, len(results), "prefix queries still work with ALL the partial terms")
         self.assertEquals("Genetics Savings & Clone, Inc.", results[0].label)
 
-        results = LookupService._run_lookup_query(lookup_table, "Inc", limit=10)
+        results = LookupService.lookup(workflow, "AllTheNames", "Inc", limit=10)
         self.assertEquals(7, len(results), "short terms get multiple correct results.")
         self.assertEquals("Genetics Savings & Clone, Inc.", results[0].label)
 
-        results = LookupService._run_lookup_query(lookup_table, "reaction design", limit=10)
+        results = LookupService.lookup(workflow, "AllTheNames", "reaction design", limit=10)
         self.assertEquals(5, len(results), "all results come back for two terms.")
         self.assertEquals("Reaction Design", results[0].label, "Exact matches come first.")
 
-    def test_prefer_exact_match(self):
-        spec = BaseTest.load_test_spec('enum_options_from_file')
-        file_model = session.query(FileModel).filter(FileModel.name == "customer_list.xls").first()
-        file_data_model = session.query(FileDataModel).filter(FileDataModel.file_model == file_model).first()
-
-        lookup_table = LookupService.get_lookup_table_from_data_model(file_data_model, "CUSTOMER_NUMBER",
-                                                                      "CUSTOMER_NAME")
-        results = LookupService._run_lookup_query(lookup_table, "1 Something", limit=10)
+        results = LookupService.lookup(workflow, "AllTheNames", "1 Something", limit=10)
         self.assertEquals("1 Something", results[0].label, "Exact matches are prefered")
 
+        results = LookupService.lookup(workflow, "AllTheNames", "1 (!-Something", limit=10)
+        self.assertEquals("1 Something", results[0].label, "special characters don't flake out")
+
 
 # 1018	10000 Something	Industry
 # 1019	1000 Something	Industry

tests/test_request_approval_script.py

@@ -1,3 +1,4 @@
+from crc.services.file_service import FileService
 from tests.base_test import BaseTest
 
 from crc.scripts.request_approval import RequestApproval
@@ -17,7 +18,10 @@ class TestRequestApprovalScript(BaseTest):
         processor = WorkflowProcessor(workflow)
         task = processor.next_task()
         task.data = {"study": {"approval1": "dhf8r", 'approval2':'lb3dp'}}
-
+        FileService.add_workflow_file(workflow_id=workflow.id,
+                                      irb_doc_code="UVACompl_PRCAppr",
+                                      name="anything.png", content_type="text",
+                                      binary_data=b'1234')
         script = RequestApproval()
         script.do_task(task, workflow.study_id, workflow.id, "study.approval1", "study.approval2")
         self.assertEquals(2, db.session.query(ApprovalModel).count())

tests/test_study_service.py

@@ -2,6 +2,8 @@ import json
 from datetime import datetime
 from unittest.mock import patch
 
+from tests.base_test import BaseTest
+
 from crc import db, app
 from crc.models.protocol_builder import ProtocolBuilderStatus
 from crc.models.study import StudyModel
@@ -12,7 +14,6 @@ from crc.services.file_service import FileService
 from crc.services.study_service import StudyService
 from crc.services.workflow_processor import WorkflowProcessor
 from example_data import ExampleDataLoader
-from tests.base_test import BaseTest
 
 
 class TestStudyService(BaseTest):
@@ -73,7 +74,6 @@ class TestStudyService(BaseTest):
 
         # workflow should not be started, and it should have 0 completed tasks, and 0 total tasks.
         self.assertEqual(WorkflowStatus.not_started, workflow.status)
-        self.assertEqual(None, workflow.spec_version)
         self.assertEqual(0, workflow.total_tasks)
         self.assertEqual(0, workflow.completed_tasks)
 
@@ -143,11 +143,9 @@ class TestStudyService(BaseTest):
         # Add a document to the study with the correct code.
         workflow = self.create_workflow('docx')
         irb_code = "UVACompl_PRCAppr"  # The first file referenced in pb required docs.
-        FileService.add_task_file(study_id=workflow.study_id, workflow_id=workflow.id,
-                                  workflow_spec_id=workflow.workflow_spec_id,
-                                  task_id="fakingthisout",
-                                  name="anything.png", content_type="text",
-                                  binary_data=b'1234', irb_doc_code=irb_code)
+        FileService.add_workflow_file(workflow_id=workflow.id,
+                                      name="anything.png", content_type="text",
+                                      binary_data=b'1234', irb_doc_code=irb_code)
 
         docs = StudyService().get_documents_status(workflow.study_id)
         self.assertIsNotNone(docs)
@@ -156,13 +154,31 @@ class TestStudyService(BaseTest):
         self.assertIsNotNone(docs["UVACompl_PRCAppr"]['files'][0])
         self.assertIsNotNone(docs["UVACompl_PRCAppr"]['files'][0]['file_id'])
         self.assertEquals(workflow.id, docs["UVACompl_PRCAppr"]['files'][0]['workflow_id'])
-        self.assertEquals(workflow.workflow_spec_id, docs["UVACompl_PRCAppr"]['files'][0]['workflow_spec_id'])
 
-    # 'file_id': 123,
-    # 'task_id': 'abcdef14236890',
-    # 'workflow_id': 456,
-    # 'workflow_spec_id': 'irb_api_details',
-    # 'status': 'complete',
+    def test_get_all_studies(self):
+        user = self.create_user_with_study_and_workflow()
+
+        # Add a document to the study with the correct code.
+        workflow1 = self.create_workflow('docx')
+        workflow2 = self.create_workflow('empty_workflow')
+
+        # Add files to both workflows.
+        FileService.add_workflow_file(workflow_id=workflow1.id,
+                                      name="anything.png", content_type="text",
+                                      binary_data=b'1234', irb_doc_code="UVACompl_PRCAppr" )
+        FileService.add_workflow_file(workflow_id=workflow1.id,
+                                      name="anything.png", content_type="text",
+                                      binary_data=b'1234', irb_doc_code="AD_Consent_Model")
+        FileService.add_workflow_file(workflow_id=workflow2.id,
+                                      name="anything.png", content_type="text",
+                                      binary_data=b'1234', irb_doc_code="UVACompl_PRCAppr" )
+
+        studies = StudyService().get_all_studies_with_files()
+        self.assertEquals(1, len(studies))
+        self.assertEquals(3, len(studies[0].files))
+
+
+
 
     @patch('crc.services.protocol_builder.ProtocolBuilderService.get_investigators')  # mock_docs
     def test_get_personnel(self, mock_docs):

tests/test_tasks_api.py

@@ -182,7 +182,6 @@ class TestTasksApi(BaseTest):
         self.assertEquals("Task 2b", nav[5]['title'])
         self.assertEquals("Task 3", nav[6]['title'])
 
-
     def test_document_added_to_workflow_shows_up_in_file_list(self):
         self.load_example_data()
         self.create_reference_document()
@@ -335,8 +334,8 @@ class TestTasksApi(BaseTest):
         workflow = self.get_workflow_api(workflow)
         task = workflow.next_task
         field_id = task.form['fields'][0]['id']
-        rv = self.app.get('/v1.0/workflow/%i/task/%s/lookup/%s?query=%s&limit=5' %
-                          (workflow.id, task.id, field_id, 'c'), # All records with a word that starts with 'c'
+        rv = self.app.get('/v1.0/workflow/%i/lookup/%s?query=%s&limit=5' %
+                          (workflow.id, field_id, 'c'), # All records with a word that starts with 'c'
                           headers=self.logged_in_headers(),
                           content_type="application/json")
         self.assert_success(rv)
@@ -351,8 +350,8 @@ class TestTasksApi(BaseTest):
         task = workflow.next_task
         field_id = task.form['fields'][0]['id']
         # lb3dp is a user record in the mock ldap responses for tests.
-        rv = self.app.get('/v1.0/workflow/%i/task/%s/lookup/%s?query=%s&limit=5' %
-                          (workflow.id, task.id, field_id, 'lb3dp'),
+        rv = self.app.get('/v1.0/workflow/%s/lookup/%s?query=%s&limit=5' %
+                          (workflow.id, field_id, 'lb3dp'),
                           headers=self.logged_in_headers(),
                           content_type="application/json")
         self.assert_success(rv)

tests/test_workflow_processor.py

@@ -223,10 +223,10 @@ class TestWorkflowProcessor(BaseTest):
         self._populate_form_with_random_data(task)
         processor.complete_task(task)
 
-        files = session.query(FileModel).filter_by(study_id=study.id, workflow_id=processor.get_workflow_id()).all()
+        files = session.query(FileModel).filter_by(workflow_id=processor.get_workflow_id()).all()
         self.assertEqual(0, len(files))
         processor.do_engine_steps()
-        files = session.query(FileModel).filter_by(study_id=study.id, workflow_id=processor.get_workflow_id()).all()
+        files = session.query(FileModel).filter_by(workflow_id=processor.get_workflow_id()).all()
         self.assertEqual(1, len(files), "The task should create a new file.")
         file_data = session.query(FileDataModel).filter(FileDataModel.file_model_id == files[0].id).first()
         self.assertIsNotNone(file_data.data)
@@ -254,12 +254,12 @@ class TestWorkflowProcessor(BaseTest):
         study = session.query(StudyModel).first()
         workflow_spec_model = self.load_test_spec("decision_table")
         processor = self.get_processor(study, workflow_spec_model)
-        self.assertTrue(processor.get_spec_version().startswith('v1.1'))
+        self.assertTrue(processor.get_version_string().startswith('v1.1'))
         file_service = FileService()
 
         file_service.add_workflow_spec_file(workflow_spec_model, "new_file.txt", "txt", b'blahblah')
         processor = self.get_processor(study, workflow_spec_model)
-        self.assertTrue(processor.get_spec_version().startswith('v1.1.1'))
+        self.assertTrue(processor.get_version_string().startswith('v1.1.1'))
 
         file_path = os.path.join(app.root_path, '..', 'tests', 'data', 'docx', 'docx.bpmn')
         file = open(file_path, "rb")
@@ -268,7 +268,7 @@ class TestWorkflowProcessor(BaseTest):
         file_model = db.session.query(FileModel).filter(FileModel.name == "decision_table.bpmn").first()
         file_service.update_file(file_model, data, "txt")
         processor = self.get_processor(study, workflow_spec_model)
-        self.assertTrue(processor.get_spec_version().startswith('v2.1.1'))
+        self.assertTrue(processor.get_version_string().startswith('v2.1.1'))
 
     def test_restart_workflow(self):
         self.load_example_data()
@@ -339,7 +339,7 @@ class TestWorkflowProcessor(BaseTest):
         # Assure that creating a new processor doesn't cause any issues, and maintains the spec version.
         processor.workflow_model.bpmn_workflow_json = processor.serialize()
         processor2 = WorkflowProcessor(processor.workflow_model)
-        self.assertTrue(processor2.get_spec_version().startswith("v1 ")) # Still at version 1.
+        self.assertFalse(processor2.is_latest_spec) # Still at version 1.
 
         # Do a hard reset, which should bring us back to the beginning, but retain the data.
         processor3 = WorkflowProcessor(processor.workflow_model, hard_reset=True)
@@ -349,10 +349,6 @@ class TestWorkflowProcessor(BaseTest):
         self.assertEqual("New Step", processor3.next_task().task_spec.description)
         self.assertEqual("blue", processor3.next_task().data["color"])
 
-    def test_get_latest_spec_version(self):
-        workflow_spec_model = self.load_test_spec("two_forms")
-        version = WorkflowProcessor.get_latest_version_string("two_forms")
-        self.assertTrue(version.startswith("v1 "))
 
     @patch('crc.services.protocol_builder.ProtocolBuilderService.get_studies')
     @patch('crc.services.protocol_builder.ProtocolBuilderService.get_investigators')

tests/test_workflow_service.py

@@ -1,7 +1,5 @@
 from tests.base_test import BaseTest
-from crc import session
-from crc.models.file import FileDataModel, FileModel, LookupFileModel, LookupDataModel
-from crc.services.lookup_service import LookupService
+
 from crc.services.workflow_processor import WorkflowProcessor
 from crc.services.workflow_service import WorkflowService
 
@@ -72,36 +70,6 @@ class TestWorkflowService(BaseTest):
         self.assertEquals('1000', options[0]['id'])
         self.assertEquals("UVA - INTERNAL - GM USE ONLY", options[0]['name'])
 
-    def test_create_lookup_file(self):
-        spec = self.load_test_spec('enum_options_from_file')
-        file_model = session.query(FileModel).filter(FileModel.name == "customer_list.xls").first()
-        file_data_model = session.query(FileDataModel).filter(FileDataModel.file_model == file_model).first()
-        LookupService.get_lookup_table_from_data_model(file_data_model, "CUSTOMER_NUMBER", "CUSTOMER_NAME")
-        lookup_records = session.query(LookupFileModel).all()
-        self.assertIsNotNone(lookup_records)
-        self.assertEqual(1, len(lookup_records))
-        lookup_record = lookup_records[0]
-        self.assertIsNotNone(lookup_record)
-        self.assertEquals("CUSTOMER_NUMBER", lookup_record.value_column)
-        self.assertEquals("CUSTOMER_NAME", lookup_record.label_column)
-        self.assertEquals("CUSTOMER_NAME", lookup_record.label_column)
-        lookup_data = session.query(LookupDataModel).filter(LookupDataModel.lookup_file_model == lookup_record).all()
-        self.assertEquals(28, len(lookup_data))
-
-        self.assertEquals("1000", lookup_data[0].value)
-        self.assertEquals("UVA - INTERNAL - GM USE ONLY", lookup_data[0].label)
-        # search_results = session.query(LookupDataModel).\
-        #     filter(LookupDataModel.lookup_file_model_id == lookup_record.id).\
-        #     filter(LookupDataModel.__ts_vector__.op('@@')(func.plainto_tsquery('INTERNAL'))).all()
-        search_results = LookupDataModel.query.filter(LookupDataModel.label.match("INTERNAL")).all()
-        self.assertEquals(1, len(search_results))
-        search_results = LookupDataModel.query.filter(LookupDataModel.label.match("internal")).all()
-        self.assertEquals(1, len(search_results))
-        # This query finds results where a word starts with "bio"
-        search_results = LookupDataModel.query.filter(LookupDataModel.label.match("bio:*")).all()
-        self.assertEquals(2, len(search_results))
-
-
     def test_random_data_populate_form_on_auto_complete(self):
         self.load_example_data()
         workflow = self.create_workflow('enum_options_with_search')