Merge branch 'dev' into bug/missing_pi_name_246

Pipfile

@@ -10,7 +10,6 @@ coverage = "*"
 
 [packages]
 alembic = "*"
-connexion = {extras = ["swagger-ui"],version = "*"}
 coverage = "*"
 docxtpl = "*"
 flask = "*"
@@ -36,11 +35,8 @@ pyjwt = "*"
 python-dateutil = "*"
 recommonmark = "*"
 requests = "*"
-sentry-sdk = {extras = ["flask"],version = "==0.14.4"}
 sphinx = "*"
 swagger-ui-bundle = "*"
-spiffworkflow = {git = "https://github.com/sartography/SpiffWorkflow.git"}
-# spiffworkflow = {editable = true, path = "./../SpiffWorkflow"}
 webtest = "*"
 werkzeug = "*"
 xlrd = "*"
@@ -50,3 +46,14 @@ apscheduler = "*"
 
 [requires]
 python_version = "3.8"
+
+[packages.connexion]
+extras = [ "swagger-ui",]
+version = "*"
+
+[packages.sentry-sdk]
+extras = [ "flask",]
+version = "==0.14.4"
+
+[packages.spiffworkflow]
+git = "https://github.com/sartography/SpiffWorkflow.git"

Pipfile.lock

@@ -979,7 +979,7 @@
         },
         "spiffworkflow": {
             "git": "https://github.com/sartography/SpiffWorkflow.git",
-            "ref": "59d12e25e5313977b83e7d65b6deb65572dee71c"
+            "ref": "1df28b940ec0d32b672e59e3d17e7a804cb2b186"
         },
         "sqlalchemy": {
             "hashes": [

config/default.py

@@ -15,6 +15,7 @@ JSON_SORT_KEYS = False  # CRITICAL.  Do not sort the data when returning values
 API_TOKEN = environ.get('API_TOKEN', default = 'af95596f327c9ecc007b60414fc84b61')
 
 NAME = "CR Connect Workflow"
+SERVER_NAME = environ.get('SERVER_NAME', default="localhost:5000")
 DEFAULT_PORT = "5000"
 FLASK_PORT = environ.get('PORT0') or environ.get('FLASK_PORT', default=DEFAULT_PORT)
 FRONTEND = environ.get('FRONTEND', default="localhost:4200")

config/testing.py

@@ -30,6 +30,3 @@ print('TESTING = ', TESTING)
 
 #Use the mock ldap.
 LDAP_URL = 'mock'
-
-from config.default import DEFAULT_PORT
-SERVER_NAME = f'localhost:{DEFAULT_PORT}'

crc/__init__.py

@@ -55,7 +55,7 @@ def process_waiting_tasks():
     with app.app_context():
         WorkflowService.do_waiting()
 
-scheduler.add_job(process_waiting_tasks,'interval',minutes=5)
+scheduler.add_job(process_waiting_tasks,'interval',minutes=1)
 scheduler.start()
 
 

crc/api.yml

@@ -424,6 +424,19 @@ paths:
       summary: Provides a list of workflows specifications that can be added to a study manually. Please note that Protocol Builder will handle this most of the time.
       tags:
         - Workflow Specifications
+      parameters :
+        - name : libraries
+          in : query
+          required : false
+          description : True if we should return just library schemas
+          schema :
+            type : boolean
+        - name : standalone
+          in : query
+          required : false
+          description : True if we should return just standalone schemas
+          schema :
+            type : boolean
       responses:
         '200':
           description: An array of workflow specifications
@@ -450,6 +463,50 @@ paths:
             application/json:
               schema:
                 $ref: "#/components/schemas/WorkflowSpec"
+
+
+  /workflow-specification/{spec_id}/library/{library_id}:
+    parameters:
+      - name: spec_id
+        in: path
+        required: true
+        description: The unique id of an existing workflow specification.
+        schema:
+          type: string
+      - name: library_id
+        in: path
+        required: true
+        description: The unique id of an existing library specification.
+        schema:
+          type: string
+
+
+    post:
+      operationId: crc.api.workflow.add_workflow_spec_library
+      summary: Adds a library to a workflow spec
+      tags:
+        - Workflow Specifications
+      responses:
+        '200':
+          description: Workflow specification.
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/WorkflowSpec"
+    delete:
+      operationId: crc.api.workflow.drop_workflow_spec_library
+      summary: Delete a library from a workflow
+      tags:
+        - Workflow Specifications
+      responses:
+        '200':
+          description: Workflow specification.
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/WorkflowSpec"
+
+
   /workflow-specification/{spec_id}:
     parameters:
       - name: spec_id
@@ -511,21 +568,6 @@ paths:
       responses:
         '204':
           description: The workflow specification has been removed.
-  /workflow-specification/standalone:
-    get:
-      operationId: crc.api.workflow.standalone_workflow_specs
-      summary: Provides a list of workflow specifications that can be run outside a study.
-      tags:
-        - Workflow Specifications
-      responses:
-        '200':
-          description: A list of workflow specifications
-          content:
-            application/json:
-              schema:
-                type: array
-                items:
-                  $ref: "#/components/schemas/WorkflowSpec"
   /workflow-specification/{spec_id}/validate:
     parameters:
       - name: spec_id

crc/api/workflow.py

@@ -10,7 +10,7 @@ from crc.models.study import StudyModel, WorkflowMetadata, StudyStatus
 from crc.models.task_event import TaskEventModel, TaskEvent, TaskEventSchema
 from crc.models.user import UserModelSchema
 from crc.models.workflow import WorkflowModel, WorkflowSpecModelSchema, WorkflowSpecModel, WorkflowSpecCategoryModel, \
-    WorkflowSpecCategoryModelSchema
+    WorkflowSpecCategoryModelSchema, WorkflowLibraryModel, WorkflowLibraryModelSchema
 from crc.services.error_service import ValidationErrorService
 from crc.services.file_service import FileService
 from crc.services.lookup_service import LookupService
@@ -20,9 +20,22 @@ from crc.services.workflow_processor import WorkflowProcessor
 from crc.services.workflow_service import WorkflowService
 
 
-def all_specifications():
+def all_specifications(libraries=False,standalone=False):
+    if libraries and standalone:
+        raise ApiError('inconceivable!', 'You should specify libraries or standalone, but not both')
     schema = WorkflowSpecModelSchema(many=True)
-    return schema.dump(session.query(WorkflowSpecModel).all())
+    if libraries:
+        return schema.dump(session.query(WorkflowSpecModel)\
+              .filter(WorkflowSpecModel.library==True).all())
+
+    if standalone:
+        return schema.dump(session.query(WorkflowSpecModel)\
+              .filter(WorkflowSpecModel.standalone==True).all())
+    # this still returns standalone workflow specs as well, but by default
+    # we do not return specs marked as library
+    return schema.dump(session.query(WorkflowSpecModel)\
+              .filter((WorkflowSpecModel.library==False)|(
+              WorkflowSpecModel.library==None)).all())
 
 
 def add_workflow_specification(body):
@@ -45,6 +58,41 @@ def get_workflow_specification(spec_id):
 
     return WorkflowSpecModelSchema().dump(spec)
 
+def validate_spec_and_library(spec_id,library_id):
+    if spec_id is None:
+        raise ApiError('unknown_spec', 'Please provide a valid Workflow Specification ID.')
+    if library_id is None:
+        raise ApiError('unknown_spec', 'Please provide a valid Library Specification ID.')
+    spec: WorkflowSpecModel = session.query(WorkflowSpecModel).filter_by(id=spec_id).first()
+    library: WorkflowSpecModel = session.query(WorkflowSpecModel).filter_by(id=library_id).first()
+    if spec is None:
+        raise ApiError('unknown_spec', 'The Workflow Specification "' + spec_id + '" is not recognized.')
+    if library is None:
+        raise ApiError('unknown_spec', 'The Library Specification "' + library_id + '" is not recognized.')
+    if not library.library:
+        raise ApiError('unknown_spec', 'Linked workflow spec is not a library.')
+
+
+def add_workflow_spec_library(spec_id,library_id):
+    validate_spec_and_library(spec_id, library_id)
+    libraries: WorkflowLibraryModel = session.query(WorkflowLibraryModel).filter_by(workflow_spec_id=spec_id).all()
+    libraryids = [x.library_spec_id for x in libraries]
+    if library_id in libraryids:
+        raise ApiError('unknown_spec', 'The Library Specification "' + spec_id + '" is already attached.')
+    newlib = WorkflowLibraryModel()
+    newlib.workflow_spec_id = spec_id
+    newlib.library_spec_id = library_id
+    session.add(newlib)
+    session.commit()
+    libraries: WorkflowLibraryModel = session.query(WorkflowLibraryModel).filter_by(workflow_spec_id=spec_id).all()
+    return WorkflowLibraryModelSchema(many=True).dump(libraries)
+
+def drop_workflow_spec_library(spec_id,library_id):
+    validate_spec_and_library(spec_id, library_id)
+    session.query(WorkflowLibraryModel).filter_by(workflow_spec_id=spec_id,library_spec_id=library_id).delete()
+    session.commit()
+    libraries: WorkflowLibraryModel = session.query(WorkflowLibraryModel).filter_by(workflow_spec_id=spec_id).all()
+    return WorkflowLibraryModelSchema(many=True).dump(libraries)
 
 def validate_workflow_specification(spec_id, study_id=None, test_until=None):
     try:
@@ -106,12 +154,6 @@ def get_workflow_from_spec(spec_id):
     return WorkflowApiSchema().dump(workflow_api_model)
 
 
-def standalone_workflow_specs():
-    schema = WorkflowSpecModelSchema(many=True)
-    specs = WorkflowService.get_standalone_workflow_specs()
-    return schema.dump(specs)
-
-
 def get_workflow(workflow_id, do_engine_steps=True):
     """Retrieve workflow based on workflow_id, and return it in the last saved State.
        If do_engine_steps is False, return the workflow without running any engine tasks or logging any events. """

crc/models/workflow.py

@@ -1,7 +1,7 @@
 import enum
 
 import marshmallow
-from marshmallow import EXCLUDE
+from marshmallow import EXCLUDE,fields
 from marshmallow_sqlalchemy import SQLAlchemyAutoSchema
 from sqlalchemy import func
 
@@ -24,6 +24,7 @@ class WorkflowSpecCategoryModelSchema(SQLAlchemyAutoSchema):
         include_relationships = True
 
 
+
 class WorkflowSpecModel(db.Model):
     __tablename__ = 'workflow_spec'
     id = db.Column(db.String, primary_key=True)
@@ -35,6 +36,19 @@ class WorkflowSpecModel(db.Model):
     category = db.relationship("WorkflowSpecCategoryModel")
     is_master_spec = db.Column(db.Boolean, default=False)
     standalone = db.Column(db.Boolean, default=False)
+    library = db.Column(db.Boolean, default=False)
+
+
+class WorkflowLibraryModel(db.Model):
+   __tablename__ = 'workflow_library'
+   id = db.Column(db.Integer, primary_key=True)
+   workflow_spec_id = db.Column(db.String, db.ForeignKey('workflow_spec.id'), nullable=True)
+   library_spec_id = db.Column(db.String, db.ForeignKey('workflow_spec.id'), nullable=True)
+   parent = db.relationship(WorkflowSpecModel,
+                                   primaryjoin=workflow_spec_id==WorkflowSpecModel.id,
+                                  backref='libraries')
+   library = db.relationship(WorkflowSpecModel,primaryjoin=library_spec_id==WorkflowSpecModel.id,
+                                  backref='parents')
 
 
 class WorkflowSpecModelSchema(SQLAlchemyAutoSchema):
@@ -46,7 +60,14 @@ class WorkflowSpecModelSchema(SQLAlchemyAutoSchema):
         unknown = EXCLUDE
 
     category = marshmallow.fields.Nested(WorkflowSpecCategoryModelSchema, dump_only=True)
-
+    libraries = marshmallow.fields.Function(lambda obj: [{'id':x.library.id,
+                                                          'name':x.library.name,
+                                                          'display_name':x.library.display_name} for x in
+                                                          obj.libraries] )
+    parents = marshmallow.fields.Function(lambda obj: [{'id':x.parent.id,
+                                                          'name':x.parent.name,
+                                                          'display_name':x.parent.display_name} for x in
+                                                          obj.parents] )
 
 class WorkflowState(enum.Enum):
     hidden = "hidden"
@@ -78,6 +99,15 @@ class WorkflowSpecDependencyFile(db.Model):
     file_data = db.relationship(FileDataModel)
 
 
+
+class WorkflowLibraryModelSchema(SQLAlchemyAutoSchema):
+    class Meta:
+        model = WorkflowLibraryModel
+        load_instance = True
+        include_relationships = True
+
+    library = marshmallow.fields.Nested('WorkflowSpecModelSchema')
+
 class WorkflowModel(db.Model):
     __tablename__ = 'workflow'
     id = db.Column(db.Integer, primary_key=True)

crc/scripts/email.py

@@ -1,3 +1,6 @@
+import sys
+import traceback
+
 from crc import app
 from crc.api.common import ApiError
 from crc.scripts.script import Script
@@ -44,16 +47,24 @@ email (subject="My Subject", recipients=["dhf8r@virginia.edu", pi.email], cc='as
         if recipients:
             message = task.task_spec.documentation
             data = task.data
-            content, content_html = EmailService().get_rendered_content(message, data)
-            EmailService.add_email(
-                subject=subject,
-                sender=app.config['DEFAULT_SENDER'],
-                recipients=recipients,
-                content=content,
-                content_html=content_html,
-                cc=cc,
-                study_id=study_id
-            )
+            try:
+                content, content_html = EmailService().get_rendered_content(message, data)
+                EmailService.add_email(
+                    subject=subject,
+                    sender=app.config['DEFAULT_SENDER'],
+                    recipients=recipients,
+                    content=content,
+                    content_html=content_html,
+                    cc=cc,
+                    study_id=study_id
+                )
+            except Exception as e:
+                exc_type, exc_value, exc_traceback = sys.exc_info()
+                print("*** format_exception:")
+                # exc_type below is ignored on 3.5 and later
+                print(repr(traceback.format_exception(exc_type, exc_value,
+                                                      exc_traceback)))
+                raise e
 
     def get_email_addresses(self, users, study_id):
         emails = []

crc/scripts/ldap.py

@@ -48,21 +48,29 @@ supervisor_info = ldap(supervisor_uid)   // Sets the supervisor information to l
                                    "UID for the person we want to look up.")
         if len(args) < 1:
             if UserService.has_user():
-               uid = UserService.current_user().uid
+                uid = UserService.current_user().uid
         else:
             uid = args[0]
-        user_info = LdapService.user_info(uid)
-        user_info_dict = {
-            "display_name": user_info.display_name,
-            "given_name": user_info.given_name,
-            "email_address": user_info.email_address,
-            "telephone_number": user_info.telephone_number,
-            "title": user_info.title,
-            "department": user_info.department,
-            "affiliation": user_info.affiliation,
-            "sponsor_type": user_info.sponsor_type,
-            "uid": user_info.uid,
-            "proper_name": user_info.proper_name()
-        }
 
-        return user_info_dict
+        try:
+            user_info = LdapService.user_info(uid)
+        except ApiError as ae:
+            app.logger.info(ae)
+            return {}
+        except Exception as e:
+            app.logger.info(e)
+            return {}
+        else:
+            user_info_dict = {
+                "display_name": user_info.display_name,
+                "given_name": user_info.given_name,
+                "email_address": user_info.email_address,
+                "telephone_number": user_info.telephone_number,
+                "title": user_info.title,
+                "department": user_info.department,
+                "affiliation": user_info.affiliation,
+                "sponsor_type": user_info.sponsor_type,
+                "uid": user_info.uid,
+                "proper_name": user_info.proper_name()
+            }
+            return user_info_dict

crc/services/email_service.py

@@ -68,4 +68,5 @@ class EmailService(object):
 
     @staticmethod
     def get_cr_connect_wrapper(email_body):
-        return render_template('mail_content_template.html', email_body=email_body, base_url=request.base_url)
+        base_url = app.config['FRONTEND']  # The frontend url
+        return render_template('mail_content_template.html', email_body=email_body, base_url=base_url)

crc/services/file_service.py

@@ -17,7 +17,7 @@ from crc import session, app
 from crc.api.common import ApiError
 from crc.models.data_store import DataStoreModel
 from crc.models.file import FileType, FileDataModel, FileModel, LookupFileModel, LookupDataModel
-from crc.models.workflow import WorkflowSpecModel, WorkflowModel, WorkflowSpecDependencyFile
+from crc.models.workflow import WorkflowSpecModel, WorkflowModel, WorkflowSpecDependencyFile, WorkflowLibraryModel
 from crc.services.cache_service import cache
 from crc.services.user_service import UserService
 import re
@@ -233,10 +233,18 @@ class FileService(object):
 
     @staticmethod
     def get_files(workflow_spec_id=None, workflow_id=None,
-                  name=None, is_reference=False, irb_doc_code=None):
+                  name=None, is_reference=False, irb_doc_code=None, include_libraries=False):
         query = session.query(FileModel).filter_by(is_reference=is_reference)
         if workflow_spec_id:
-            query = query.filter_by(workflow_spec_id=workflow_spec_id)
+            if include_libraries:
+                libraries = session.query(WorkflowLibraryModel).filter(
+                   WorkflowLibraryModel.workflow_spec_id==workflow_spec_id).all()
+                library_workflow_specs = [x.library_spec_id for x in libraries]
+                library_workflow_specs.append(workflow_spec_id)
+                query = query.filter(FileModel.workflow_spec_id.in_(library_workflow_specs))
+            else:
+                query = query.filter(FileModel.workflow_spec_id == workflow_spec_id)
+
         elif workflow_id:
             query = query.filter_by(workflow_id=workflow_id)
             if irb_doc_code:
@@ -255,9 +263,9 @@ class FileService(object):
         return results
 
     @staticmethod
-    def get_spec_data_files(workflow_spec_id, workflow_id=None, name=None):
+    def get_spec_data_files(workflow_spec_id, workflow_id=None, name=None, include_libraries=False):
         """Returns all the FileDataModels related to a workflow specification.
-        If a workflow is specified, returns the version of the spec relatted
+        If a workflow is specified, returns the version of the spec related
         to that workflow, otherwise, returns the lastest files."""
         if workflow_id:
             query = session.query(FileDataModel) \
@@ -269,7 +277,7 @@ class FileService(object):
             return query.all()
         else:
             """Returns all the latest files related to a workflow specification"""
-            file_models = FileService.get_files(workflow_spec_id=workflow_spec_id)
+            file_models = FileService.get_files(workflow_spec_id=workflow_spec_id,include_libraries=include_libraries)
             latest_data_files = []
             for file_model in file_models:
                 if name and file_model.name == name:

crc/services/ldap_service.py

@@ -58,6 +58,7 @@ class LdapService(object):
 
     @staticmethod
     def user_info(uva_uid):
+        uva_uid = uva_uid.lower()
         user_info = db.session.query(LdapModel).filter(LdapModel.uid == uva_uid).first()
         if not user_info:
             app.logger.info("No cache for " + uva_uid)

crc/services/workflow_processor.py

@@ -117,7 +117,7 @@ class WorkflowProcessor(object):
 
         if workflow_model.bpmn_workflow_json is None:  # The workflow was never started.
             self.spec_data_files = FileService.get_spec_data_files(
-                workflow_spec_id=workflow_model.workflow_spec_id)
+                workflow_spec_id=workflow_model.workflow_spec_id,include_libraries=True)
             spec = self.get_spec(self.spec_data_files, workflow_model.workflow_spec_id)
         else:
             self.spec_data_files = FileService.get_spec_data_files(
@@ -312,7 +312,7 @@ class WorkflowProcessor(object):
         for file_data in file_data_models:
             if file_data.file_model.type == FileType.bpmn:
                 bpmn: etree.Element = etree.fromstring(file_data.data)
-                if file_data.file_model.primary:
+                if file_data.file_model.primary and file_data.file_model.workflow_spec_id == workflow_spec_id:
                     process_id = FileService.get_process_id(bpmn)
                 parser.add_bpmn_xml(bpmn, filename=file_data.file_model.name)
             elif file_data.file_model.type == FileType.dmn:
@@ -389,10 +389,15 @@ class WorkflowProcessor(object):
         """
 
         # If the whole blessed mess is done, return the end_event task in the tree
+        # This was failing in the case of a call activity where we have an intermediate EndEvent
+        # what we really want is the LAST EndEvent
+
+        endtasks = []
         if self.bpmn_workflow.is_completed():
             for task in SpiffTask.Iterator(self.bpmn_workflow.task_tree, SpiffTask.ANY_MASK):
                 if isinstance(task.task_spec, EndEvent):
-                    return task
+                    endtasks.append(task)
+            return endtasks[-1]
 
         # If there are ready tasks to complete, return the next ready task, but return the one
         # in the active parallel path if possible.  In some cases the active parallel path may itself be

crc/services/workflow_service.py

@@ -30,6 +30,8 @@ from crc.models.study import StudyModel
 from crc.models.task_event import TaskEventModel
 from crc.models.user import UserModel, UserModelSchema
 from crc.models.workflow import WorkflowModel, WorkflowStatus, WorkflowSpecModel
+from crc.services.data_store_service import DataStoreBase
+
 from crc.services.document_service import DocumentService
 from crc.services.file_service import FileService
 from crc.services.lookup_service import LookupService
@@ -312,8 +314,11 @@ class WorkflowService(object):
                 field.get_property(Task.FIELD_PROP_FILE_DATA) in data and \
                 field.id in data:
             file_id = data[field.get_property(Task.FIELD_PROP_FILE_DATA)]["id"]
-            data_store = DataStoreModel(file_id=file_id, key=field.id, value=data[field.id])
-            db.session.add(data_store)
+            if field.type == 'enum':
+                data_args = (field.id, data[field.id]['label'])
+            else:
+                data_args = (field.id, data[field.id])
+            DataStoreBase().set_data_common(task.id, None, None, None, None, None, file_id, *data_args)
 
     @staticmethod
     def evaluate_property(property_name, field, task):
@@ -893,6 +898,11 @@ class WorkflowService(object):
         specs = db.session.query(WorkflowSpecModel).filter_by(standalone=True).all()
         return specs
 
+    @staticmethod
+    def get_library_workflow_specs():
+        specs = db.session.query(WorkflowSpecModel).filter_by(library=True).all()
+        return specs
+
     @staticmethod
     def get_primary_workflow(workflow_spec_id):
         # Returns the FileModel of the primary workflow for a workflow_spec

example_data.py

@@ -16,8 +16,11 @@ class ExampleDataLoader:
     @staticmethod
     def clean_db():
         session.flush()  # Clear out any transactions before deleting it all to avoid spurious errors.
+        engine = session.bind.engine
+        connection = engine.connect()
         for table in reversed(db.metadata.sorted_tables):
-            session.execute(table.delete())
+            if engine.dialect.has_table(connection, table):
+                session.execute(table.delete())
         session.commit()
         session.flush()
 
@@ -268,7 +271,7 @@ class ExampleDataLoader:
                          from_tests=True)
 
     def create_spec(self, id, name, display_name="", description="", filepath=None, master_spec=False,
-                    category_id=None, display_order=None, from_tests=False, standalone=False):
+                    category_id=None, display_order=None, from_tests=False, standalone=False, library=False):
         """Assumes that a directory exists in static/bpmn with the same name as the given id.
            further assumes that the [id].bpmn is the primary file for the workflow.
            returns an array of data models to be added to the database."""
@@ -281,7 +284,8 @@ class ExampleDataLoader:
                                  is_master_spec=master_spec,
                                  category_id=category_id,
                                  display_order=display_order,
-                                 standalone=standalone)
+                                 standalone=standalone,
+                                 library=library)
         db.session.add(spec)
         db.session.commit()
         if not filepath and not from_tests:

migrations/versions/2a6f7ea00e5f_.py

@@ -0,0 +1,37 @@
+"""empty message
+
+Revision ID: 2a6f7ea00e5f
+Revises: dc30b8f6571c
+Create Date: 2021-07-29 10:51:56.857341
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '2a6f7ea00e5f'
+down_revision = 'dc30b8f6571c'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('workflow_library',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('workflow_spec_id', sa.String(), nullable=True),
+    sa.Column('library_spec_id', sa.String(), nullable=True),
+    sa.ForeignKeyConstraint(['library_spec_id'], ['workflow_spec.id'], ),
+    sa.ForeignKeyConstraint(['workflow_spec_id'], ['workflow_spec.id'], ),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.add_column('workflow_spec', sa.Column('library', sa.Boolean(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('workflow_spec', 'library')
+    op.drop_table('workflow_library')
+    # ### end Alembic commands ###

migrations/versions/c16d3047abbe_.py

@@ -11,7 +11,7 @@ import sqlalchemy as sa
 
 # revision identifiers, used by Alembic.
 revision = 'c16d3047abbe'
-down_revision = 'bbf064082623'
+down_revision = '30e017a03948'
 branch_labels = None
 depends_on = None
 

tests/data/call_activity_end_event/call_activity_call_activity.bpmn

@@ -0,0 +1,42 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<bpmn:definitions xmlns:bpmn="http://www.omg.org/spec/BPMN/20100524/MODEL" xmlns:bpmndi="http://www.omg.org/spec/BPMN/20100524/DI" xmlns:dc="http://www.omg.org/spec/DD/20100524/DC" xmlns:di="http://www.omg.org/spec/DD/20100524/DI" id="Definitions_34b94b6" targetNamespace="http://bpmn.io/schema/bpmn" exporter="Camunda Modeler" exporterVersion="3.0.0-dev">
+  <bpmn:process id="Call_Activity_Get_Data" name="Call Activity Get Data" isExecutable="true">
+    <bpmn:startEvent id="StartEvent_1">
+      <bpmn:outgoing>Flow_07uhaa7</bpmn:outgoing>
+    </bpmn:startEvent>
+    <bpmn:sequenceFlow id="Flow_07uhaa7" sourceRef="StartEvent_1" targetRef="Activity_1mb2mnf" />
+    <bpmn:endEvent id="Event_1rokcus">
+      <bpmn:documentation># Call Event
+&lt;div&gt;&lt;span&gt;Hello {{my_var}}&lt;/span&gt;&lt;/div&gt;</bpmn:documentation>
+      <bpmn:incoming>Flow_0apfnjq</bpmn:incoming>
+    </bpmn:endEvent>
+    <bpmn:sequenceFlow id="Flow_0apfnjq" sourceRef="Activity_1mb2mnf" targetRef="Event_1rokcus" />
+    <bpmn:scriptTask id="Activity_1mb2mnf" name="Create Data">
+      <bpmn:incoming>Flow_07uhaa7</bpmn:incoming>
+      <bpmn:outgoing>Flow_0apfnjq</bpmn:outgoing>
+      <bpmn:script>my_var = 'World'
+my_other_var = 'Mike'</bpmn:script>
+    </bpmn:scriptTask>
+  </bpmn:process>
+  <bpmndi:BPMNDiagram id="BPMNDiagram_1">
+    <bpmndi:BPMNPlane id="BPMNPlane_1" bpmnElement="Call_Activity_Get_Data">
+      <bpmndi:BPMNEdge id="Flow_07uhaa7_di" bpmnElement="Flow_07uhaa7">
+        <di:waypoint x="215" y="177" />
+        <di:waypoint x="270" y="177" />
+      </bpmndi:BPMNEdge>
+      <bpmndi:BPMNEdge id="Flow_0apfnjq_di" bpmnElement="Flow_0apfnjq">
+        <di:waypoint x="370" y="177" />
+        <di:waypoint x="432" y="177" />
+      </bpmndi:BPMNEdge>
+      <bpmndi:BPMNShape id="_BPMNShape_StartEvent_2" bpmnElement="StartEvent_1">
+        <dc:Bounds x="179" y="159" width="36" height="36" />
+      </bpmndi:BPMNShape>
+      <bpmndi:BPMNShape id="Event_1rokcus_di" bpmnElement="Event_1rokcus">
+        <dc:Bounds x="432" y="159" width="36" height="36" />
+      </bpmndi:BPMNShape>
+      <bpmndi:BPMNShape id="Activity_0esr09m_di" bpmnElement="Activity_1mb2mnf">
+        <dc:Bounds x="270" y="137" width="100" height="80" />
+      </bpmndi:BPMNShape>
+    </bpmndi:BPMNPlane>
+  </bpmndi:BPMNDiagram>
+</bpmn:definitions>

tests/data/call_activity_end_event/call_activity_end_event.bpmn

@@ -0,0 +1,70 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<bpmn:definitions xmlns:bpmn="http://www.omg.org/spec/BPMN/20100524/MODEL" xmlns:bpmndi="http://www.omg.org/spec/BPMN/20100524/DI" xmlns:dc="http://www.omg.org/spec/DD/20100524/DC" xmlns:di="http://www.omg.org/spec/DD/20100524/DI" id="Definitions_f07329e" targetNamespace="http://bpmn.io/schema/bpmn" exporter="Camunda Modeler" exporterVersion="3.0.0-dev">
+  <bpmn:process id="Process_8200379" isExecutable="true">
+    <bpmn:startEvent id="StartEvent_1">
+      <bpmn:outgoing>Flow_1g3dpd7</bpmn:outgoing>
+    </bpmn:startEvent>
+    <bpmn:sequenceFlow id="Flow_1g3dpd7" sourceRef="StartEvent_1" targetRef="Activity_0wppf2v" />
+    <bpmn:sequenceFlow id="Flow_0ovgj6c" sourceRef="Activity_0wppf2v" targetRef="Activity_12zat0d" />
+    <bpmn:callActivity id="Activity_12zat0d" name="Get Data Call Activity" calledElement="Call_Activity_Get_Data">
+      <bpmn:incoming>Flow_0ovgj6c</bpmn:incoming>
+      <bpmn:outgoing>Flow_0qdgvah</bpmn:outgoing>
+    </bpmn:callActivity>
+    <bpmn:sequenceFlow id="Flow_0qdgvah" sourceRef="Activity_12zat0d" targetRef="Activity_1ta6769" />
+    <bpmn:endEvent id="Event_18dla68">
+      <bpmn:documentation># Main Workflow
+Hello {{my_other_var}}
+
+</bpmn:documentation>
+      <bpmn:incoming>Flow_0izaz4f</bpmn:incoming>
+    </bpmn:endEvent>
+    <bpmn:sequenceFlow id="Flow_0izaz4f" sourceRef="Activity_1ta6769" targetRef="Event_18dla68" />
+    <bpmn:scriptTask id="Activity_1ta6769" name="Print Data">
+      <bpmn:incoming>Flow_0qdgvah</bpmn:incoming>
+      <bpmn:outgoing>Flow_0izaz4f</bpmn:outgoing>
+      <bpmn:script>print(pre_var)
+print(my_var)
+print(my_other_var)</bpmn:script>
+    </bpmn:scriptTask>
+    <bpmn:scriptTask id="Activity_0wppf2v" name="Pre Data">
+      <bpmn:incoming>Flow_1g3dpd7</bpmn:incoming>
+      <bpmn:outgoing>Flow_0ovgj6c</bpmn:outgoing>
+      <bpmn:script>pre_var = 'some string'</bpmn:script>
+    </bpmn:scriptTask>
+  </bpmn:process>
+  <bpmndi:BPMNDiagram id="BPMNDiagram_1">
+    <bpmndi:BPMNPlane id="BPMNPlane_1" bpmnElement="Process_8200379">
+      <bpmndi:BPMNEdge id="Flow_0izaz4f_di" bpmnElement="Flow_0izaz4f">
+        <di:waypoint x="690" y="177" />
+        <di:waypoint x="752" y="177" />
+      </bpmndi:BPMNEdge>
+      <bpmndi:BPMNEdge id="Flow_0qdgvah_di" bpmnElement="Flow_0qdgvah">
+        <di:waypoint x="530" y="177" />
+        <di:waypoint x="590" y="177" />
+      </bpmndi:BPMNEdge>
+      <bpmndi:BPMNEdge id="Flow_0ovgj6c_di" bpmnElement="Flow_0ovgj6c">
+        <di:waypoint x="370" y="177" />
+        <di:waypoint x="430" y="177" />
+      </bpmndi:BPMNEdge>
+      <bpmndi:BPMNEdge id="Flow_1g3dpd7_di" bpmnElement="Flow_1g3dpd7">
+        <di:waypoint x="215" y="177" />
+        <di:waypoint x="270" y="177" />
+      </bpmndi:BPMNEdge>
+      <bpmndi:BPMNShape id="_BPMNShape_StartEvent_2" bpmnElement="StartEvent_1">
+        <dc:Bounds x="179" y="159" width="36" height="36" />
+      </bpmndi:BPMNShape>
+      <bpmndi:BPMNShape id="Activity_0mcej1g_di" bpmnElement="Activity_12zat0d">
+        <dc:Bounds x="430" y="137" width="100" height="80" />
+      </bpmndi:BPMNShape>
+      <bpmndi:BPMNShape id="Event_18dla68_di" bpmnElement="Event_18dla68">
+        <dc:Bounds x="752" y="159" width="36" height="36" />
+      </bpmndi:BPMNShape>
+      <bpmndi:BPMNShape id="Activity_1v8hse1_di" bpmnElement="Activity_1ta6769">
+        <dc:Bounds x="590" y="137" width="100" height="80" />
+      </bpmndi:BPMNShape>
+      <bpmndi:BPMNShape id="Activity_1mhwjko_di" bpmnElement="Activity_0wppf2v">
+        <dc:Bounds x="270" y="137" width="100" height="80" />
+      </bpmndi:BPMNShape>
+    </bpmndi:BPMNPlane>
+  </bpmndi:BPMNDiagram>
+</bpmn:definitions>

tests/data/enum_file_data/enum_file_data.bpmn

@@ -0,0 +1,88 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<bpmn:definitions xmlns:bpmn="http://www.omg.org/spec/BPMN/20100524/MODEL" xmlns:bpmndi="http://www.omg.org/spec/BPMN/20100524/DI" xmlns:dc="http://www.omg.org/spec/DD/20100524/DC" xmlns:camunda="http://camunda.org/schema/1.0/bpmn" xmlns:di="http://www.omg.org/spec/DD/20100524/DI" id="Definitions_0b469f0" targetNamespace="http://bpmn.io/schema/bpmn" exporter="Camunda Modeler" exporterVersion="3.5.0">
+  <bpmn:process id="Process_4b7fa29" isExecutable="true">
+    <bpmn:startEvent id="StartEvent_1">
+      <bpmn:outgoing>Flow_1kvuzs1</bpmn:outgoing>
+    </bpmn:startEvent>
+    <bpmn:sequenceFlow id="Flow_1kvuzs1" sourceRef="StartEvent_1" targetRef="Activity_0gtrm5e" />
+    <bpmn:userTask id="Activity_0gtrm5e" name="Select Enum" camunda:formKey="Upload Application">
+      <bpmn:extensionElements>
+        <camunda:formData>
+          <camunda:formField id="IRB_HSR_Application_Type" label="IRB-HSR Application Type" type="enum">
+            <camunda:properties>
+              <camunda:property id="file_data" value="Study_App_Doc" />
+              <camunda:property id="spreadsheet.name" value="IRB_HSR_Application_Type.xlsx" />
+              <camunda:property id="spreadsheet.label.column" value="Label" />
+              <camunda:property id="spreadsheet.value.column" value="Value" />
+              <camunda:property id="group" value="Application" />
+            </camunda:properties>
+            <camunda:validation>
+              <camunda:constraint name="required" config="True" />
+            </camunda:validation>
+          </camunda:formField>
+          <camunda:formField id="Study_App_Doc" label="IRB-HSR Application" type="file" />
+          <camunda:formField id="my_test_field" label="Nickname" type="string">
+            <camunda:properties>
+              <camunda:property id="file_data" value="Study_App_Doc" />
+            </camunda:properties>
+          </camunda:formField>
+          <camunda:formField id="some_date" label="Date" type="date">
+            <camunda:properties>
+              <camunda:property id="file_data" value="Study_App_Doc" />
+            </camunda:properties>
+          </camunda:formField>
+          <camunda:formField id="a_boolean" label="A Boolean" type="boolean">
+            <camunda:properties>
+              <camunda:property id="file_data" value="Study_App_Doc" />
+            </camunda:properties>
+          </camunda:formField>
+          <camunda:formField id="the_number" label="The Number" type="long">
+            <camunda:properties>
+              <camunda:property id="file_data" value="Study_App_Doc" />
+            </camunda:properties>
+          </camunda:formField>
+        </camunda:formData>
+      </bpmn:extensionElements>
+      <bpmn:incoming>Flow_1kvuzs1</bpmn:incoming>
+      <bpmn:outgoing>Flow_0han7ki</bpmn:outgoing>
+    </bpmn:userTask>
+    <bpmn:sequenceFlow id="Flow_0han7ki" sourceRef="Activity_0gtrm5e" targetRef="Activity_0gpetln" />
+    <bpmn:manualTask id="Activity_0gpetln" name="Print Enum">
+      <bpmn:documentation># Enum</bpmn:documentation>
+      <bpmn:incoming>Flow_0han7ki</bpmn:incoming>
+      <bpmn:outgoing>Flow_0h5cdg9</bpmn:outgoing>
+    </bpmn:manualTask>
+    <bpmn:endEvent id="Event_124aupd">
+      <bpmn:incoming>Flow_0h5cdg9</bpmn:incoming>
+    </bpmn:endEvent>
+    <bpmn:sequenceFlow id="Flow_0h5cdg9" sourceRef="Activity_0gpetln" targetRef="Event_124aupd" />
+  </bpmn:process>
+  <bpmndi:BPMNDiagram id="BPMNDiagram_1">
+    <bpmndi:BPMNPlane id="BPMNPlane_1" bpmnElement="Process_4b7fa29">
+      <bpmndi:BPMNEdge id="Flow_0h5cdg9_di" bpmnElement="Flow_0h5cdg9">
+        <di:waypoint x="530" y="117" />
+        <di:waypoint x="592" y="117" />
+      </bpmndi:BPMNEdge>
+      <bpmndi:BPMNEdge id="Flow_0han7ki_di" bpmnElement="Flow_0han7ki">
+        <di:waypoint x="370" y="117" />
+        <di:waypoint x="430" y="117" />
+      </bpmndi:BPMNEdge>
+      <bpmndi:BPMNEdge id="Flow_1kvuzs1_di" bpmnElement="Flow_1kvuzs1">
+        <di:waypoint x="215" y="117" />
+        <di:waypoint x="270" y="117" />
+      </bpmndi:BPMNEdge>
+      <bpmndi:BPMNShape id="_BPMNShape_StartEvent_2" bpmnElement="StartEvent_1">
+        <dc:Bounds x="179" y="99" width="36" height="36" />
+      </bpmndi:BPMNShape>
+      <bpmndi:BPMNShape id="Activity_11t0hqf_di" bpmnElement="Activity_0gtrm5e">
+        <dc:Bounds x="270" y="77" width="100" height="80" />
+      </bpmndi:BPMNShape>
+      <bpmndi:BPMNShape id="Activity_0fax1lv_di" bpmnElement="Activity_0gpetln">
+        <dc:Bounds x="430" y="77" width="100" height="80" />
+      </bpmndi:BPMNShape>
+      <bpmndi:BPMNShape id="Event_124aupd_di" bpmnElement="Event_124aupd">
+        <dc:Bounds x="592" y="99" width="36" height="36" />
+      </bpmndi:BPMNShape>
+    </bpmndi:BPMNPlane>
+  </bpmndi:BPMNDiagram>
+</bpmn:definitions>

tests/ldap/test_ldap_lookup_script.py

@@ -45,8 +45,8 @@ class TestLdapLookupScript(BaseTest):
         }
 
         script = Ldap()
-        with(self.assertRaises(ApiError)):
-            user_details = script.do_task(task, workflow.study_id, workflow.id, "PIComputingID")
+        user_details = script.do_task(task, workflow.study_id, workflow.id, "PIComputingID")
+        self.assertEqual({}, user_details)
 
     def test_get_current_user_details(self):
         self.load_example_data()

tests/test_file_datastore.py

@@ -1,18 +1,17 @@
-import json
+from tests.base_test import BaseTest
 
+from crc import db
+from crc.models.data_store import DataStoreModel
 from crc.services.file_service import FileService
 from crc.services.workflow_processor import WorkflowProcessor
-from tests.base_test import BaseTest
-from crc.models.workflow import WorkflowStatus
-from crc import db
-from crc.api.common import ApiError
-from crc.models.task_event import TaskEventModel, TaskEventSchema
-from crc.services.workflow_service import WorkflowService
+
+from io import BytesIO
+
+import json
 
 
 class TestFileDatastore(BaseTest):
 
-
     def test_file_datastore_workflow(self):
         self.load_example_data()
         self.create_reference_document()
@@ -32,3 +31,41 @@ class TestFileDatastore(BaseTest):
         self.assertEqual(task_data['output'], 'me')
         self.assertEqual(task_data['output2'], 'nope')
 
+    def test_file_data_store_file_data_property(self):
+        self.load_example_data()
+        workflow = self.create_workflow('enum_file_data')
+        workflow_api = self.get_workflow_api(workflow)
+        task = workflow_api.next_task
+
+        # upload the file
+        correct_name = task.form['fields'][1]['id']
+        data = {'file': (BytesIO(b"abcdef"), 'test_file.txt')}
+        rv = self.app.post('/v1.0/file?study_id=%i&workflow_id=%s&task_id=%s&form_field_key=%s' %
+                           (workflow.study_id, workflow.id, task.id, correct_name), data=data, follow_redirects=True,
+                           content_type='multipart/form-data', headers=self.logged_in_headers())
+        self.assert_success(rv)
+        file_id = json.loads(rv.get_data())['id']
+
+        # process the form that sets the datastore values
+        self.complete_form(workflow, task, {'Study_App_Doc': {'id': file_id},
+                                            'IRB_HSR_Application_Type': {'label': 'Expedited Application'},
+                                            'my_test_field': 'some string',
+                                            'the_number': 8,
+                                            'a_boolean': True,
+                                            'some_date': '2021-07-23'})
+
+        # assert the data_store was set correctly
+        data_store_keys = ['IRB_HSR_Application_Type', 'my_test_field', 'the_number', 'a_boolean', 'some_date']
+        data_store = db.session.query(DataStoreModel).filter(DataStoreModel.file_id==file_id).all()
+        for item in data_store:
+            self.assertIn(item.key, data_store_keys)
+            if item.key == 'IRB_HSR_Application_Type':
+                self.assertEqual('Expedited Application', item.value)
+            if item.key == 'my_test_field':
+                self.assertEqual('some string', item.value)
+            if item.key == 'the_number':
+                self.assertEqual('8', item.value)
+            if item.key == 'a_boolean':
+                self.assertEqual('true', item.value)
+            if item.key == 'some_date':
+                self.assertEqual('2021-07-23', item.value)

tests/test_ldap_service.py

@@ -31,3 +31,8 @@ class TestLdapService(BaseTest):
             self.assertFalse(True, "An API error should be raised.")
         except ApiError as ae:
             self.assertEqual("missing_ldap_record", ae.code)
+
+    def test_get_user_with_caps(self):
+        user_info = LdapService.user_info("LB3DP")
+        self.assertIsNotNone(user_info)
+        self.assertEqual("lb3dp", user_info.uid)

tests/test_workflow_api.py

@@ -25,3 +25,40 @@ class TestWorkflowApi(BaseTest):
                           content_type="application/json",
                           headers=self.logged_in_headers())
         self.assert_success(rv)
+
+
+    def test_library_code(self):
+        self.load_example_data()
+        spec1 = ExampleDataLoader().create_spec('hello_world', 'Hello World', category_id=0, library=False,
+                                               from_tests=True)
+
+        spec2 = ExampleDataLoader().create_spec('hello_world_lib', 'Hello World Library', category_id=0, library=True,
+                                               from_tests=True)
+        user = session.query(UserModel).first()
+        self.assertIsNotNone(user)
+
+        rv = self.app.post(f'/v1.0/workflow-specification/%s/library/%s'%(spec1.id,spec2.id),
+                          follow_redirects=True,
+                          content_type="application/json",
+                          headers=self.logged_in_headers())
+        self.assert_success(rv)
+
+        rv = self.app.get(f'/v1.0/workflow-specification/%s'%spec1.id,follow_redirects=True,
+                          content_type="application/json",
+                          headers=self.logged_in_headers())
+        returned=rv.json
+        self.assertIsNotNone(returned.get('libraries'))
+        self.assertEqual(len(returned['libraries']),1)
+        self.assertEqual(returned['libraries'][0].get('id'),'hello_world_lib')
+        rv = self.app.delete(f'/v1.0/workflow-specification/%s/library/%s'%(spec1.id,spec2.id),follow_redirects=True,
+                          content_type="application/json",
+                          headers=self.logged_in_headers())
+        rv = self.app.get(f'/v1.0/workflow-specification/%s'%spec1.id,follow_redirects=True,
+                          content_type="application/json",
+                          headers=self.logged_in_headers())
+        returned=rv.json
+        self.assertIsNotNone(returned.get('libraries'))
+        self.assertEqual(len(returned['libraries']),0)
+
+
+

tests/workflow/test_workflow_call_activity_end_event.py

@@ -0,0 +1,23 @@
+from tests.base_test import BaseTest
+
+
+class TestCallActivityEndEvent(BaseTest):
+
+    def test_call_activity_end_event(self):
+        workflow = self.create_workflow('call_activity_end_event')
+        workflow_api = self.get_workflow_api(workflow)
+        first_task = workflow_api.next_task
+
+        # The tests looks at Element Documentation
+        # The actual end event has 'Main Workflow'
+        # The call activity has 'Call Event'
+
+        # Make sure we have the correct end event,
+        # and not the end event from the call activity
+
+        # This should fail
+        with self.assertRaises(AssertionError):
+            self.assertIn('Call Event', first_task.documentation)
+
+        # This should pass
+        self.assertIn('Main Workflow', first_task.documentation)

tests/workflow/test_workflow_spec_api.py

@@ -109,13 +109,13 @@ class TestWorkflowSpec(BaseTest):
         category = session.query(WorkflowSpecCategoryModel).first()
         ExampleDataLoader().create_spec('hello_world', 'Hello World', category_id=category.id,
                                         standalone=True, from_tests=True)
-        rv = self.app.get('/v1.0/workflow-specification/standalone', headers=self.logged_in_headers())
+        rv = self.app.get('/v1.0/workflow-specification?standalone=true', headers=self.logged_in_headers())
         self.assertEqual(1, len(rv.json))
 
         ExampleDataLoader().create_spec('email_script', 'Email Script', category_id=category.id,
                                         standalone=True, from_tests=True)
 
-        rv = self.app.get('/v1.0/workflow-specification/standalone', headers=self.logged_in_headers())
+        rv = self.app.get('/v1.0/workflow-specification?standalone=true', headers=self.logged_in_headers())
         self.assertEqual(2, len(rv.json))
 
     def test_get_workflow_from_workflow_spec(self):