Merge branch 'dev' into dmn-from-spreadsheet-395

Pipfile

@@ -10,7 +10,6 @@ coverage = "*"
 
 [packages]
 alembic = "*"
-connexion = {extras = ["swagger-ui"],version = "*"}
 coverage = "*"
 docxtpl = "*"
 flask = "*"
@@ -36,11 +35,8 @@ pyjwt = "*"
 python-dateutil = "*"
 recommonmark = "*"
 requests = "*"
-sentry-sdk = {extras = ["flask"],version = "==0.14.4"}
 sphinx = "*"
 swagger-ui-bundle = "*"
-spiffworkflow = {git = "https://github.com/sartography/SpiffWorkflow.git"}
-# spiffworkflow = {editable = true, path = "./../SpiffWorkflow"}
 webtest = "*"
 werkzeug = "*"
 xlrd = "*"
@@ -50,3 +46,14 @@ apscheduler = "*"
 
 [requires]
 python_version = "3.8"
+
+[packages.connexion]
+extras = [ "swagger-ui",]
+version = "*"
+
+[packages.sentry-sdk]
+extras = [ "flask",]
+version = "==0.14.4"
+
+[packages.spiffworkflow]
+git = "https://github.com/sartography/SpiffWorkflow.git"

crc/api.yml

@@ -400,6 +400,19 @@ paths:
       summary: Provides a list of workflows specifications that can be added to a study manually. Please note that Protocol Builder will handle this most of the time.
       tags:
         - Workflow Specifications
+      parameters :
+        - name : libraries
+          in : query
+          required : false
+          description : True if we should return just library schemas
+          schema :
+            type : boolean
+        - name : standalone
+          in : query
+          required : false
+          description : True if we should return just standalone schemas
+          schema :
+            type : boolean
       responses:
         '200':
           description: An array of workflow specifications
@@ -426,6 +439,50 @@ paths:
             application/json:
               schema:
                 $ref: "#/components/schemas/WorkflowSpec"
+
+
+  /workflow-specification/{spec_id}/library/{library_id}:
+    parameters:
+      - name: spec_id
+        in: path
+        required: true
+        description: The unique id of an existing workflow specification.
+        schema:
+          type: string
+      - name: library_id
+        in: path
+        required: true
+        description: The unique id of an existing library specification.
+        schema:
+          type: string
+
+
+    post:
+      operationId: crc.api.workflow.add_workflow_spec_library
+      summary: Adds a library to a workflow spec
+      tags:
+        - Workflow Specifications
+      responses:
+        '200':
+          description: Workflow specification.
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/WorkflowSpec"
+    delete:
+      operationId: crc.api.workflow.drop_workflow_spec_library
+      summary: Delete a library from a workflow
+      tags:
+        - Workflow Specifications
+      responses:
+        '200':
+          description: Workflow specification.
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/WorkflowSpec"
+
+
   /workflow-specification/{spec_id}:
     parameters:
       - name: spec_id
@@ -487,21 +544,6 @@ paths:
       responses:
         '204':
           description: The workflow specification has been removed.
-  /workflow-specification/standalone:
-    get:
-      operationId: crc.api.workflow.standalone_workflow_specs
-      summary: Provides a list of workflow specifications that can be run outside a study.
-      tags:
-        - Workflow Specifications
-      responses:
-        '200':
-          description: A list of workflow specifications
-          content:
-            application/json:
-              schema:
-                type: array
-                items:
-                  $ref: "#/components/schemas/WorkflowSpec"
   /workflow-specification/{spec_id}/validate:
     parameters:
       - name: spec_id

crc/api/workflow.py

@@ -10,7 +10,7 @@ from crc.models.study import StudyModel, WorkflowMetadata, StudyStatus
 from crc.models.task_event import TaskEventModel, TaskEvent, TaskEventSchema
 from crc.models.user import UserModelSchema
 from crc.models.workflow import WorkflowModel, WorkflowSpecModelSchema, WorkflowSpecModel, WorkflowSpecCategoryModel, \
-    WorkflowSpecCategoryModelSchema
+    WorkflowSpecCategoryModelSchema, WorkflowLibraryModel, WorkflowLibraryModelSchema
 from crc.services.error_service import ValidationErrorService
 from crc.services.file_service import FileService
 from crc.services.lookup_service import LookupService
@@ -20,9 +20,22 @@ from crc.services.workflow_processor import WorkflowProcessor
 from crc.services.workflow_service import WorkflowService
 
 
-def all_specifications():
+def all_specifications(libraries=False,standalone=False):
+    if libraries and standalone:
+        raise ApiError('inconceivable!', 'You should specify libraries or standalone, but not both')
     schema = WorkflowSpecModelSchema(many=True)
-    return schema.dump(session.query(WorkflowSpecModel).all())
+    if libraries:
+        return schema.dump(session.query(WorkflowSpecModel)\
+              .filter(WorkflowSpecModel.library==True).all())
+
+    if standalone:
+        return schema.dump(session.query(WorkflowSpecModel)\
+              .filter(WorkflowSpecModel.standalone==True).all())
+    # this still returns standalone workflow specs as well, but by default
+    # we do not return specs marked as library
+    return schema.dump(session.query(WorkflowSpecModel)\
+              .filter((WorkflowSpecModel.library==False)|(
+              WorkflowSpecModel.library==None)).all())
 
 
 def add_workflow_specification(body):
@@ -45,6 +58,41 @@ def get_workflow_specification(spec_id):
 
     return WorkflowSpecModelSchema().dump(spec)
 
+def validate_spec_and_library(spec_id,library_id):
+    if spec_id is None:
+        raise ApiError('unknown_spec', 'Please provide a valid Workflow Specification ID.')
+    if library_id is None:
+        raise ApiError('unknown_spec', 'Please provide a valid Library Specification ID.')
+    spec: WorkflowSpecModel = session.query(WorkflowSpecModel).filter_by(id=spec_id).first()
+    library: WorkflowSpecModel = session.query(WorkflowSpecModel).filter_by(id=library_id).first()
+    if spec is None:
+        raise ApiError('unknown_spec', 'The Workflow Specification "' + spec_id + '" is not recognized.')
+    if library is None:
+        raise ApiError('unknown_spec', 'The Library Specification "' + library_id + '" is not recognized.')
+    if not library.library:
+        raise ApiError('unknown_spec', 'Linked workflow spec is not a library.')
+
+
+def add_workflow_spec_library(spec_id,library_id):
+    validate_spec_and_library(spec_id, library_id)
+    libraries: WorkflowLibraryModel = session.query(WorkflowLibraryModel).filter_by(workflow_spec_id=spec_id).all()
+    libraryids = [x.library_spec_id for x in libraries]
+    if library_id in libraryids:
+        raise ApiError('unknown_spec', 'The Library Specification "' + spec_id + '" is already attached.')
+    newlib = WorkflowLibraryModel()
+    newlib.workflow_spec_id = spec_id
+    newlib.library_spec_id = library_id
+    session.add(newlib)
+    session.commit()
+    libraries: WorkflowLibraryModel = session.query(WorkflowLibraryModel).filter_by(workflow_spec_id=spec_id).all()
+    return WorkflowLibraryModelSchema(many=True).dump(libraries)
+
+def drop_workflow_spec_library(spec_id,library_id):
+    validate_spec_and_library(spec_id, library_id)
+    session.query(WorkflowLibraryModel).filter_by(workflow_spec_id=spec_id,library_spec_id=library_id).delete()
+    session.commit()
+    libraries: WorkflowLibraryModel = session.query(WorkflowLibraryModel).filter_by(workflow_spec_id=spec_id).all()
+    return WorkflowLibraryModelSchema(many=True).dump(libraries)
 
 def validate_workflow_specification(spec_id, study_id=None, test_until=None):
     try:
@@ -106,12 +154,6 @@ def get_workflow_from_spec(spec_id):
     return WorkflowApiSchema().dump(workflow_api_model)
 
 
-def standalone_workflow_specs():
-    schema = WorkflowSpecModelSchema(many=True)
-    specs = WorkflowService.get_standalone_workflow_specs()
-    return schema.dump(specs)
-
-
 def get_workflow(workflow_id, do_engine_steps=True):
     """Retrieve workflow based on workflow_id, and return it in the last saved State.
        If do_engine_steps is False, return the workflow without running any engine tasks or logging any events. """

crc/models/workflow.py

@@ -1,7 +1,7 @@
 import enum
 
 import marshmallow
-from marshmallow import EXCLUDE
+from marshmallow import EXCLUDE,fields
 from marshmallow_sqlalchemy import SQLAlchemyAutoSchema
 from sqlalchemy import func
 
@@ -24,6 +24,7 @@ class WorkflowSpecCategoryModelSchema(SQLAlchemyAutoSchema):
         include_relationships = True
 
 
+
 class WorkflowSpecModel(db.Model):
     __tablename__ = 'workflow_spec'
     id = db.Column(db.String, primary_key=True)
@@ -35,6 +36,19 @@ class WorkflowSpecModel(db.Model):
     category = db.relationship("WorkflowSpecCategoryModel")
     is_master_spec = db.Column(db.Boolean, default=False)
     standalone = db.Column(db.Boolean, default=False)
+    library = db.Column(db.Boolean, default=False)
+
+
+class WorkflowLibraryModel(db.Model):
+   __tablename__ = 'workflow_library'
+   id = db.Column(db.Integer, primary_key=True)
+   workflow_spec_id = db.Column(db.String, db.ForeignKey('workflow_spec.id'), nullable=True)
+   library_spec_id = db.Column(db.String, db.ForeignKey('workflow_spec.id'), nullable=True)
+   parent = db.relationship(WorkflowSpecModel,
+                                   primaryjoin=workflow_spec_id==WorkflowSpecModel.id,
+                                  backref='libraries')
+   library = db.relationship(WorkflowSpecModel,primaryjoin=library_spec_id==WorkflowSpecModel.id,
+                                  backref='parents')
 
 
 class WorkflowSpecModelSchema(SQLAlchemyAutoSchema):
@@ -46,7 +60,14 @@ class WorkflowSpecModelSchema(SQLAlchemyAutoSchema):
         unknown = EXCLUDE
 
     category = marshmallow.fields.Nested(WorkflowSpecCategoryModelSchema, dump_only=True)
-
+    libraries = marshmallow.fields.Function(lambda obj: [{'id':x.library.id,
+                                                          'name':x.library.name,
+                                                          'display_name':x.library.display_name} for x in
+                                                          obj.libraries] )
+    parents = marshmallow.fields.Function(lambda obj: [{'id':x.parent.id,
+                                                          'name':x.parent.name,
+                                                          'display_name':x.parent.display_name} for x in
+                                                          obj.parents] )
 
 class WorkflowState(enum.Enum):
     hidden = "hidden"
@@ -78,6 +99,15 @@ class WorkflowSpecDependencyFile(db.Model):
     file_data = db.relationship(FileDataModel)
 
 
+
+class WorkflowLibraryModelSchema(SQLAlchemyAutoSchema):
+    class Meta:
+        model = WorkflowLibraryModel
+        load_instance = True
+        include_relationships = True
+
+    library = marshmallow.fields.Nested('WorkflowSpecModelSchema')
+
 class WorkflowModel(db.Model):
     __tablename__ = 'workflow'
     id = db.Column(db.Integer, primary_key=True)

crc/services/workflow_service.py

@@ -898,6 +898,11 @@ class WorkflowService(object):
         specs = db.session.query(WorkflowSpecModel).filter_by(standalone=True).all()
         return specs
 
+    @staticmethod
+    def get_library_workflow_specs():
+        specs = db.session.query(WorkflowSpecModel).filter_by(library=True).all()
+        return specs
+
     @staticmethod
     def get_primary_workflow(workflow_spec_id):
         # Returns the FileModel of the primary workflow for a workflow_spec

docker_run.sh

@@ -13,14 +13,10 @@ if [ "$UPGRADE_DB" = "true" ]; then
   pipenv run flask db upgrade
 fi
 
-# This is commented to keep humans from clearing the db accidentally.
+if [ "$RESET_DB" = "true" ]; then
-# If you need to reset the db, you can uncomment,
+  echo 'Resetting database and seeding it with example CR Connect data...'
-# then set RESET_DB to true in config, push code, and pray
+  pipenv run flask load-example-data
-
+fi
-#if [ "$RESET_DB" = "true" ]; then
-#  echo 'Resetting database and seeding it with example CR Connect data...'
-#  pipenv run flask load-example-data
-#fi
 
 if [ "$RESET_DB_RRT" = "true" ]; then
   echo 'Resetting database and seeding it with example RRT data...'

example_data.py

@@ -16,7 +16,10 @@ class ExampleDataLoader:
     @staticmethod
     def clean_db():
         session.flush()  # Clear out any transactions before deleting it all to avoid spurious errors.
+        engine = session.bind.engine
+        connection = engine.connect()
         for table in reversed(db.metadata.sorted_tables):
+            if engine.dialect.has_table(connection, table):
                 session.execute(table.delete())
         session.commit()
         session.flush()
@@ -268,7 +271,7 @@ class ExampleDataLoader:
                          from_tests=True)
 
     def create_spec(self, id, name, display_name="", description="", filepath=None, master_spec=False,
-                    category_id=None, display_order=None, from_tests=False, standalone=False):
+                    category_id=None, display_order=None, from_tests=False, standalone=False, library=False):
         """Assumes that a directory exists in static/bpmn with the same name as the given id.
            further assumes that the [id].bpmn is the primary file for the workflow.
            returns an array of data models to be added to the database."""
@@ -281,7 +284,8 @@ class ExampleDataLoader:
                                  is_master_spec=master_spec,
                                  category_id=category_id,
                                  display_order=display_order,
-                                 standalone=standalone)
+                                 standalone=standalone,
+                                 library=library)
         db.session.add(spec)
         db.session.commit()
         if not filepath and not from_tests:

migrations/versions/2a6f7ea00e5f_.py

@@ -0,0 +1,37 @@
+"""empty message
+
+Revision ID: 2a6f7ea00e5f
+Revises: dc30b8f6571c
+Create Date: 2021-07-29 10:51:56.857341
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '2a6f7ea00e5f'
+down_revision = 'dc30b8f6571c'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('workflow_library',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('workflow_spec_id', sa.String(), nullable=True),
+    sa.Column('library_spec_id', sa.String(), nullable=True),
+    sa.ForeignKeyConstraint(['library_spec_id'], ['workflow_spec.id'], ),
+    sa.ForeignKeyConstraint(['workflow_spec_id'], ['workflow_spec.id'], ),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.add_column('workflow_spec', sa.Column('library', sa.Boolean(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('workflow_spec', 'library')
+    op.drop_table('workflow_library')
+    # ### end Alembic commands ###

migrations/versions/c16d3047abbe_.py

@@ -11,7 +11,7 @@ import sqlalchemy as sa
 
 # revision identifiers, used by Alembic.
 revision = 'c16d3047abbe'
-down_revision = 'bbf064082623'
+down_revision = '30e017a03948'
 branch_labels = None
 depends_on = None
 

tests/test_workflow_api.py

@@ -25,3 +25,41 @@ class TestWorkflowApi(BaseTest):
                           content_type="application/json",
                           headers=self.logged_in_headers())
         self.assert_success(rv)
+
+
+    def test_library_code(self):
+        self.load_example_data()
+        spec1 = ExampleDataLoader().create_spec('hello_world', 'Hello World', category_id=0, library=False,
+                                               from_tests=True)
+
+        spec2 = ExampleDataLoader().create_spec('hello_world_lib', 'Hello World Library', category_id=0, library=True,
+                                               from_tests=True)
+        user = session.query(UserModel).first()
+        self.assertIsNotNone(user)
+        #WorkflowService.get_workflow_from_spec(spec.id, user)
+
+        rv = self.app.post(f'/v1.0/workflow-specification/%s/library/%s'%(spec1.id,spec2.id),
+                          follow_redirects=True,
+                          content_type="application/json",
+                          headers=self.logged_in_headers())
+        self.assert_success(rv)
+
+        rv = self.app.get(f'/v1.0/workflow-specification/%s'%spec1.id,follow_redirects=True,
+                          content_type="application/json",
+                          headers=self.logged_in_headers())
+        returned=rv.json
+        self.assertIsNotNone(returned.get('libraries'))
+        self.assertEqual(len(returned['libraries']),1)
+        self.assertEqual(returned['libraries'][0].get('id'),'hello_world_lib')
+        rv = self.app.delete(f'/v1.0/workflow-specification/%s/library/%s'%(spec1.id,spec2.id),follow_redirects=True,
+                          content_type="application/json",
+                          headers=self.logged_in_headers())
+        rv = self.app.get(f'/v1.0/workflow-specification/%s'%spec1.id,follow_redirects=True,
+                          content_type="application/json",
+                          headers=self.logged_in_headers())
+        returned=rv.json
+        self.assertIsNotNone(returned.get('libraries'))
+        self.assertEqual(len(returned['libraries']),0)
+
+
+

tests/workflow/test_workflow_spec_api.py

@@ -109,13 +109,13 @@ class TestWorkflowSpec(BaseTest):
         category = session.query(WorkflowSpecCategoryModel).first()
         ExampleDataLoader().create_spec('hello_world', 'Hello World', category_id=category.id,
                                         standalone=True, from_tests=True)
-        rv = self.app.get('/v1.0/workflow-specification/standalone', headers=self.logged_in_headers())
+        rv = self.app.get('/v1.0/workflow-specification?standalone=true', headers=self.logged_in_headers())
         self.assertEqual(1, len(rv.json))
 
         ExampleDataLoader().create_spec('email_script', 'Email Script', category_id=category.id,
                                         standalone=True, from_tests=True)
 
-        rv = self.app.get('/v1.0/workflow-specification/standalone', headers=self.logged_in_headers())
+        rv = self.app.get('/v1.0/workflow-specification?standalone=true', headers=self.logged_in_headers())
         self.assertEqual(2, len(rv.json))
 
     def test_get_workflow_from_workflow_spec(self):