sartography
/
cr-connect-workflow
mirror of https://github.com/sartography/cr-connect-workflow.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/feature/reference_files' into feature/workflow_spec_categories

This commit is contained in:
Aaron Louie 2020-03-18 16:58:57 -04:00
parent f0678b43d4 02be8ede75
commit 9a26fc8e80
8 changed files with 234 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
crc/api.yml
View File

@ -538,6 +538,67 @@ paths:
format: binary
example: '<?xml version="1.0" encoding="UTF-8"?><bpmn:definitions></bpmn:definitions>'
# /v1.0/workflow/0
/reference_file:
get:
operationId: crc.api.file.get_reference_files
summary: Provides a list of existing reference files that are available in the system.
tags:
- Files
responses:
'200':
description: An array of file descriptions (not the file content)
content:
application/json:
schema:
type: array
items:
$ref: "#/components/schemas/File"
/reference_file/{name}:
parameters:
- name: name
in: path
required: true
description: The special name of the reference file.
schema:
type: string
get:
operationId: crc.api.file.get_reference_file
summary: Reference files are called by name rather than by id.
tags:
- Files
responses:
'200':
description: Returns the actual file
content:
application/octet-stream:
schema:
type: string
format: binary
example: '<?xml version="1.0" encoding="UTF-8"?><bpmn:definitions></bpmn:definitions>'
put:
operationId: crc.api.file.set_reference_file
summary: Update the contents of a named reference file.
tags:
- Files
requestBody:
content:
multipart/form-data:
schema:
type: object
properties:
file:
type: string
format: binary
responses:
'200':
description: Returns the actual file
content:
application/octet-stream:
schema:
type: string
format: binary
example: '<?xml version="1.0" encoding="UTF-8"?><bpmn:definitions></bpmn:definitions>'
# /v1.0/workflow/0
/workflow/{workflow_id}:
parameters:
- name: workflow_id

41
crc/api/file.py
View File

@ -1,3 +1,4 @@
import enum
import io
import os
from datetime import datetime
@ -21,6 +22,10 @@ def get_files(workflow_spec_id=None, study_id=None, workflow_id=None, task_id=No
results = FileService.get_files(workflow_spec_id, study_id, workflow_id, task_id, form_field_key)
return FileModelSchema(many=True).dump(results)
def get_reference_files():
results = FileService.get_files(is_reference=True)
return FileModelSchema(many=True).dump(results)
def add_file(workflow_spec_id=None, study_id=None, workflow_id=None, task_id=None, form_field_key=None):
all_none = all(v is None for v in [workflow_spec_id, study_id, workflow_id, task_id, form_field_key])
@ -43,6 +48,42 @@ def add_file(workflow_spec_id=None, study_id=None, workflow_id=None, task_id=Non
return FileModelSchema().dump(file_model)
def get_reference_file(name):
file_data = FileService.get_reference_file_data(name)
return send_file(
io.BytesIO(file_data.data),
attachment_filename=file_data.file_model.name,
mimetype=file_data.file_model.content_type,
cache_timeout=-1 # Don't cache these files on the browser.
)
def set_reference_file(name):
"""Uses the file service to manage reference-files. They will be used in script tasks to compute values."""
if 'file' not in connexion.request.files:
raise ApiError('invalid_file',
'Expected a file named "file" in the multipart form request', status_code=400)
file = connexion.request.files['file']
name_extension = FileService.get_extension(name)
file_extension = FileService.get_extension(file.filename)
if name_extension != file_extension:
raise ApiError('invalid_file_type',
"The file you uploaded has an extension '%s', but it should have an extension of '%s' " %
(file_extension, name_extension))
file_models = FileService.get_files(name=name, is_reference=True)
if len(file_models) == 0:
file_model = FileService.add_reference_file(name, file.content_type, file.stream.read())
else:
file_model = file_models[0]
FileService.update_file(file_models[0], file.stream.read(), file.content_type)
return FileModelSchema().dump(file_model)
def update_file_data(file_id):
file_model = session.query(FileModel).filter_by(id=file_id).with_for_update().first()
file = connexion.request.files['file']

1
crc/models/api_models.py
View File

@ -107,6 +107,7 @@ class WorkflowApi(object):
self.is_latest_spec = is_latest_spec
self.is_active = is_active
class WorkflowApiSchema(ma.Schema):
class Meta:
model = WorkflowApi

8
crc/models/file.py
View File

@ -66,6 +66,12 @@ class FileDataModel(db.Model):
class FileModel(db.Model):
"""A file model defines one of the following increasingly specific types:
* A Reference file. Which just has a name and a reference flag set to true. These are global, and available everywhere.
* A Workflow Specification (such as BPMN or DMN model or a template)
* A Script generated file in a workflow. Which is specific to a study, workflow and task.
* An Uploaded file in a workflow. specific to a study, workflow, task, AND a field value.
"""
__tablename__ = 'file'
id = db.Column(db.Integer, primary_key=True)
name = db.Column(db.String)
@ -73,6 +79,8 @@ class FileModel(db.Model):
primary = db.Column(db.Boolean)
is_status = db.Column(db.Boolean)
content_type = db.Column(db.String)
is_reference = db.Column(db.Boolean, nullable=False, default=False) # A global reference file.
primary = db.Column(db.Boolean) # Is this the primary BPMN in a workflow?
workflow_spec_id = db.Column(db.String, db.ForeignKey('workflow_spec.id'), nullable=True)
workflow_id = db.Column(db.Integer, db.ForeignKey('workflow.id'), nullable=True)
study_id = db.Column(db.Integer, db.ForeignKey('study.id'), nullable=True)

52
crc/services/file_service.py
View File

@ -1,14 +1,13 @@
import hashlib
import os
from datetime import datetime
from uuid import UUID
from xml.etree import ElementTree
from crc import session
from crc.api.common import ApiErrorSchema, ApiError
from crc.models.file import FileType, FileDataModel, FileModelSchema, FileModel, CONTENT_TYPES
from crc.api.common import ApiError
from crc.models.file import FileType, FileDataModel, FileModel
from crc.models.workflow import WorkflowSpecModel
from crc.services.workflow_processor import WorkflowProcessor
import hashlib
class FileService(object):
@ -55,6 +54,21 @@ class FileService(object):
)
return FileService.update_file(file_model, binary_data, content_type)
@staticmethod
def add_reference_file(name, content_type, binary_data):
"""Create a file with the given name, but not associated with a spec or workflow.
Only one file with the given reference name can exist."""
file_model = FileModel(
name=name,
is_reference=True
)
return FileService.update_file(file_model, binary_data, content_type)
@staticmethod
def get_extension(file_name):
basename, file_extension = os.path.splitext(file_name)
return file_extension.lower().strip()[1:]
@staticmethod
def update_file(file_model, binary_data, content_type):
@ -68,12 +82,11 @@ class FileService(object):
return file_model
# Verify the extension
basename, file_extension = os.path.splitext(file_model.name)
file_extension = file_extension.lower().strip()[1:]
file_extension = FileService.get_extension(file_model.name)
if file_extension not in FileType._member_names_:
return ApiErrorSchema().dump(ApiError('unknown_extension',
raise ApiError('unknown_extension',
'The file you provided does not have an accepted extension:' +
file_extension)), 404
file_extension, status_code=404)
else:
file_model.type = FileType[file_extension]
file_model.content_type = content_type
@ -93,8 +106,10 @@ class FileService(object):
return file_model
@staticmethod
def get_files(workflow_spec_id=None, study_id=None, workflow_id=None, task_id=None, form_field_key=None):
query = session.query(FileModel)
def get_files(workflow_spec_id=None,
study_id=None, workflow_id=None, task_id=None, form_field_key=None,
name=None, is_reference=False):
query = session.query(FileModel).filter_by(is_reference=is_reference)
if workflow_spec_id:
query = query.filter_by(workflow_spec_id=workflow_spec_id)
if study_id:
@ -105,15 +120,28 @@ class FileService(object):
query = query.filter_by(task_id=str(task_id))
if form_field_key:
query = query.filter_by(form_field_key=form_field_key)
if name:
query = query.filter_by(name=form_field_key)
results = query.all()
return results
@staticmethod
def get_file_data(file_id):
"""Returns the file_data that is associated with the file model id"""
def get_file_data(file_id, file_model=None):
"""Returns the file_data that is associated with the file model id, if an actual file_model
is provided, uses that rather than looking it up again."""
if file_model is None:
file_model = session.query(FileModel).filter(FileModel.id == file_id).first()
return session.query(FileDataModel) \
.filter(FileDataModel.file_model_id == file_id) \
.filter(FileDataModel.version == file_model.latest_version) \
.first()
@staticmethod
def get_reference_file_data(file_name):
file_model = session.query(FileModel). \
filter(FileModel.is_reference == True). \
filter(FileModel.name == file_name).first()
if not file_model:
raise ApiError("file_not_found", "There is no reference file with the name '%s'" % file_name)
return FileService.get_file_data(file_model.id, file_model)

28
migrations/versions/0c8a2f8db28c_.py Normal file
View File

@ -0,0 +1,28 @@
"""empty message
Revision ID: 0c8a2f8db28c
Revises: 8856126b6658
Create Date: 2020-03-13 14:05:46.983484
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '0c8a2f8db28c'
down_revision = '8856126b6658'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('file', sa.Column('is_reference', sa.Boolean(), nullable=False))
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column('file', 'is_reference')
# ### end Alembic commands ###

BIN
tests/data/reference/irb_documents.xlsx Normal file
View File

Binary file not shown.

51
tests/test_files_api.py
View File

@ -1,9 +1,8 @@
import io
import json
from datetime import datetime
from crc import session
from crc.models.file import FileModel, FileType, FileModelSchema, FileDataModel
from crc.models.file import FileModel, FileType, FileModelSchema
from crc.models.workflow import WorkflowSpecModel
from tests.base_test import BaseTest
@ -58,6 +57,52 @@ class TestFilesApi(BaseTest):
file2 = FileModelSchema().load(json_data, session=session)
self.assertEqual(file, file2)
def test_set_reference_file(self):
file_name = "irb_document_types.xls"
data = {'file': (io.BytesIO(b"abcdef"), "does_not_matter.xls")}
rv = self.app.put('/v1.0/reference_file/%s' % file_name, data=data, follow_redirects=True,
content_type='multipart/form-data')
self.assert_success(rv)
self.assertIsNotNone(rv.get_data())
json_data = json.loads(rv.get_data(as_text=True))
file = FileModelSchema().load(json_data, session=session)
self.assertEqual(FileType.xls, file.type)
self.assertTrue(file.is_reference)
self.assertEqual("application/vnd.ms-excel", file.content_type)
def test_set_reference_file_bad_extension(self):
file_name = "irb_document_types.xls"
data = {'file': (io.BytesIO(b"abcdef"), "does_not_matter.ppt")}
rv = self.app.put('/v1.0/reference_file/%s' % file_name, data=data, follow_redirects=True,
content_type='multipart/form-data')
self.assert_failure(rv, error_code="invalid_file_type")
def test_get_reference_file(self):
file_name = "irb_document_types.xls"
data = {'file': (io.BytesIO(b"abcdef"), "some crazy thing do not care.xls")}
rv = self.app.put('/v1.0/reference_file/%s' % file_name, data=data, follow_redirects=True,
content_type='multipart/form-data')
rv = self.app.get('/v1.0/reference_file/%s' % file_name)
self.assert_success(rv)
data_out = rv.get_data()
self.assertEqual(b"abcdef", data_out)
def test_list_reference_files(self):
file_name = "irb_document_types.xls"
data = {'file': (io.BytesIO(b"abcdef"), file_name)}
rv = self.app.put('/v1.0/reference_file/%s' % file_name, data=data, follow_redirects=True,
content_type='multipart/form-data')
rv = self.app.get('/v1.0/reference_file',
follow_redirects=True,
content_type="application/json")
self.assert_success(rv)
json_data = json.loads(rv.get_data(as_text=True))
self.assertEqual(1, len(json_data))
file = FileModelSchema(many=True).load(json_data, session=session)
self.assertEqual(file_name, file[0].name)
self.assertTrue(file[0].is_reference)
def test_update_file_info(self):
self.load_example_data()
file: FileModel = session.query(FileModel).first()
@ -118,7 +163,6 @@ class TestFilesApi(BaseTest):
file = FileModelSchema().load(json_data, session=session)
self.assertEqual(1, file.latest_version)
def test_get_file(self):
self.load_example_data()
spec = session.query(WorkflowSpecModel).first()
@ -137,3 +181,4 @@ class TestFilesApi(BaseTest):
rv = self.app.delete('/v1.0/file/%i' % file.id)
rv = self.app.get('/v1.0/file/%i' % file.id)
self.assertEqual(404, rv.status_code)